From 8d9f62c40908b11991bd780fa6820bdce88c2063 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 07:40:18 -0400 Subject: [PATCH 001/255] Preserve complete Frame metadata in graph storage --- .../20260906-frame-metadata-storage.fixed.md | 1 + .../src/microcosm/graph/store.py | 103 +++++++++++++++++- .../tests/test_frame_metadata_store.py | 101 +++++++++++++++++ 3 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20260906-frame-metadata-storage.fixed.md create mode 100644 packages/microcosm-graph/tests/test_frame_metadata_store.py diff --git a/changelog.d/20260906-frame-metadata-storage.fixed.md b/changelog.d/20260906-frame-metadata-storage.fixed.md new file mode 100644 index 000000000..dfa6dd022 --- /dev/null +++ b/changelog.d/20260906-frame-metadata-storage.fixed.md @@ -0,0 +1 @@ +Preserve complete Frame metadata in graph content-store format v2, including nested source evidence, and refuse older frame caches that cannot restore it or reuse of one frame key with different metadata. diff --git a/packages/microcosm-graph/src/microcosm/graph/store.py b/packages/microcosm-graph/src/microcosm/graph/store.py index 6f93b8b65..5d0fa17fd 100644 --- a/packages/microcosm-graph/src/microcosm/graph/store.py +++ b/packages/microcosm-graph/src/microcosm/graph/store.py @@ -62,7 +62,7 @@ type ResumePolicy = Literal["auto", "require", "forbid"] _STORE_FORMAT = "microcosm-graph-content-store-v1" -_FRAME_FORMAT = "microcosm-graph-frame-v1" +_FRAME_FORMAT = "microcosm-graph-frame-v2" _KEY = re.compile(r"[0-9a-f]{64}\Z") _ENCODING_NUMPY = "numpy-v1" @@ -697,11 +697,14 @@ def _put( build: Callable[[Path], Mapping[str, object]], *, verify_existing: bool = True, + validate_existing: Callable[[Mapping[str, Any]], None] | None = None, ) -> Path: key = _require_key(key) destination = self.object_path(key) if verify_existing and destination.exists(): - _verified_meta(destination, expected_kind=kind) + existing = _verified_meta(destination, expected_kind=kind) + if validate_existing is not None: + validate_existing(existing) return destination staging = self.tmp / uuid.uuid4().hex staging.mkdir(parents=False, exist_ok=False) @@ -729,7 +732,9 @@ def _put( if verify_existing: if not destination.exists(): raise - _verified_meta(destination, expected_kind=kind) + existing = _verified_meta(destination, expected_kind=kind) + if validate_existing is not None: + validate_existing(existing) else: self._replace_write_only_collision(staging, destination) _fsync_directory(destination.parent) @@ -895,12 +900,35 @@ def put_frame( raise TypeError(f"frame must be a Frame, got {type(frame).__name__}.") frame.revalidate() bound_node_key = key if node_key is None else node_key + metadata_sha256 = hashlib.sha256( + _canonical_json(_encode_frame_metadata(frame.metadata)) + ).hexdigest() + + def validate_existing(metadata: Mapping[str, Any]) -> None: + if metadata.get("frame_format") != _FRAME_FORMAT: + raise StoreUnavailable( + "Stored frame predates complete metadata storage." + ) + if metadata.get("frame_metadata_sha256") != metadata_sha256: + raise StoreCorrupt( + "The same frame key cannot carry different metadata." + ) def build(root: Path) -> Mapping[str, object]: _write_frame(root, frame) - return {"frame_format": _FRAME_FORMAT, "node_key": bound_node_key} + return { + "frame_format": _FRAME_FORMAT, + "node_key": bound_node_key, + "frame_metadata_sha256": metadata_sha256, + } - return self._put(key, "frame", build, verify_existing=verify_existing) + return self._put( + key, + "frame", + build, + verify_existing=verify_existing, + validate_existing=validate_existing, + ) write_frame = put_frame @@ -999,6 +1027,61 @@ def _schema_payload(schema: EntitySchema) -> dict[str, object]: } +def _encode_frame_metadata(value: object) -> object: + """Preserve Frame metadata kinds without pickle or lossy scalar coercion.""" + if isinstance(value, Mapping): + return [ + "mapping", + [[key, _encode_frame_metadata(item)] for key, item in value.items()], + ] + if isinstance(value, tuple): + return ["tuple", [_encode_frame_metadata(item) for item in value]] + if isinstance(value, frozenset): + return [ + "frozenset", + sorted( + (_encode_frame_metadata(item) for item in value), key=_canonical_json + ), + ] + if isinstance(value, float): + # Binary float bytes preserve signed zero and any allowed NaN payload. + return ["float64", struct.pack(">d", value).hex()] + if value is None or isinstance(value, (str, int, bool)): + return ["scalar", value] + raise TypeError(f"Unsupported Frame metadata value {type(value).__name__}.") + + +def _decode_frame_metadata(encoded: object) -> object: + try: + if not isinstance(encoded, list) or len(encoded) != 2: + raise ValueError + kind, value = encoded + if kind == "mapping" and isinstance(value, list): + result = {} + for pair in value: + if not isinstance(pair, list) or len(pair) != 2: + raise ValueError + key, item = pair + if not isinstance(key, str) or not key or key in result: + raise ValueError + result[key] = _decode_frame_metadata(item) + return result + if kind in ("tuple", "frozenset") and isinstance(value, list): + items = [_decode_frame_metadata(item) for item in value] + return tuple(items) if kind == "tuple" else frozenset(items) + if ( + kind == "float64" + and isinstance(value, str) + and re.fullmatch(r"[0-9a-f]{16}", value) + ): + return struct.unpack(">d", bytes.fromhex(value))[0] + if kind == "scalar" and (value is None or type(value) in (str, int, bool)): + return value + raise ValueError + except (ValueError, TypeError, KeyError, struct.error) as error: + raise StoreCorrupt("Stored Frame metadata is malformed.") from error + + def _write_frame(root: Path, frame: Frame) -> None: _write_json(root / "schema.json", _schema_payload(frame.schema)) table_specs: list[dict[str, object]] = [] @@ -1053,6 +1136,7 @@ def _write_frame(root: Path, frame: Frame) -> None: "weights": weight_specs, "strata": strata_spec, "mass_log": mass_log, + "metadata": _encode_frame_metadata(frame.metadata), }, ) @@ -1099,6 +1183,14 @@ def _read_frame(path: Path, metadata: Mapping[str, Any]) -> Frame: raw_weights = manifest.get("weights") strata_spec = manifest.get("strata") raw_mass_log = manifest.get("mass_log") + raw_metadata = manifest.get("metadata") + if hashlib.sha256(_canonical_json(raw_metadata)).hexdigest() != metadata.get( + "frame_metadata_sha256" + ): + raise StoreCorrupt("Stored Frame metadata identity is missing or incorrect.") + frame_metadata = _decode_frame_metadata(raw_metadata) + if not isinstance(frame_metadata, Mapping): + raise StoreCorrupt("Stored Frame metadata must be a mapping.") if not all( ( isinstance(raw_tables, list), @@ -1238,6 +1330,7 @@ def _read_frame(path: Path, metadata: Mapping[str, Any]) -> Frame: weights, strata, mass_log=tuple(mass_log), + metadata=frame_metadata, ) except ImportError as error: raise StoreUnavailable( diff --git a/packages/microcosm-graph/tests/test_frame_metadata_store.py b/packages/microcosm-graph/tests/test_frame_metadata_store.py new file mode 100644 index 000000000..29aaa9116 --- /dev/null +++ b/packages/microcosm-graph/tests/test_frame_metadata_store.py @@ -0,0 +1,101 @@ +"""Invented complete Frame metadata persistence and stale codec refusal.""" + +from __future__ import annotations + +import math +import struct + +import numpy as np +import pandas as pd +import pytest + +import microcosm.graph.store as store_module +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.store import ContentStore, StoreCorrupt, StoreUnavailable + + +def _frame(metadata): + return Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [1, 1]} + ), + "household": pd.DataFrame({"household_id": [1]}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([100.0]), WeightKind.DESIGN)}, + metadata=metadata, + ) + + +def test_roundtrip_preserves_complete_nested_source_metadata_and_types(tmp_path): + original = _frame( + { + "us_spine_assembly_manifest": { + "source_arm": "invented", + "sources": ({"sha256": "a" * 64, "rows": 2, "design_total": 100.0},), + "flags": frozenset({"native", "unknown"}), + "nullable": None, + "flag": True, + }, + "signed_zero": -0.0, + "nan": struct.unpack(">d", bytes.fromhex("7ff8000000000011"))[0], + "infinity": float("inf"), + } + ) + store = ContentStore(tmp_path / "invented-store") + store.put_frame("a" * 64, original) + restored = store.load_frame("a" * 64) + assert store_module._encode_frame_metadata( + restored.metadata + ) == store_module._encode_frame_metadata(original.metadata) + assert isinstance(restored.metadata["us_spine_assembly_manifest"]["sources"], tuple) + assert isinstance( + restored.metadata["us_spine_assembly_manifest"]["flags"], frozenset + ) + assert math.copysign(1.0, restored.metadata["signed_zero"]) == -1.0 + assert struct.pack(">d", restored.metadata["nan"]).hex() == "7ff8000000000011" + assert ( + restored.weights_for("household").values.tobytes() + == original.weights_for("household").values.tobytes() + ) + with pytest.raises(TypeError): + restored.metadata["us_spine_assembly_manifest"]["source_arm"] = "changed" + + +def test_same_key_with_changed_metadata_refuses_without_replacing_original(tmp_path): + store = ContentStore(tmp_path / "invented-store") + original = _frame({"source_sha256": "a" * 64}) + store.put_frame("b" * 64, original) + store.put_frame("b" * 64, original) + with pytest.raises(StoreCorrupt, match="different metadata"): + store.put_frame("b" * 64, _frame({"source_sha256": "b" * 64})) + assert store.load_frame("b" * 64).metadata == original.metadata + + +def test_v1_frame_is_unavailable_instead_of_silently_losing_metadata( + tmp_path, monkeypatch +): + store = ContentStore(tmp_path / "invented-store") + with monkeypatch.context() as old: + old.setattr(store_module, "_FRAME_FORMAT", "microcosm-graph-frame-v1") + store.put_frame("c" * 64, _frame({"source": "invented"})) + with pytest.raises(StoreUnavailable, match="codec"): + store.load_frame("c" * 64) + with pytest.raises(StoreUnavailable, match="predates complete metadata"): + store.put_frame("c" * 64, _frame({"source": "invented"})) + + +@pytest.mark.parametrize( + "value", + [ + ["mapping", [["duplicate", ["scalar", 1]], ["duplicate", ["scalar", 2]]]], + ["float64", "bad"], + ["scalar", 1.2], + ["mapping", [["", ["scalar", None]]]], + ["unknown", []], + ], +) +def test_malformed_metadata_refuses(value): + with pytest.raises(StoreCorrupt, match="metadata"): + store_module._decode_frame_metadata(value) From f397e8a7172508b51ec3d649b57767f5b6c1e432 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:14 -0400 Subject: [PATCH 002/255] Start US launch integration source staging ledger --- PROGRESS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/PROGRESS.md b/PROGRESS.md index e528270f7..e85cce8eb 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -726,3 +726,14 @@ The still-earlier PolicyEngine-US 1.819.0 lock-bump lane merged into `origin/main` at `7b90bb18` on 2026-08-24; its final state remains at commit `05d254aa` and its detailed receipts remain in the historical section of `_LANE-NOTES.md`. + +## US launch integration staging — 2026-09-09 + +### State +Source-only staging in progress. Execution and source/data admission remain root-owned. + +### Done +Verified requested clean branch, base HEAD, main ancestry and preservation pins. + +### Next +Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. From 124fd688ea78cf995d5295308f0fe4cc4102850c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:14 -0400 Subject: [PATCH 003/255] Stage integration layer 01: SAFE-ADDITIVE.patch --- PROGRESS.md | 2 + .../microcosm/build/cd_benchmark/__init__.py | 74 + .../microcosm/build/cd_benchmark/canonical.py | 115 + .../microcosm/build/cd_benchmark/origin.py | 186 ++ .../microcosm/build/cd_benchmark/protocol.py | 871 ++++++ .../microcosm/build/cd_benchmark/reasons.py | 135 + .../src/microcosm/build/survey_allocation.py | 202 ++ .../microcosm/build/survey_domain_sample.py | 369 +++ .../src/microcosm/build/table_identity.py | 365 +++ .../us_runtime/_asec_current_money_codec.py | 143 + .../us_runtime/_person_signal_summary.py | 51 + .../build/us_runtime/acs_housing_universe.py | 203 ++ .../us_runtime/acs_housing_universe_source.py | 931 ++++++ .../us_runtime/acs_native_coverage_binding.py | 565 ++++ .../acs_person_coverage_authentication.py | 667 +++++ .../us_runtime/acs_person_coverage_columns.py | 295 ++ .../us_runtime/acs_population_catalogue.py | 579 ++++ .../us_runtime/asec_2024_native_population.py | 652 ++++ .../asec_coverage_authentication.py | 754 +++++ .../build/us_runtime/asec_current_money.py | 1099 +++++++ .../asec_current_money_graph_resources.py | 34 + .../asec_current_money_resources.py | 114 + .../asec_current_money_selection.py | 549 ++++ .../us_runtime/asec_current_money_source.py | 591 ++++ .../us_runtime/asec_current_money_units.py | 411 +++ .../us_runtime/asec_engine_evaluation.py | 473 +++ .../asec_household_coverage_fields.py | 475 +++ .../us_runtime/asec_household_observations.py | 392 +++ .../build/us_runtime/asec_housing_status.py | 474 +++ .../us_runtime/asec_housing_status_source.py | 338 +++ .../build/us_runtime/asec_housing_universe.py | 423 +++ .../asec_housing_universe_source.py | 336 +++ .../us_runtime/asec_income_observations.py | 574 ++++ .../asec_original_household_weights.py | 527 ++++ .../us_runtime/asec_person_coverage_source.py | 299 ++ .../us_runtime/asec_person_income_source.py | 488 +++ .../us_runtime/asec_population_catalogue.py | 507 ++++ .../build/us_runtime/asec_prepared_source.py | 567 ++++ .../build/us_runtime/asec_student_controls.py | 511 ++++ .../build/us_runtime/cd_reference.py | 652 ++++ .../build/us_runtime/cd_reference_sources.py | 188 ++ .../build/us_runtime/cps_carried_current.py | 329 ++ .../demographic_calibration_graph.py | 330 +++ .../build/us_runtime/full_puf_enrichment.py | 528 ++++ .../build/us_runtime/graph_asec_income.py | 665 +++++ .../build/us_runtime/graph_asec_prepared.py | 1136 +++++++ .../build/us_runtime/graph_combined_clone.py | 830 ++++++ .../us_runtime/graph_composed_asec_binding.py | 1400 +++++++++ .../graph_composed_asec_measures.py | 650 ++++ .../us_runtime/graph_composed_contracts.py | 16 + .../us_runtime/graph_composed_population.py | 751 +++++ .../build/us_runtime/graph_context.py | 278 ++ .../graph_current_survey_puf_transfer.py | 407 +++ .../build/us_runtime/graph_geography.py | 380 +++ .../us_runtime/graph_housing_universe.py | 230 ++ .../build/us_runtime/graph_implementation.py | 532 ++++ .../graph_implementation_inventory.json | 2488 ++++++++++++++++ .../us_runtime/graph_national_age_counts.py | 451 +++ .../graph_native_household_origin.py | 415 +++ .../graph_native_origin_implementation.py | 111 + .../us_runtime/graph_puf_detail_transfer.py | 780 +++++ .../graph_puf_diagnostic_consumer.py | 763 +++++ .../build/us_runtime/graph_sources.py | 555 ++++ .../us_runtime/graph_survey_age_artifact.py | 221 ++ .../build/us_runtime/graph_survey_budget.py | 270 ++ .../us_runtime/graph_survey_calibration.py | 417 +++ .../us_runtime/graph_survey_population.py | 1259 ++++++++ .../us_runtime/national_age_activation.py | 648 ++++ .../us_runtime/native_household_origin.py | 666 +++++ .../us_runtime/operator_column_contracts.py | 157 + .../build/us_runtime/puf_detail_transfer.py | 451 +++ .../us_runtime/puf_diagnostic_consumer.py | 846 ++++++ .../microcosm/build/us_runtime/puf_growth.py | 1713 +++++++++++ .../build/us_runtime/puf_growth_graph.py | 375 +++ .../us_runtime/puf_monetary_agi_projection.py | 689 +++++ .../build/us_runtime/puf_monetary_source.py | 2036 +++++++++++++ .../build/us_runtime/puf_price_baseline.py | 1214 ++++++++ .../build/us_runtime/puf_raw_source.py | 1726 +++++++++++ .../us_runtime/reported_coverage_source.py | 448 +++ .../build/us_runtime/source_csv_builtin.py | 31 + .../build/us_runtime/survey_age_activation.py | 217 ++ .../us_runtime/survey_age_calibration.py | 773 +++++ .../build/us_runtime/survey_age_sources.py | 181 ++ .../survey_calibration_diagnostics.py | 196 ++ .../us_runtime/survey_catalogue_selection.py | 259 ++ .../build/us_runtime/survey_observed_age.py | 96 + .../build/us_runtime/survey_origin_budget.py | 842 ++++++ .../us_runtime/survey_population_domains.py | 577 ++++ .../survey_population_preparation.py | 1338 +++++++++ .../us_runtime/survey_population_replay.py | 208 ++ .../tests/test_us_acs_housing_source.py | 2634 +++++++++++++++++ ...t_us_acs_person_coverage_authentication.py | 588 ++++ .../test_us_acs_person_coverage_columns.py | 496 ++++ .../tests/test_us_acs_population_catalogue.py | 966 ++++++ .../test_us_asec_2024_native_population.py | 664 +++++ .../test_us_asec_coverage_authentication.py | 944 ++++++ .../test_us_asec_current_money_source.py | 394 +++ .../test_us_asec_person_income_source.py | 723 +++++ .../tests/test_us_current_survey_puf_host.py | 443 +++ .../test_us_current_survey_puf_transfer.py | 397 +++ .../tests/test_us_full_puf_enrichment.py | 477 +++ .../tests/test_us_graph_survey_population.py | 753 +++++ .../tests/test_us_national_age_counts.py | 953 ++++++ .../tests/test_us_puf_detail_transfer.py | 1026 +++++++ .../tests/test_us_puf_price_baseline.py | 1843 ++++++++++++ .../tests/test_us_survey_age_activation.py | 151 + .../tests/test_us_survey_age_artifact.py | 214 ++ .../test_us_survey_age_calibration_run.py | 455 +++ .../tests/test_us_survey_age_development.py | 258 ++ .../tests/test_us_survey_age_sources.py | 155 + .../tests/test_us_survey_calibration.py | 189 ++ .../tests/test_us_survey_observed_age.py | 120 + .../tests/test_us_survey_origin_budget.py | 562 ++++ .../test_us_survey_population_preparation.py | 636 ++++ .../tests/test_us_survey_population_replay.py | 454 +++ .../src/microcosm/calibrate/group_bounds.py | 185 ++ .../src/microcosm/fit/_graph_legacy_apply.py | 139 + .../src/microcosm/fit/_graph_legacy_qrf.py | 197 ++ .../src/microcosm/fit/graph_legacy_apply.py | 40 + .../fit/graph_legacy_apply_matrix.py | 107 + .../src/microcosm/fit/graph_legacy_qrf.py | 186 ++ .../src/microcosm/fit/graph_legacy_train.py | 147 + .../src/microcosm/fit/model_input.py | 140 + .../src/microcosm/fit/qrf_target.py | 440 +++ .../adapters/_policyengine_us_source_index.py | 66 +- .../frame/adapters/policyengine_us.py | 71 +- .../test_policyengine_us_ownership_index.py | 100 + .../src/microcosm/graph/artifact_edges.py | 164 + .../src/microcosm/graph/attachments.py | 239 ++ .../src/microcosm/graph/availability.py | 92 + .../src/microcosm/graph/randomness.py | 69 + .../src/microcosm/graph/schema.py | 316 ++ 132 files changed, 68206 insertions(+), 47 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py create mode 100644 packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py create mode 100644 packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py create mode 100644 packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py create mode 100644 packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py create mode 100644 packages/microcosm-build/src/microcosm/build/survey_allocation.py create mode 100644 packages/microcosm-build/src/microcosm/build/survey_domain_sample.py create mode 100644 packages/microcosm-build/src/microcosm/build/table_identity.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/_person_signal_summary.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_coverage_fields.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_original_household_weights.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_budget.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py create mode 100644 packages/microcosm-build/tests/test_us_acs_housing_source.py create mode 100644 packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py create mode 100644 packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py create mode 100644 packages/microcosm-build/tests/test_us_acs_population_catalogue.py create mode 100644 packages/microcosm-build/tests/test_us_asec_2024_native_population.py create mode 100644 packages/microcosm-build/tests/test_us_asec_coverage_authentication.py create mode 100644 packages/microcosm-build/tests/test_us_asec_current_money_source.py create mode 100644 packages/microcosm-build/tests/test_us_asec_person_income_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_puf_host.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_puf_transfer.py create mode 100644 packages/microcosm-build/tests/test_us_full_puf_enrichment.py create mode 100644 packages/microcosm-build/tests/test_us_graph_survey_population.py create mode 100644 packages/microcosm-build/tests/test_us_national_age_counts.py create mode 100644 packages/microcosm-build/tests/test_us_puf_detail_transfer.py create mode 100644 packages/microcosm-build/tests/test_us_puf_price_baseline.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_activation.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_artifact.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_calibration_run.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_development.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_sources.py create mode 100644 packages/microcosm-build/tests/test_us_survey_calibration.py create mode 100644 packages/microcosm-build/tests/test_us_survey_observed_age.py create mode 100644 packages/microcosm-build/tests/test_us_survey_origin_budget.py create mode 100644 packages/microcosm-build/tests/test_us_survey_population_preparation.py create mode 100644 packages/microcosm-build/tests/test_us_survey_population_replay.py create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/graph_legacy_apply_matrix.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/model_input.py create mode 100644 packages/microcosm-fit/src/microcosm/fit/qrf_target.py create mode 100644 packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/artifact_edges.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/attachments.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/availability.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/randomness.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/schema.py diff --git a/PROGRESS.md b/PROGRESS.md index e85cce8eb..aeb510209 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -737,3 +737,5 @@ Verified requested clean branch, base HEAD, main ancestry and preservation pins. ### Next Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. + +Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py new file mode 100644 index 000000000..2812fd3a5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py @@ -0,0 +1,74 @@ +"""The US congressional-district published-benchmark contract, as executable rules. + +This package implements protocol ``us-cd-published-benchmark-v4`` (SHA-256 +``f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea``) under an +approval whose scope is **invented-fixture implementation only**. Candidate +scoring and production ledger registration are explicitly not approved, and +nothing here performs either: no candidate, population, ACS or Census artifact +is read, no ledger file is created or appended, and no release, promotion or +publication decision is produced. + +The approved document itself ships as a packaged resource beside +:mod:`~microcosm.build.cd_benchmark.protocol` and is hash-verified on load. Every +threshold, vocabulary and payload roster is compiled out of those bytes rather +than retyped, so the code cannot drift from the contract without a test failing. +Paths named inside the document are locators recording what was inspected; this +package never opens them. + +Import cost: the modules below are pure and depend only on the standard library +and this package. The two source adapters the contract pins are reached through +:mod:`~microcosm.build.cd_benchmark.universe`, which defers those imports to call +time, so importing this package does not load ``microcosm.build.us_runtime`` and +does not load PolicyEngine. + +Layout: + +* :mod:`~microcosm.build.cd_benchmark.canonical` — canonical JSON and digests. +* :mod:`~microcosm.build.cd_benchmark.reasons` — the closed vocabularies. +* :mod:`~microcosm.build.cd_benchmark.protocol` — compile the approved document. +* :mod:`~microcosm.build.cd_benchmark.metrics` — point and categorical checks. +* :mod:`~microcosm.build.cd_benchmark.reference` — reference validity and precision. +* :mod:`~microcosm.build.cd_benchmark.support` — origin support and concentration. +* :mod:`~microcosm.build.cd_benchmark.origin` — typed original-source origin keys. +* :mod:`~microcosm.build.cd_benchmark.approval` — injected approval and registration. +* :mod:`~microcosm.build.cd_benchmark.ledger` — in-memory ledger records and states. +* :mod:`~microcosm.build.cd_benchmark.binding` — candidate binding slots. +* :mod:`~microcosm.build.cd_benchmark.universe` — the pinned source adapters. +* :mod:`~microcosm.build.cd_benchmark.evaluation` — the six axes and one reduction. +""" + +from __future__ import annotations + +from microcosm.build.cd_benchmark.protocol import ( + APPROVED_PROTOCOL_ID, + APPROVED_PROTOCOL_SHA256, + CompiledProtocol, + ProtocolError, + canonical_protocol, + compile_protocol, +) +from microcosm.build.cd_benchmark.reasons import ( + AXES, + BenchmarkVerdict, + ExposureStatus, + MappingStatus, + PrecisionStatus, + Reason, + SupportVerdict, +) + +__all__ = [ + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "AXES", + "BenchmarkVerdict", + "CompiledProtocol", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "ProtocolError", + "Reason", + "SupportVerdict", + "canonical_protocol", + "compile_protocol", +] diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py new file mode 100644 index 000000000..15108b494 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py @@ -0,0 +1,115 @@ +"""Canonical JSON bytes and digests, exactly as the protocol defines them. + +``/identity/encoding`` and ``/exposure_ledger/format`` require UTF-8 canonical +JSON with sorted keys, compact separators, no BOM, no nonfinite values and no +trailing LF inside the hashed bytes, and they additionally reject duplicate +object keys. The shared aggregate-JSON reader in +:mod:`microcosm.build.candidate_quality` enforces the first set but accepts +duplicate keys (``json.loads`` keeps the last one), so this module supplies the +stricter loader the contract names rather than relaxing the contract to match an +existing reader. The protocol also fixes ``ensure_ascii=false``, which that +reader leaves at the escaping default; for ASCII payloads the two encoders agree +byte for byte apart from its trailing newline, and a contract test pins that. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from typing import Any + +__all__ = [ + "CanonicalJsonError", + "SHA256_HEX", + "canonical_bytes", + "digest", + "is_sha256_hex", + "strict_load", +] + +SHA256_HEX = re.compile(r"[0-9a-f]{64}\Z") + + +class CanonicalJsonError(ValueError): + """Bytes are not the canonical JSON the protocol requires.""" + + +def is_sha256_hex(value: object) -> bool: + """Return whether ``value`` is a lowercase 64-character hex digest.""" + return isinstance(value, str) and SHA256_HEX.fullmatch(value) is not None + + +def _pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise CanonicalJsonError(f"Duplicate JSON key {key!r}.") + result[key] = value + return result + + +def _nonfinite(token: str) -> None: + raise CanonicalJsonError(f"Nonfinite JSON value {token}.") + + +def strict_load(data: bytes) -> Any: + """Parse canonical JSON bytes, refusing BOM, duplicate keys and nonfinite. + + Raises: + CanonicalJsonError: On any of those, or on invalid UTF-8/JSON. + """ + if not isinstance(data, bytes): + raise CanonicalJsonError("Canonical JSON input must be bytes.") + if data.startswith(b"\xef\xbb\xbf"): + raise CanonicalJsonError("Canonical JSON must not begin with a BOM.") + try: + text = data.decode("utf-8") + value = json.loads(text, object_pairs_hook=_pairs, parse_constant=_nonfinite) + except CanonicalJsonError: + raise + except (UnicodeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Invalid canonical JSON: {error}") from None + _assert_finite(value) + return value + + +def _assert_finite(value: object) -> None: + if isinstance(value, bool): + return + if isinstance(value, float | int): + try: + finite = math.isfinite(value) + except OverflowError: + finite = False + if not finite: + raise CanonicalJsonError("JSON numbers must be finite.") + elif isinstance(value, dict): + for child in value.values(): + _assert_finite(child) + elif isinstance(value, list): + for child in value: + _assert_finite(child) + + +def canonical_bytes(value: object) -> bytes: + """Serialize to the protocol's canonical UTF-8 JSON, with no trailing LF.""" + try: + text = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + except (TypeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Value is not canonical JSON: {error}") from None + return text.encode("utf-8") + + +def digest(data: bytes) -> str: + """Return the lowercase hex SHA-256 of exact bytes.""" + if not isinstance(data, bytes): + raise CanonicalJsonError("Digest input must be bytes.") + return hashlib.sha256(data).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py new file mode 100644 index 000000000..2912a5e32 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py @@ -0,0 +1,186 @@ +"""Typed original-source household origin keys, exactly as ``v4`` encodes them. + +``/source_household_origin/key_encoding`` fixes the encoding: +``SHA256(UTF8(domain) + one NUL byte + canonical payload bytes)``, lowercase +hex, where the payload is a canonical UTF-8 JSON array of four ordered typed +pairs. Each pair is a two-element array whose first element is the literal type +name. Integers are JSON integers, never booleans, and raw strings receive no +normalization, trimming or coercion. + +The two arms differ in their third and fourth components, and this module names +them per arm rather than generically, so a survey vintage cannot be filed as an +income cohort year: + +* ACS — ``arm=ACS_PUMS``, authenticated original household *archive* member + identity, **survey vintage**, and the **raw ``SERIALNO`` string** with its + leading and lexical characters preserved. +* ASEC — ``arm=ASEC``, authenticated original household member identity, + **income cohort year**, and the **native integer ``H_SEQ``**. + +Two limits carried from the document. A generated ``source_household_id`` or a +normalized ``household_id`` is positional and cannot replace this identity +(``/source_household_origin/ACS/not_equivalent``). Cohort or vintage +distinguishes statistical source records; it does not prove different real +households across years, and an origin key does not establish independent +households across surveys or independent imputation donors. + +Physical staging paths, recompressed archive hashes and clone or cache +filenames are not identities: the authenticated source manifest supplies +``canonical_member_id`` and the original member bytes hash, and aliases must +resolve to that same member before hashing. Typed components stay private in +authenticated lineage; public diagnostics use counts and digests only. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from enum import StrEnum + +from microcosm.build.cd_benchmark.canonical import canonical_bytes, is_sha256_hex + +__all__ = [ + "AcsHouseholdOrigin", + "AsecHouseholdOrigin", + "HouseholdOrigin", + "ORIGIN_DOMAIN", + "OriginKeyError", + "SourceArm", + "SourceMember", + "origin_key", + "origin_payload", + "origin_payload_bytes", +] + +#: ``/source_household_origin/key_encoding/domain``. +ORIGIN_DOMAIN = "microcosm.statistical_source_household.v1" + + +class OriginKeyError(ValueError): + """The typed origin components are not what the encoding accepts.""" + + +class SourceArm(StrEnum): + """The two arms with a registered origin definition. + + ``/source_household_origin/other_arms``: any other arm needs its own + registered, authenticated source-specific definition; there is deliberately + no default to a synthetic row id here. + """ + + ACS_PUMS = "ACS_PUMS" + ASEC = "ASEC" + + +def _identifier(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OriginKeyError(f"{label} must be a nonempty string.") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise OriginKeyError(f"{label} must be a JSON integer, not a boolean.") + return value + + +@dataclass(frozen=True) +class SourceMember: + """Canonical original member identity from the authenticated manifest.""" + + canonical_member_id: str + member_sha256: str + + def __post_init__(self) -> None: + _identifier(self.canonical_member_id, "canonical_member_id") + if not is_sha256_hex(self.member_sha256): + raise OriginKeyError( + "member_sha256 must be a lowercase 64-character hex digest." + ) + + def as_payload(self) -> dict[str, str]: + """The member pair's object value, with lexicographically sorted keys.""" + return { + "canonical_member_id": self.canonical_member_id, + "member_sha256": self.member_sha256, + } + + +@dataclass(frozen=True) +class AcsHouseholdOrigin: + """ACS PUMS origin: archive member identity, survey vintage, raw SERIALNO.""" + + member: SourceMember + survey_vintage: int + raw_serialno: str + + arm = SourceArm.ACS_PUMS + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.survey_vintage, "survey_vintage") + # No normalization, trimming or coercion: an integer SERIALNO would be a + # coercion of the published string, and leading characters are meaning. + if not isinstance(self.raw_serialno, str) or not self.raw_serialno: + raise OriginKeyError("raw_serialno must be the nonempty published string.") + + +@dataclass(frozen=True) +class AsecHouseholdOrigin: + """ASEC origin: member identity, income cohort year, native integer H_SEQ.""" + + member: SourceMember + income_cohort_year: int + native_h_seq: int + + arm = SourceArm.ASEC + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.income_cohort_year, "income_cohort_year") + # The shipped ASEC adapter's NATIVE_KEY invariant is H_SEQ > 0; a zero or + # negative sequence is not a native household key. + if _integer(self.native_h_seq, "native_h_seq") <= 0: + raise OriginKeyError("native_h_seq must be a positive native key.") + + +HouseholdOrigin = AcsHouseholdOrigin | AsecHouseholdOrigin + + +def origin_payload(origin: HouseholdOrigin) -> list[list[object]]: + """Return the four ordered typed pairs for ``origin``. + + Raises: + OriginKeyError: If ``origin`` is not a registered arm's origin. + """ + if isinstance(origin, AcsHouseholdOrigin): + return [ + ["string", str(SourceArm.ACS_PUMS)], + ["member", origin.member.as_payload()], + ["integer", origin.survey_vintage], + ["string", origin.raw_serialno], + ] + if isinstance(origin, AsecHouseholdOrigin): + return [ + ["string", str(SourceArm.ASEC)], + ["member", origin.member.as_payload()], + ["integer", origin.income_cohort_year], + ["integer", origin.native_h_seq], + ] + raise OriginKeyError( + "Only registered ACS and ASEC origins have a defined encoding; another " + "arm requires its own registered source-specific origin definition." + ) + + +def origin_payload_bytes(origin: HouseholdOrigin) -> bytes: + """Canonical payload bytes: sorted object keys, compact, no BOM or LF.""" + return canonical_bytes(origin_payload(origin)) + + +def origin_key(origin: HouseholdOrigin) -> str: + """Return the lowercase hex origin key for one original source household.""" + payload = origin_payload_bytes(origin) + return hashlib.sha256(ORIGIN_DOMAIN.encode("utf-8") + b"\x00" + payload).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py new file mode 100644 index 000000000..18c24024c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py @@ -0,0 +1,871 @@ +"""Compile the approved canonical CD published-benchmark protocol document. + +The exact approved bytes ship as a packaged resource beside this module. Nothing +here reads a path named inside the document: ``/identity/legacy_design_parents``, +``/reference_authority`` and the code locators are *locators* recording what was +inspected when the contract was written, never inputs this module opens. + +Compilation is total and closed. Every threshold is carried as an exact +:class:`~decimal.Decimal` parsed from the document's decimal string, never as a +float; every vocabulary is compared against the document rather than assumed; +unknown block shapes refuse compilation. :func:`compile_protocol` will compile +any candidate document so that a successor can be inspected, but only bytes +whose SHA-256 equals :data:`APPROVED_PROTOCOL_SHA256` compile to a protocol with +``canonical`` set, and approval is resolved from that digest through an injected +authority (:mod:`microcosm.build.cd_benchmark.approval`) — never from a field +inside the document and never from a caller's label. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from enum import StrEnum +from functools import lru_cache +from importlib import resources +from types import MappingProxyType +from typing import Any + +from microcosm.build.cd_benchmark.canonical import ( + CanonicalJsonError, + canonical_bytes, + digest, + is_sha256_hex, + strict_load, +) +from microcosm.build.cd_benchmark.reasons import ExposureStatus, Reason + +__all__ = [ + "APPROVED_PROTOCOL_BYTES", + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "CompiledProtocol", + "DistributionBlock", + "PartitionRules", + "PointBlock", + "PrecisionKind", + "ProtocolError", + "ReferenceBin", + "RESERVED_FAMILIES", + "ScopeRules", + "SupportBounds", + "assert_vocabulary_closed", + "assign_bin", + "canonical_protocol", + "compile_protocol", + "load_canonical_bytes", +] + +#: Digest and byte count of the approved canonical document, verified against +#: the review directory's plan, its independent response and the peer review. +APPROVED_PROTOCOL_SHA256 = ( + "f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea" +) +APPROVED_PROTOCOL_BYTES = 86626 +APPROVED_PROTOCOL_ID = "us-cd-published-benchmark-v4" +_RESOURCE = "us-cd-published-benchmark-v4.json" + +#: ``/exposure_ledger/reservation``: these table families and every derivative +#: stay out of fitting, tuning and selection. +RESERVED_FAMILIES = frozenset({"B19001", "B25003"}) + +_FAMILY_ROOT = re.compile(r"([A-Z][0-9]{4,5})(?:_|\Z)") + + +class ProtocolError(ValueError): + """The document is not a compilable CD published-benchmark contract.""" + + +class PrecisionKind(StrEnum): + """Which ``/reference_status/share_precision`` rule a distribution uses.""" + + PUBLISHED_PERCENTAGE_MOE = "published_percentage_moe" + SUBSET_SHARE_APPROXIMATION = "subset_share_approximation" + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ProtocolError(f"{label} must be a JSON object.") + return value + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise ProtocolError(f"{label} must be a nonempty string.") + return value + + +def _flag(value: object, label: str) -> bool: + if not isinstance(value, bool): + raise ProtocolError(f"{label} must be a JSON boolean.") + return value + + +def _decimal(value: object, label: str) -> Decimal: + """Parse an exact decimal-string threshold; floats are refused.""" + if not isinstance(value, str): + raise ProtocolError(f"{label} must be an exact decimal string.") + try: + parsed = Decimal(value) + except InvalidOperation: + raise ProtocolError(f"{label} is not a decimal: {value!r}.") from None + if not parsed.is_finite(): + raise ProtocolError(f"{label} must be finite.") + return parsed + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{label} must be a JSON integer.") + return value + + +def _list(container: Mapping[str, Any], key: str, label: str) -> list[Any]: + """Read a required roster, refusing an absent key and a non-list value. + + Compilation is total: an absent section leaks no ``KeyError`` and a string + in a roster's place is never iterated per character into field names. + """ + if key not in container: + raise ProtocolError(f"{label} is required and absent.") + value = container[key] + if not isinstance(value, list): + raise ProtocolError(f"{label} must be a JSON array.") + return value + + +def _bound(value: object, label: str) -> Decimal | None: + """A bin edge: an exact integer, or ``None`` for an open side.""" + if value is None: + return None + return Decimal(_integer(value, label)) + + +@dataclass(frozen=True) +class ReferenceBin: + """One published bin, or the protocol-constructed ``unclassified`` residual. + + ``lower_inclusive``/``upper_exclusive`` follow the document's boundary rule: + lower inclusive, upper exclusive, ``None`` meaning minus/plus infinity. The + residual bin carries no publisher variable and a reference share of exactly + zero; it is a protocol construct, not a publisher cell. + """ + + id: str + publisher_variable: str | None + publisher_label: str | None + lower_inclusive: Decimal | None + upper_exclusive: Decimal | None + reference_share: Decimal | None + unclassified: bool + authority: str | None + + +@dataclass(frozen=True) +class PointBlock: + """A mandatory scalar block compared by relative discrepancy.""" + + id: str + family: str + dataset: str + unit: str + universe: str + mandatory: bool + rule: str + variables: tuple[str, ...] + point_relative_bound: Decimal + reference_relative_moe_ceiling: Decimal + controlled_reference: str | None + exposure_families: frozenset[str] + count_once: bool + duplicate_reference_aliases: tuple[str, ...] + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + +@dataclass(frozen=True) +class DistributionBlock: + """A mandatory categorical block compared by per-bin share and total variation. + + ``independent_votes`` is ``False`` where the document says so (tenure's owner + and renter bins) and ``None`` where it says nothing. Undeclared is not a + licence to treat bins as independent votes: ``/claim`` withholds that + licence for overlapping cells, and ``/status_contract/no_rescue`` forbids + rescuing a block with them. + """ + + id: str + family: str + dataset: str + unit: str + mandatory: bool + denominator_variable: str + bins: tuple[ReferenceBin, ...] + max_each_bin_discrepancy_pp: Decimal + max_total_variation: Decimal + reference_moe_ceiling_pp: Decimal + precision_kind: PrecisionKind + precision_variables: tuple[str, ...] + exposure_families: frozenset[str] + independent_votes: bool | None + open_upper: bool + boundary_rule: str | None + additional_bounds: MappingProxyType + complement_only_when: str | None + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + @property + def unclassified_bin(self) -> ReferenceBin: + """The protocol-constructed residual bin.""" + return next(item for item in self.bins if item.unclassified) + + @property + def known_bins(self) -> tuple[ReferenceBin, ...]: + """Published bins, excluding the residual.""" + return tuple(item for item in self.bins if not item.unclassified) + + +@dataclass(frozen=True) +class SupportBounds: + """``/support`` floors and ceilings, as exact decimals and integers.""" + + minimum_positive_source_household_origins: int + minimum_positive_origins_per_positive_reference_bin: int + minimum_origin_cluster_concentration_ess: Decimal + maximum_origin_denominator_share: Decimal + maximum_origin_positive_bin_share: Decimal + nonnegative_finite_weights_required: bool + required_for_every_district_and_relevant_denominator: bool + + +@dataclass(frozen=True) +class PartitionRules: + """``/categorical_partition`` closure and residual-mass rules.""" + + maximum_unclassified_fraction: Decimal + comparison: str + renormalize_known_bins: bool + unknown_bin_id: str + applies_to: frozenset[str] + + +@dataclass(frozen=True) +class ScopeRules: + """``/scope`` — the predeclared geographic and product scope.""" + + country: str + area: str + congress: int + district_count: int + acs_year: int + acs_product: str + puerto_rico: str + allowed_missing_mandatory_block_fraction: Decimal + allowed_missing_mandatory_district_fraction: Decimal + dc_canonical_id: str + dc_published_id: str + published_and_canonical_geography_ids_retained: bool + + +@dataclass(frozen=True) +class CompiledProtocol: + """A compiled contract document plus the digest of the bytes it came from.""" + + sha256: str + size_bytes: int + protocol_id: str + schema: str + status: str + parent_sha256: str + hash_scope: str + point_blocks: tuple[PointBlock, ...] + distribution_blocks: tuple[DistributionBlock, ...] + partition: PartitionRules + support: SupportBounds + scope: ScopeRules + reason_codes: frozenset[str] + exposure_status_values: frozenset[str] + independent_axes: tuple[str, ...] + scope_reduction_order: tuple[str, ...] + scoring_permitted_in_document: bool + approval_installed_in_document: bool + reference_inventory_sha256: str + block_definition_hashes: Mapping[str, str] + pinned_code_sha256: Mapping[str, str] + ledger_event_types: frozenset[str] + ledger_record_fields: frozenset[str] + ledger_snapshot_fields: tuple[str, ...] + attempt_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + completion_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + registration_payload_fields: tuple[str, ...] + quality_required_fields: tuple[str, ...] + amendment_change_fields: tuple[str, ...] + required_binding_slots: tuple[str, ...] + + @property + def canonical(self) -> bool: + """Whether these bytes are the exact approved canonical document.""" + return ( + self.sha256 == APPROVED_PROTOCOL_SHA256 + and self.size_bytes == APPROVED_PROTOCOL_BYTES + ) + + @property + def blocks(self) -> tuple[PointBlock | DistributionBlock, ...]: + """Every mandatory block, point blocks first, in document order.""" + return (*self.point_blocks, *self.distribution_blocks) + + def block(self, block_id: str) -> PointBlock | DistributionBlock: + """Return the block with ``block_id``. + + Raises: + ProtocolError: If no block carries that id. + """ + for item in self.blocks: + if item.id == block_id: + return item + raise ProtocolError(f"Unknown block id {block_id!r}.") + + +def _families(block: dict[str, Any]) -> frozenset[str]: + """Reference table-family roots whose exposure this block consumes.""" + declared = block.get("exposure_families") + if declared is not None: + if not isinstance(declared, list) or not declared: + raise ProtocolError("exposure_families must be a nonempty list.") + return frozenset(_text(item, "exposure family") for item in declared) + match = _FAMILY_ROOT.match(_text(block.get("family"), "block family")) + if match is None: + raise ProtocolError( + f"Block family {block.get('family')!r} names no published table root " + "and declares no exposure_families." + ) + return frozenset({match.group(1)}) + + +def _bin(item: object) -> ReferenceBin: + data = _object(item, "distribution bin") + identifier = _text(data.get("id"), "bin id") + unclassified = "authority" in data + if unclassified: + if data.get("publisher_variable") is not None: + raise ProtocolError("The residual bin cannot name a publisher variable.") + share = _decimal(data.get("reference_share"), "residual reference_share") + if share != 0: + raise ProtocolError("The residual bin's reference share must be zero.") + return ReferenceBin( + id=identifier, + publisher_variable=None, + publisher_label=None, + lower_inclusive=None, + upper_exclusive=None, + reference_share=share, + unclassified=True, + authority=_text(data.get("authority"), "residual bin authority"), + ) + variable = ( + data["publisher_variable"] if "publisher_variable" in data else identifier + ) + return ReferenceBin( + id=identifier, + publisher_variable=_text(variable, "bin publisher variable"), + publisher_label=data.get("publisher_label"), + lower_inclusive=_bound(data.get("lower_inclusive"), "bin lower_inclusive"), + upper_exclusive=_bound(data.get("upper_exclusive"), "bin upper_exclusive"), + reference_share=None, + unclassified=False, + authority=None, + ) + + +def _point_block(block: dict[str, Any]) -> PointBlock: + ceilings = [ + key + for key in ( + "ordinary_reference_relative_moe_ceiling", + "reference_relative_moe_ceiling", + ) + if key in block + ] + if len(ceilings) != 1: + raise ProtocolError( + "A point block declares exactly one relative reference MOE ceiling." + ) + variables = block.get("variables") + if not isinstance(variables, list) or not variables: + raise ProtocolError("A point block declares its reference variables.") + aliases = block.get("duplicate_reference_aliases", []) + if not isinstance(aliases, list): + raise ProtocolError("duplicate_reference_aliases must be a list.") + return PointBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + universe=_text(block.get("universe"), "block universe"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + rule=_text(block.get("rule"), "block rule"), + variables=tuple(_text(item, "block variable") for item in variables), + point_relative_bound=_decimal( + block.get("point_relative_bound"), "point_relative_bound" + ), + reference_relative_moe_ceiling=_decimal(block[ceilings[0]], ceilings[0]), + controlled_reference=block.get("controlled_reference"), + exposure_families=_families(block), + count_once=bool(block.get("count_once", False)), + duplicate_reference_aliases=tuple( + _text(item, "duplicate reference alias") for item in aliases + ), + ) + + +_PRECISION_CEILINGS = { + "reference_published_moe_ceiling_pp": PrecisionKind.PUBLISHED_PERCENTAGE_MOE, + "reference_approximate_moe_ceiling_pp": PrecisionKind.SUBSET_SHARE_APPROXIMATION, +} + + +def _distribution_block(block: dict[str, Any]) -> DistributionBlock: + present = [key for key in _PRECISION_CEILINGS if key in block] + if len(present) != 1: + raise ProtocolError( + "A distribution block declares exactly one reference precision ceiling." + ) + bins = tuple(_bin(item) for item in block.get("bins", [])) + if len(bins) < 2 or sum(item.unclassified for item in bins) != 1: + raise ProtocolError("A distribution declares its bins plus one residual bin.") + if not bins[-1].unclassified: + raise ProtocolError("The residual bin is declared last.") + identifiers = [item.id for item in bins] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate bin ids.") + extra = { + key: _decimal(value, key) + for key, value in block.items() + if key.startswith("max_") + and key.endswith("_pp") + and key not in {"max_each_bin_discrepancy_pp"} + } + if set(extra) - {"max_renter_discrepancy_pp"}: + raise ProtocolError("Unsupported additional distribution bound.") + if "max_renter_discrepancy_pp" in extra and "renter" not in identifiers: + raise ProtocolError("The renter bound requires a renter bin.") + precision_variables = block.get("precision_variables", []) + if not isinstance(precision_variables, list): + raise ProtocolError("precision_variables must be a JSON array.") + return DistributionBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + denominator_variable=_text(block.get("denominator"), "block denominator"), + bins=bins, + max_each_bin_discrepancy_pp=_decimal( + block.get("max_each_bin_discrepancy_pp"), "max_each_bin_discrepancy_pp" + ), + max_total_variation=_decimal( + block.get("max_total_variation"), "max_total_variation" + ), + reference_moe_ceiling_pp=_decimal(block[present[0]], present[0]), + precision_kind=_PRECISION_CEILINGS[present[0]], + precision_variables=tuple( + _text(item, "precision variable") for item in precision_variables + ), + exposure_families=_families(block), + independent_votes=block.get("owner_and_renter_independent_votes"), + open_upper=bool(block.get("open_upper_age", False)), + boundary_rule=block.get("boundary_rule"), + additional_bounds=MappingProxyType(extra), + complement_only_when=block.get("owner_is_renter_complement_only_when"), + ) + + +def compile_protocol(raw: bytes) -> CompiledProtocol: + """Compile canonical protocol bytes into typed, exact rules. + + Args: + raw: The exact document bytes. Their SHA-256 is the protocol identity; + the document carries no self-hash member (``/identity/hash_scope``). + + Raises: + ProtocolError: On any missing, mistyped or unknown-shaped section. + """ + try: + document = _object(strict_load(raw), "protocol document") + except CanonicalJsonError as error: + raise ProtocolError(str(error)) from None + identity = _object(document.get("identity"), "/identity") + if identity.get("algorithm") != "sha256": + raise ProtocolError("Only sha256 protocol identity is supported.") + parent = _text(identity.get("protocol_parent_sha256"), "protocol_parent_sha256") + if not is_sha256_hex(parent): + raise ProtocolError("protocol_parent_sha256 must be a lowercase digest.") + blocks = document.get("blocks") + if not isinstance(blocks, list) or not blocks: + raise ProtocolError("/blocks must be a nonempty list.") + point, distribution = [], [] + for item in blocks: + block = _object(item, "block") + if "bins" in block: + distribution.append(_distribution_block(block)) + else: + point.append(_point_block(block)) + identifiers = [item.id for item in (*point, *distribution)] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate block ids.") + + partition_doc = _object( + document.get("categorical_partition"), "/categorical_partition" + ) + unknown_bin = _object(partition_doc.get("unknown_bin"), "unknown_bin") + partition = PartitionRules( + maximum_unclassified_fraction=_decimal( + partition_doc.get("maximum_unclassified_fraction_of_known_denominator"), + "maximum_unclassified_fraction_of_known_denominator", + ), + comparison=_text( + partition_doc.get("maximum_unclassified_fraction_comparison"), + "maximum_unclassified_fraction_comparison", + ), + renormalize_known_bins=_flag( + partition_doc.get("renormalize_known_bins"), "renormalize_known_bins" + ), + unknown_bin_id=_text(unknown_bin.get("id"), "unknown bin id"), + applies_to=frozenset( + _text(item, "partition applies_to") + for item in _list( + partition_doc, "applies_to", "/categorical_partition/applies_to" + ) + ), + ) + support_doc = _object(document.get("support"), "/support") + support = SupportBounds( + minimum_positive_source_household_origins=_integer( + support_doc.get("minimum_positive_source_household_origins"), + "minimum_positive_source_household_origins", + ), + minimum_positive_origins_per_positive_reference_bin=_integer( + support_doc.get("minimum_positive_origins_per_positive_reference_bin"), + "minimum_positive_origins_per_positive_reference_bin", + ), + minimum_origin_cluster_concentration_ess=_decimal( + support_doc.get("minimum_origin_cluster_concentration_ESS"), + "minimum_origin_cluster_concentration_ESS", + ), + maximum_origin_denominator_share=_decimal( + support_doc.get("maximum_origin_denominator_share"), + "maximum_origin_denominator_share", + ), + maximum_origin_positive_bin_share=_decimal( + support_doc.get("maximum_origin_positive_bin_share"), + "maximum_origin_positive_bin_share", + ), + nonnegative_finite_weights_required=_flag( + support_doc.get("nonnegative_finite_weights_required"), + "nonnegative_finite_weights_required", + ), + required_for_every_district_and_relevant_denominator=_flag( + support_doc.get("required_for_every_district_and_relevant_denominator"), + "required_for_every_district_and_relevant_denominator", + ), + ) + scope_doc = _object(document.get("scope"), "/scope") + alias = _object(scope_doc.get("DC_identity_alias"), "DC_identity_alias") + scope = ScopeRules( + country=_text(scope_doc.get("country"), "scope country"), + area=_text(scope_doc.get("area"), "scope area"), + congress=_integer(scope_doc.get("congress"), "scope congress"), + district_count=_integer(scope_doc.get("district_count"), "district_count"), + acs_year=_integer(scope_doc.get("acs_year"), "acs_year"), + acs_product=_text(scope_doc.get("acs_product"), "acs_product"), + puerto_rico=_text(scope_doc.get("puerto_rico"), "puerto_rico"), + allowed_missing_mandatory_block_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_block_fraction"), + "allowed_missing_mandatory_block_fraction", + ), + allowed_missing_mandatory_district_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_district_fraction"), + "allowed_missing_mandatory_district_fraction", + ), + dc_canonical_id=_text(alias.get("canonical"), "DC canonical id"), + dc_published_id=_text(alias.get("published"), "DC published id"), + published_and_canonical_geography_ids_retained=_flag( + scope_doc.get("published_and_canonical_geography_ids_retained"), + "published_and_canonical_geography_ids_retained", + ), + ) + status_doc = _object(document.get("status_contract"), "/status_contract") + ledger_doc = _object(document.get("exposure_ledger"), "/exposure_ledger") + approval_doc = _object(document.get("approval"), "/approval") + return CompiledProtocol( + sha256=digest(raw), + size_bytes=len(raw), + protocol_id=_text(document.get("protocol_id"), "/protocol_id"), + schema=_text(document.get("schema"), "/schema"), + status=_text(document.get("status"), "/status"), + parent_sha256=parent, + hash_scope=_text(identity.get("hash_scope"), "/identity/hash_scope"), + point_blocks=tuple(point), + distribution_blocks=tuple(distribution), + partition=partition, + support=support, + scope=scope, + reason_codes=frozenset(_object(status_doc.get("reason_codes"), "reason_codes")), + exposure_status_values=frozenset( + _object(ledger_doc.get("exposure_status_values"), "exposure_status_values") + ), + independent_axes=tuple( + _text(item, "independent axis") + for item in _list( + status_doc, + "independent_axes_required", + "/status_contract/independent_axes_required", + ) + ), + scope_reduction_order=tuple( + _text(item, "scope reduction step") + for item in _list( + status_doc, + "scope_reduction_order", + "/status_contract/scope_reduction_order", + ) + ), + scoring_permitted_in_document=_flag( + approval_doc.get("scoring_permitted"), "/approval/scoring_permitted" + ), + approval_installed_in_document=_flag( + approval_doc.get("installed"), "/approval/installed" + ), + reference_inventory_sha256=_inventory_digest(document, scope_doc), + block_definition_hashes=MappingProxyType( + { + _text(_object(item, "block").get("id"), "block id"): digest( + canonical_bytes(item) + ) + for item in blocks + } + ), + pinned_code_sha256=MappingProxyType(_pinned_code(document)), + ledger_event_types=frozenset( + _object(ledger_doc.get("other_event_types"), "other_event_types") + ), + ledger_record_fields=frozenset( + _object(ledger_doc.get("record_fields"), "record_fields") + ), + ledger_snapshot_fields=tuple( + _text(item, "snapshot field") + for item in _list( + ledger_doc, "snapshot_fields", "/exposure_ledger/snapshot_fields" + ) + ), + attempt_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_attempt_payload", + "/exposure_ledger/evaluation_attempt_payload", + ) + ), + completion_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_completion_payload", + "/exposure_ledger/evaluation_completion_payload", + ) + ), + registration_payload_fields=tuple( + _text(item, "registration payload field") + for item in _list( + ledger_doc, + "protocol_registration_payload", + "/exposure_ledger/protocol_registration_payload", + ) + ), + quality_required_fields=tuple( + _text(item, "quality required field") + for item in _list( + ledger_doc, + "quality_required_fields", + "/exposure_ledger/quality_required_fields", + ) + ), + amendment_change_fields=tuple( + _text(item, "amendment change field") + for item in _list( + _object(document.get("protocol_amendments"), "/protocol_amendments"), + "required_change_fields", + "/protocol_amendments/required_change_fields", + ) + ), + required_binding_slots=tuple( + _text(item, "binding slot") + for item in _list( + _object( + document.get("candidate_binding_requirements"), + "/candidate_binding_requirements", + ), + "must_authenticate_before_scoring", + "/candidate_binding_requirements/must_authenticate_before_scoring", + ) + ), + ) + + +def _payload_fields( + declared: object, +) -> tuple[tuple[str, tuple[str, ...] | None], ...]: + """Split a declared payload roster into names and any pinned values. + + The document writes a constrained field as ``name=value`` and an + alternative as ``a_or_b``: ``attempt_state=started`` and + ``terminal_state=completed_or_failed``. Everything else is an unconstrained + required field name. + """ + if not isinstance(declared, list) or not declared: + raise ProtocolError("A ledger payload roster must be a nonempty list.") + fields: list[tuple[str, tuple[str, ...] | None]] = [] + for item in declared: + entry = _text(item, "ledger payload field") + name, separator, value = entry.partition("=") + if not separator: + fields.append((name, None)) + continue + fields.append((name, tuple(value.split("_or_")))) + names = [name for name, _ in fields] + if len(set(names)) != len(names): + raise ProtocolError("Duplicate ledger payload field names.") + return tuple(fields) + + +def _inventory_digest(document: dict[str, Any], scope_doc: dict[str, Any]) -> str: + """The reference inventory digest, required to agree in both places.""" + authority = _object(document.get("reference_authority"), "/reference_authority") + inventory = _object(authority.get("inventory"), "reference inventory") + required = _object(scope_doc.get("required_district_set"), "required_district_set") + primary = _text(inventory.get("sha256"), "reference inventory sha256") + secondary = _text(required.get("inventory_sha256"), "district inventory sha256") + if not is_sha256_hex(primary) or primary != secondary: + raise ProtocolError( + "The reference inventory digest must be a lowercase digest and must " + "agree between /reference_authority and /scope/required_district_set." + ) + return primary + + +def _pinned_code(document: dict[str, Any]) -> dict[str, str]: + """Source-file digests the contract inspected, keyed by file name. + + These are locators recording what was read when the contract was written. + Nothing in this package opens the paths beside them; the digests exist so a + binding can be checked against the adapter it was reviewed against, and a + changed adapter is a scope question rather than an automatic pin refresh. + """ + sections = ( + _object( + _object(document.get("reference_authority"), "/reference_authority").get( + "HU_adapter_clarification" + ), + "HU_adapter_clarification", + ), + _object( + _object(document.get("candidate_semantics"), "/candidate_semantics").get( + "definition_authority" + ), + "definition_authority", + ), + ) + pins: dict[str, str] = {} + for section in sections: + for item in section.get("code", []): + entry = _object(item, "pinned code entry") + name = _text(entry.get("path"), "pinned code path").rsplit("/", 1)[-1] + value = _text(entry.get("sha256"), "pinned code sha256") + if not is_sha256_hex(value): + raise ProtocolError(f"Pinned code digest for {name} is not a digest.") + if pins.setdefault(name, value) != value: + raise ProtocolError(f"Conflicting pinned code digests for {name}.") + if not pins: + raise ProtocolError("The document pins no inspected source code.") + return pins + + +def load_canonical_bytes() -> bytes: + """Read the packaged approved document and verify its digest and size. + + Raises: + ProtocolError: If the packaged bytes are not the approved document. + """ + raw = resources.files(__package__).joinpath(_RESOURCE).read_bytes() + if len(raw) != APPROVED_PROTOCOL_BYTES or digest(raw) != APPROVED_PROTOCOL_SHA256: + raise ProtocolError("Packaged protocol resource is not the approved document.") + return raw + + +@lru_cache(maxsize=1) +def canonical_protocol() -> CompiledProtocol: + """Return the compiled approved canonical protocol. + + Raises: + ProtocolError: If the packaged document is not the approved bytes, does + not identify itself as the approved protocol, or has drifted from + the closed reason/exposure vocabularies this package implements. + """ + compiled = compile_protocol(load_canonical_bytes()) + if not compiled.canonical or compiled.protocol_id != APPROVED_PROTOCOL_ID: + raise ProtocolError("Packaged protocol is not the approved canonical contract.") + assert_vocabulary_closed(compiled) + return compiled + + +def assert_vocabulary_closed(compiled: CompiledProtocol) -> None: + """Check this package's closed vocabularies equal the document's. + + Raises: + ProtocolError: On any code this package would emit that the document + does not define, or any document code this package cannot emit. + """ + for name, implemented, declared in ( + ("reason", {item.value for item in Reason}, set(compiled.reason_codes)), + ( + "exposure status", + {item.value for item in ExposureStatus}, + set(compiled.exposure_status_values), + ), + ): + if implemented != declared: + raise ProtocolError( + f"Closed {name} vocabulary differs from the document: " + f"only in code {sorted(implemented - declared)}, " + f"only in document {sorted(declared - implemented)}." + ) + + +def assign_bin(block: DistributionBlock, value: Decimal) -> str | None: + """Return the bin id for ``value`` under the declared boundary rule. + + Lower bounds are inclusive and upper bounds exclusive; a ``None`` lower or + upper bound is minus or plus infinity. No rounding is applied — a monetary + amount of ``9999.5`` stays below a ``10000`` cutpoint. Returns ``None`` when + no published bin covers the value, which the caller may treat as + unclassified mass only under known membership and an approved mapping. + """ + if not isinstance(value, Decimal) or not value.is_finite(): + raise ProtocolError("Bin assignment needs a finite Decimal value.") + for item in block.known_bins: + if item.lower_inclusive is None and item.upper_exclusive is None: + raise ProtocolError(f"Bin {item.id!r} declares no numeric boundary.") + if item.lower_inclusive is not None and value < item.lower_inclusive: + continue + if item.upper_exclusive is not None and value >= item.upper_exclusive: + continue + return item.id + return None diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py new file mode 100644 index 000000000..855d938b4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py @@ -0,0 +1,135 @@ +"""Closed reason, exposure and status vocabularies for the CD benchmark protocol. + +Every member here is checked against the canonical protocol document by +:func:`microcosm.build.cd_benchmark.protocol.assert_vocabulary_closed`, so the +code's vocabulary is verified equal to the approved document's rather than +merely resembling it. Per ``/status_contract/reason_encoding`` the primary +reason namespace is closed: unknown codes refuse validation, several +independently established reasons may coexist, and block/bin/metric identifiers +and raw source detail are separate bounded evidence, never a replacement code. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from enum import StrEnum + +__all__ = [ + "AXES", + "BenchmarkVerdict", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "Reason", + "SupportVerdict", + "dedupe", +] + + +class Reason(StrEnum): + """The 21 primary reason codes at ``/status_contract/reason_codes``.""" + + CANDIDATE_IDENTITY_UNAVAILABLE = "candidate_identity_unavailable" + DENOMINATOR_UNKNOWN = "denominator_unknown" + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + GQ_WEIGHT_BRIDGE_MISSING = "gq_weight_bridge_missing" + IDENTITY_MISMATCH = "identity_mismatch" + INVALID_REDUCER_INPUT = "invalid_reducer_input" + LEDGER_BUSY = "ledger_busy" + LEDGER_INTEGRITY_UNKNOWN = "ledger_integrity_unknown" + MAPPING_UNAPPROVED = "mapping_unapproved" + PROTOCOL_UNAPPROVED = "protocol_unapproved" + PUBLISHED_BENCHMARK_NONCONFORMITY = "published_benchmark_nonconformity" + REFERENCE_INVALID = "reference_invalid" + REFERENCE_PRECISION_INADEQUATE = "reference_precision_inadequate" + REQUIRED_CELL_OR_DISTRICT_MISSING = "required_cell_or_district_missing" + SOURCE_DOMAIN_REFUSED = "source_domain_refused" + SUPPORT_BOUND_EXCEEDED = "support_bound_exceeded" + UNCLASSIFIED_FRACTION_EXCEEDED = "unclassified_fraction_exceeded" + UNRESOLVED_SOURCE_LINEAGE = "unresolved_source_lineage" + VACANCY_BRIDGE_MISSING = "vacancy_bridge_missing" + ZERO_CANDIDATE_DENOMINATOR = "zero_candidate_denominator" + ZERO_REFERENCE_DENOMINATOR = "zero_reference_denominator" + + +class ExposureStatus(StrEnum): + """The four values at ``/exposure_ledger/exposure_status_values``.""" + + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + FRESH_SAME_SOURCE_HOLDOUT_ELIGIBLE = "fresh_same_source_holdout_eligible" + NOT_RESERVED_CONFORMITY = "not_reserved_conformity" + POST_EXPOSURE_CONFORMITY = "post_exposure_conformity" + + +class BenchmarkVerdict(StrEnum): + """Scope-level outcomes named by ``/status_contract``. + + There is deliberately no "release", "certified" or "passing" member: a + conforming verdict is operational published-benchmark conformity for the + stated scope and nothing else (``/claim``). + """ + + BENCHMARK_CONFORMING = "benchmark_conforming" + BENCHMARK_NONCONFORMING = "benchmark_nonconforming" + SCOPE_INCOMPLETE = "scope_incomplete" + + +class PrecisionStatus(StrEnum): + """The reference-precision axis, independent of the metric axis. + + ``CONTROLLED_UNKNOWN`` is the ``/reference_status/controlled_estimate`` + class: the point comparison stays valid, no variance is claimed, and no + adequacy is asserted. It is neither an adequacy verdict nor a failure. + """ + + ADEQUATE = "adequate" + INADEQUATE = "inadequate" + CONTROLLED_UNKNOWN = "controlled_unknown" + UNAVAILABLE = "unavailable" + + +class MappingStatus(StrEnum): + """Whether an authenticated producer bound the required mapping slots.""" + + APPROVED = "approved" + UNAPPROVED = "unapproved" + + +class SupportVerdict(StrEnum): + """Origin-support outcome, independent of the benchmark metric. + + ``NOT_APPLICABLE`` is the ``/support/reference_zero_unknown_bin_carrier_floor`` + case: the carrier floor does not apply to a bin whose published reference + share is zero, so mass and origin diagnostics are retained without a bound. + """ + + WITHIN_BOUNDS = "within_bounds" + BOUND_EXCEEDED = "bound_exceeded" + UNDETERMINED = "undetermined" + NOT_APPLICABLE = "not_applicable" + + +#: ``/status_contract/independent_axes_required`` — reported separately and +#: never collapsed into one scalar. +AXES = ( + "benchmark_metric_status", + "reference_precision_status", + "mapping_status", + "support_status", + "exposure_status", + "scope_completeness", +) + + +def dedupe(items: Iterable[Reason]) -> tuple[Reason, ...]: + """Deduplicate reasons, preserving first-seen order. + + Raises: + TypeError: If any item is outside the closed :class:`Reason` namespace. + """ + seen: dict[Reason, None] = {} + for item in items: + if not isinstance(item, Reason): + raise TypeError(f"Unknown primary reason code {item!r}.") + seen.setdefault(item) + return tuple(seen) diff --git a/packages/microcosm-build/src/microcosm/build/survey_allocation.py b/packages/microcosm-build/src/microcosm/build/survey_allocation.py new file mode 100644 index 000000000..417689e5a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_allocation.py @@ -0,0 +1,202 @@ +"""Domain allocation of supplied design weights, before support cloning. + +This numerical primitive performs no source classification, subsampling, frame +mutation or graph activation. Callers must supply an aligned household axis and +reviewed domain declarations. A row can be a physical housing unit or a cluster +of people; explicit unit counts keep its statistical grain out of its weight. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral +from types import MappingProxyType + +import numpy as np + +from microcosm.frame import WeightKind, Weights + + +class SurveyAllocationError(ValueError): + """A supplied allocation contract cannot represent its household axis.""" + + +def _label(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise SurveyAllocationError(f"{field} must be a nonempty string") + return value + + +@dataclass(frozen=True) +class SurveyDomain: + """One declared population/period and its prespecified source shares. + + Shares must be exact Fractions summing to one. Unit and basis labels are + declarations, not evidence of survey equivalence or source authentication. + """ + + key: str + statistical_unit: str + period: str + basis: str + source_shares: Mapping[str, Fraction] + + def __post_init__(self) -> None: + for field in ("key", "statistical_unit", "period", "basis"): + _label(getattr(self, field), field) + if not isinstance(self.source_shares, Mapping) or not self.source_shares: + raise SurveyAllocationError("source_shares must be a nonempty mapping") + shares = {} + for source, share in self.source_shares.items(): + _label(source, "source channel") + if not isinstance(share, Fraction) or not 0 <= share <= 1: + raise SurveyAllocationError( + "shares must be Fractions between zero and one" + ) + shares[source] = share + if sum(shares.values(), Fraction()) != 1: + raise SurveyAllocationError("domain shares must sum exactly to one") + object.__setattr__( + self, "source_shares", MappingProxyType(dict(sorted(shares.items()))) + ) + + +@dataclass(frozen=True) +class DomainSourceEstimate: + """Separate estimates in one declared unit, period and source domain.""" + + domain: str + source: str + statistical_unit: str + period: str + basis: str + source_share: Fraction + rows: int + design_estimate: float + allocated_estimate: float + + +@dataclass(frozen=True) +class SurveyAllocation: + """Output weights aligned to input row IDs; estimates never pool units.""" + + row_ids: tuple[int, ...] + weights: Weights + estimates: tuple[DomainSourceEstimate, ...] + + +def _axis(values: Sequence, size: int, field: str) -> tuple: + if isinstance(values, (str, bytes)): + raise SurveyAllocationError(f"{field} must be a row-aligned sequence") + result = tuple(values) + if len(result) != size: + raise SurveyAllocationError(f"{field} must match the household weight axis") + return result + + +def allocate_domain_weights( + design_weights: Weights, + *, + row_ids: Sequence[int], + source_channels: Sequence[str], + domain_keys: Sequence[str | None], + unit_counts: Sequence[int], + domains: Sequence[SurveyDomain], +) -> SurveyAllocation: + """Multiply each household weight by its declared domain/source share. + + ``unit_counts`` is mandatory: one for a housing unit or individual GQ + placeholder, the number of covered people for a person-counting cluster. + It affects accounting, never the household weight multiplier. No source + total is normalized to another source. Unknown domains and positive-share + cells lacking positive design support refuse. Known exclusions must be + selected explicitly before this operation; rows are never dropped here. + + The declarations do not authenticate their inputs. A production graph + adapter must bind IDs, source membership, coverage and period assumptions + to the actual population and use a new allocation context/receipt version. + """ + if ( + not isinstance(design_weights, Weights) + or design_weights.kind is not WeightKind.DESIGN + ): + raise SurveyAllocationError("allocation requires DESIGN weights") + size = len(design_weights.values) + ids = _axis(row_ids, size, "row_ids") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**63 - 1 + for x in ids + ): + raise SurveyAllocationError( + "row_ids must be positive int64 household identities" + ) + if len(set(ids)) != size: + raise SurveyAllocationError("household identities must be unique") + sources = _axis(source_channels, size, "source_channels") + keys = _axis(domain_keys, size, "domain_keys") + counts = _axis(unit_counts, size, "unit_counts") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**53 + for x in counts + ): + raise SurveyAllocationError( + "unit_counts must be positive exactly representable integers" + ) + declared = {} + for domain in domains: + if not isinstance(domain, SurveyDomain) or domain.key in declared: + raise SurveyAllocationError( + "domains must be unique SurveyDomain declarations" + ) + declared[domain.key] = domain + cells: dict[tuple[str, str], list[int]] = {} + for position, (key, source) in enumerate(zip(keys, sources, strict=True)): + _label(source, "source channel") + if not isinstance(key, str) or key not in declared: + raise SurveyAllocationError("unknown or undeclared coverage domain") + if source not in declared[key].source_shares: + raise SurveyAllocationError( + "source is not declared for its coverage domain" + ) + cells.setdefault((key, source), []).append(position) + incoming = design_weights.values + allocated = np.zeros(size, dtype=np.float64) + unit_array = np.asarray(counts, dtype=np.float64) + estimates = [] + for key, domain in sorted(declared.items()): + for source, share in domain.source_shares.items(): + positions = np.asarray(cells.get((key, source), ()), dtype=np.int64) + values = incoming[positions] + if share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled support" + ) + with np.errstate(over="ignore", under="ignore", invalid="ignore"): + output = values * float(share) + design_total = float(np.sum(values * unit_array[positions])) + allocated_total = float(np.sum(output * unit_array[positions])) + if not np.isfinite(design_total) or not np.isfinite(allocated_total): + raise SurveyAllocationError("domain estimate overflow") + if share > 0 and np.any((values > 0) & (output == 0)): + raise SurveyAllocationError("positive allocated weight underflow") + allocated[positions] = output + estimates.append( + DomainSourceEstimate( + key, + source, + domain.statistical_unit, + domain.period, + domain.basis, + share, + len(positions), + design_total, + allocated_total, + ) + ) + return SurveyAllocation( + tuple(int(x) for x in ids), + Weights(allocated, WeightKind.IMPORTANCE), + tuple(estimates), + ) diff --git a/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py new file mode 100644 index 000000000..5d28fd57e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py @@ -0,0 +1,369 @@ +"""Whole-household domain sampling composed with survey-share allocation. + +Coverage columns are supplied declarations bound to this Frame's household +axis. This module does not authenticate their survey meaning, align periods, +classify sources, or authorize a production graph. It never normalizes a +sampled domain estimate to another estimate or to a mixed-unit weight sum. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral + +import numpy as np + +from microcosm.build.survey_allocation import ( + DomainSourceEstimate, + SurveyAllocationError, + SurveyDomain, + allocate_domain_weights, +) +from microcosm.frame import Frame, MassChange, WeightKind, Weights + +PROTOCOL = "microcosm.survey-domain-sample-allocation.v1" + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _id_digest(ids: Sequence[int]) -> str: + return hashlib.sha256(np.asarray(ids, dtype=" tuple[DomainHouseholdSelection, ...]: + """Use the same per-cell draw for native catalogues and full Frames. + + ``cells`` contains (domain, source) pairs, including declared empty cells. + IDs must be uniformly positive int64 integers or nonempty literal strings. + They are unique within each source, and sort numerically or lexically, + respectively. No ID conversion, source-share exclusion, classification, + weighting, or claim of full-source completeness occurs here. A source owner + must establish the complete eligible axis before calling this function. + + Each populated cell retains max(1, floor(fraction * N)) rows. Empty cells + have no inclusion probability; full selection consumes no random draw. + The existing protocol and PCG64 seed construction remain unchanged. + """ + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + axes = (row_ids, source_channels, domain_keys, cells) + if any(isinstance(axis, (str, bytes)) for axis in axes): + raise SurveyAllocationError("selection inputs must be row sequences") + ids, sources, domains, declared = (tuple(axis) for axis in axes) + if len(ids) != len(sources) or len(ids) != len(domains): + raise SurveyAllocationError("selection axes must have equal lengths") + integer_ids = all( + not isinstance(value, bool) + and isinstance(value, Integral) + and 0 < value <= 2**63 - 1 + for value in ids + ) + string_ids = all(type(value) is str and bool(value.strip()) for value in ids) + if not integer_ids and not string_ids: + raise SurveyAllocationError( + "selection IDs must be uniformly positive int64 or literal strings" + ) + if any( + not isinstance(value, str) or not value.strip() + for value in (*sources, *domains) + ): + raise SurveyAllocationError("selection source/domain labels must be nonempty") + if any( + type(cell) is not tuple + or len(cell) != 2 + or any(not isinstance(label, str) or not label.strip() for label in cell) + for cell in declared + ): + raise SurveyAllocationError("selection cells require (domain, source) pairs") + if len(set(declared)) != len(declared): + raise SurveyAllocationError("selection cells must be unique") + if len(set(zip(sources, ids, strict=True))) != len(ids): + raise SurveyAllocationError("selection identities must be unique within source") + groups: dict[tuple[str, str], list[int]] = {cell: [] for cell in declared} + for position, cell in enumerate(zip(domains, sources, strict=True)): + if cell not in groups: + raise SurveyAllocationError("selection row has an undeclared cell") + groups[cell].append(position) + result = [] + for (domain, source), group in sorted(groups.items()): + positions = np.asarray(sorted(group, key=ids.__getitem__), dtype=np.int64) + eligible = len(positions) + if not eligible: + chosen, probability = (), None + else: + count = max(1, fraction.numerator * eligible // fraction.denominator) + probability = Fraction(count, eligible) + if count == eligible: + chosen = positions + else: + material = _json([PROTOCOL, seed, source, domain]) + cell_seed = int.from_bytes(hashlib.sha256(material).digest(), "big") + rng = np.random.Generator(np.random.PCG64(cell_seed)) + chosen = np.sort(rng.choice(positions, size=count, replace=False)) + result.append( + DomainHouseholdSelection( + domain, source, eligible, tuple(int(p) for p in chosen), probability + ) + ) + return tuple(result) + + +@dataclass(frozen=True) +class SampledDomainCell: + """One source/domain estimate in its declared unit, period and basis.""" + + full: DomainSourceEstimate + selected_households: int + inclusion_probability: Fraction | None + selected_household_ids_sha256: str + sampled_allocated_estimate: float + + +@dataclass(frozen=True) +class DomainSampleAllocation: + """A sampled IMPORTANCE Frame and separate, immutable domain accounting.""" + + frame: Frame + cells: tuple[SampledDomainCell, ...] + fraction: Fraction + seed: int + input_axis_sha256: str + domain_declarations_sha256: str + numpy_version: str + coverage_columns: tuple[str, str, str] + + def receipt(self) -> dict[str, object]: + """Describe the executed arithmetic without claiming source admission.""" + return { + "protocol": PROTOCOL, + "coverage_status": "declared_frame_columns_only", + "release_eligible": False, + "coverage_columns": dict( + zip( + ("source", "domain", "unit_count"), + self.coverage_columns, + strict=True, + ) + ), + "fraction": [self.fraction.numerator, self.fraction.denominator], + "seed": self.seed, + "rng": {"family": "PCG64", "numpy_version": self.numpy_version}, + "input_axis_sha256": self.input_axis_sha256, + "domain_declarations_sha256": self.domain_declarations_sha256, + "cells": [ + { + "domain": cell.full.domain, + "source": cell.full.source, + "statistical_unit": cell.full.statistical_unit, + "period": cell.full.period, + "basis": cell.full.basis, + "source_share": [ + cell.full.source_share.numerator, + cell.full.source_share.denominator, + ], + "eligible_households": cell.full.rows, + "selected_households": cell.selected_households, + "inclusion_probability": ( + None + if cell.inclusion_probability is None + else [ + cell.inclusion_probability.numerator, + cell.inclusion_probability.denominator, + ] + ), + "selected_household_ids_sha256": cell.selected_household_ids_sha256, + "full_design_estimate": cell.full.design_estimate, + "full_allocated_estimate": cell.full.allocated_estimate, + "sampled_allocated_estimate": cell.sampled_allocated_estimate, + } + for cell in self.cells + ], + } + + +def sample_and_allocate_domains( + frame: Frame, + *, + source_column: str, + domain_column: str, + unit_count_column: str, + domains: Sequence[SurveyDomain], + fraction: Fraction, + seed: int, +) -> DomainSampleAllocation: + """Sample each declared source/domain cell, then compose its multipliers. + + Each populated cell retains ``max(1, floor(fraction * N))`` households by + simple random sampling without replacement. Its output household weight + is ``source_design_weight * domain_source_share * N / n``. Unit counts + affect estimates only. Small domains therefore remain represented, with + their own actual inclusion probability recorded; no totals are forced to + match. Full selection does not draw. + + PCG64 streams are separated by protocol, source and domain. Sorted household + IDs make selection independent of row order and unrelated cells. The + returned Frame preserves complete selected lineages through Frame.select. + It never labels the sampling/allocation result DESIGN. + + A positive-share cell whose random sample has no positive weight refuses. + Callers must not search seeds to evade that refusal. Coverage exclusions and + period harmonization must be resolved explicitly before this operation. + """ + if not isinstance(frame, Frame): + raise TypeError("domain sampling requires a Frame") + frame.revalidate() + if frame.weighted_entities != ("household",): + raise SurveyAllocationError("domain sampling requires household-only weights") + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + names = (source_column, domain_column, unit_count_column) + if any(not isinstance(name, str) or not name for name in names): + raise SurveyAllocationError("coverage column names must be nonempty strings") + if len(set(names)) != 3 or "household_id" in names: + raise SurveyAllocationError("coverage columns must be distinct from identities") + household = frame.table("household") + if not set(names).issubset(household): + raise SurveyAllocationError("household coverage columns are missing") + ids = household["household_id"].to_numpy() + sources = tuple(household[source_column]) + keys = tuple(household[domain_column]) + counts = tuple(household[unit_count_column]) + domains = tuple(domains) + weights = frame.weights_for("household") + full = allocate_domain_weights( + weights, + row_ids=ids, + source_channels=sources, + domain_keys=keys, + unit_counts=counts, + domains=domains, + ) + axis = hashlib.sha256() + for row, source, key, count, weight in zip( + ids, sources, keys, counts, weights.values, strict=True + ): + axis.update(_json([int(row), source, key, int(count), float(weight)])) + axis.update(b"\n") + declarations = [ + { + "key": domain.key, + "unit": domain.statistical_unit, + "period": domain.period, + "basis": domain.basis, + "shares": { + source: [share.numerator, share.denominator] + for source, share in domain.source_shares.items() + }, + } + for domain in sorted(domains, key=lambda domain: domain.key) + ] + selection = select_domain_households( + row_ids=ids, + source_channels=sources, + domain_keys=keys, + cells=tuple((estimate.domain, estimate.source) for estimate in full.estimates), + fraction=fraction, + seed=seed, + ) + selected_positions = [] + output_by_id = {} + cells = [] + for estimate, cell in zip(full.estimates, selection, strict=True): + chosen = np.asarray(cell.positions, dtype=np.int64) + selected = len(chosen) + probability = cell.inclusion_probability + if selected: + values = full.weights.values[chosen] + if estimate.source_share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled positive weight" + ) + with np.errstate(over="ignore", invalid="ignore"): + output = values * float(1 / probability) + sample_estimate = float( + np.sum(output * np.asarray([counts[p] for p in chosen])) + ) + if not np.isfinite(output).all() or not np.isfinite(sample_estimate): + raise SurveyAllocationError( + "sampled domain weight or estimate overflow" + ) + selected_positions.extend(chosen.tolist()) + output_by_id.update(zip(ids[chosen].tolist(), output.tolist(), strict=True)) + selected_ids = np.sort(ids[chosen]) + else: + selected, probability, sample_estimate = 0, None, 0.0 + selected_ids = () + cells.append( + SampledDomainCell( + estimate, + selected, + probability, + _id_digest(selected_ids), + sample_estimate, + ) + ) + selected_ids = ids[np.asarray(selected_positions, dtype=np.int64)] + if len(selected_ids) == len(ids): + sampled = frame + else: + membership = frame.schema.membership_column("household") + sampled = frame.select(frame.person[membership].isin(selected_ids).to_numpy()) + realized = sampled.table("household")["household_id"].to_numpy() + if set(realized) != set(selected_ids): + raise SurveyAllocationError("whole-household sample differs from selection") + result = sampled.with_weights( + "household", + Weights( + np.asarray([output_by_id[int(row)] for row in realized]), + WeightKind.IMPORTANCE, + ), + mass=MassChange( + factor=None, + reason=f"{PROTOCOL}: domain source shares and inverse household inclusion", + ), + ) + return DomainSampleAllocation( + result, + tuple(cells), + fraction, + seed, + axis.hexdigest(), + hashlib.sha256(_json(declarations)).hexdigest(), + np.__version__, + names, + ) diff --git a/packages/microcosm-build/src/microcosm/build/table_identity.py b/packages/microcosm-build/src/microcosm/build/table_identity.py new file mode 100644 index 000000000..7dcc9725f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/table_identity.py @@ -0,0 +1,365 @@ +"""Native table identity without constructing a population Frame. + +The canonical_scalar_v1 values codec is factored unchanged from the US late +producer runtime. Its domain labels and optional string normalization remain +stable for existing receipts. native_table_identity additionally binds a closed +native dtype/schema descriptor, including pandas string storage and NA kind. +""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes + +TABLE_VALUES_DIGEST_CODEC = "canonical_scalar_v1" +_TABLE_DIGEST_CHUNK_ROWS = 65_536 + + +def _digest_part( + digest, + *, + domain: str, + payload: bytes | bytearray | memoryview | np.ndarray, +) -> None: + """Append one length-framed, domain-separated byte field to a digest.""" + + domain_bytes = domain.encode("utf-8") + payload_view = memoryview(payload) + if payload_view.format != "B" or payload_view.ndim != 1: + payload_view = payload_view.cast("B") + digest.update(struct.pack(" np.ndarray: + """Return a contiguous, explicitly little-endian numeric byte source.""" + + array = np.asarray(values) + array = array.astype(array.dtype.newbyteorder("<"), copy=False) + return np.ascontiguousarray(array) + + +def _scalar_bytes(value: object) -> bytes: + """Encode one supported object scalar without lossy intermediary hashes.""" + + missing = pd.isna(value) + if isinstance(missing, (bool, np.bool_)) and bool(missing): + return b"null" + if isinstance(value, (bool, np.bool_)): + return b"bool\x01" if bool(value) else b"bool\x00" + if isinstance(value, (int, np.integer)): + return b"integer\x00" + str(int(value)).encode("ascii") + if isinstance(value, (float, np.floating)): + if isinstance(value, np.floating) and value.dtype.itemsize > 8: + raise TypeError( + "US late-producer content digest does not support object " + f"floating scalar {value.dtype!s}." + ) + return b"float64\x00" + struct.pack(" 16: + raise TypeError( + "US late-producer content digest does not support object " + f"complex scalar {value.dtype!s}." + ) + numeric = complex(value) + return b"complex128\x00" + struct.pack(" None: + """Stream framed string or object scalars in bounded-memory chunks.""" + + for chunk_index, start in enumerate( + range(0, len(values), _TABLE_DIGEST_CHUNK_ROWS) + ): + stop = min(start + _TABLE_DIGEST_CHUNK_ROWS, len(values)) + lengths = np.zeros(stop - start, dtype=" None: + """Hash one ordered logical Series with explicit dtype and null domains.""" + + dtype = series.dtype + missing = series.isna().to_numpy(dtype=bool) + _digest_part( + digest, + domain=f"{domain}/dtype", + payload=str(dtype).encode("utf-8"), + ) + _digest_part( + digest, + domain=f"{domain}/row_count", + payload=struct.pack(" str: + """Hash ordered table scalars directly with typed, null-aware framing.""" + + values = ( + canonicalize_table_string_dtypes( + table, + boundary="late-producer content digest", + table_name="declared_surface", + ) + if normalize_strings + else table + ) + if isinstance(values.index, pd.MultiIndex): + index_levels = [ + pd.Series(values.index.get_level_values(level), copy=False) + for level in range(values.index.nlevels) + ] + else: + index_levels = [pd.Series(values.index, copy=False)] + header = { + "codec": TABLE_VALUES_DIGEST_CODEC, + "columns": [str(column) for column in values.columns], + "dtypes": [ + str(values.iloc[:, index].dtype) for index in range(values.shape[1]) + ], + "index_type": type(values.index).__name__, + "index_dtype": str(values.index.dtype), + "index_level_dtypes": [str(level.dtype) for level in index_levels], + "index_names": [ + None if name is None else str(name) for name in values.index.names + ], + } + digest = hashlib.sha256() + _digest_part( + digest, + domain="late_table_digest_codec", + payload=TABLE_VALUES_DIGEST_CODEC.encode("ascii"), + ) + _digest_part( + digest, + domain="late_table_header_json", + payload=json.dumps( + header, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8"), + ) + for level_index, level in enumerate(index_levels): + _digest_series_values( + digest, + level, + domain=f"index_level/{level_index}", + ) + for column_index in range(values.shape[1]): + _digest_series_values( + digest, + values.iloc[:, column_index], + domain=f"column/{column_index}", + ) + return digest.hexdigest() + + +def _native_dtype(dtype: object) -> dict[str, str]: + if isinstance(dtype, pd.StringDtype): + return { + "kind": "pandas_string", + "storage": dtype.storage, + "na_value": "pd.NA" if dtype.na_value is pd.NA else "nan", + } + if isinstance(dtype, np.dtype) and dtype.kind in "biufO": + return {"kind": "numpy", "dtype": dtype.str} + if isinstance( + dtype, + ( + pd.BooleanDtype, + pd.Int8Dtype, + pd.Int16Dtype, + pd.Int32Dtype, + pd.Int64Dtype, + pd.UInt8Dtype, + pd.UInt16Dtype, + pd.UInt32Dtype, + pd.UInt64Dtype, + pd.Float32Dtype, + pd.Float64Dtype, + ), + ): + return {"kind": "pandas_nullable", "dtype": str(dtype)} + raise TypeError(f"Unsupported native table identity dtype: {dtype!r}.") + + +def native_table_identity(table: pd.DataFrame) -> dict[str, Any]: + """Bind native values, order, index and a closed exact dtype schema. + + This narrow identity supports numeric, boolean and string donor tables, + including nullable numeric columns. Categorical/temporal/custom extension + dtypes and MultiIndex axes require a separately declared contract. Nulls + retain their positions; individual floating NaN payload bits are canonical. + No normalization or coercion is performed on the caller's table. + """ + if not isinstance(table, pd.DataFrame): + raise TypeError("Native table identity requires a pandas DataFrame.") + if isinstance(table.index, pd.MultiIndex) or isinstance( + table.columns, pd.MultiIndex + ): + raise TypeError("Native table identity does not support MultiIndex axes.") + if table.columns.has_duplicates or not all( + isinstance(x, str) and x for x in table.columns + ): + raise TypeError( + "Native table identity requires unique nonempty string columns." + ) + for axis in (table.index, table.columns): + if axis.name is not None and not isinstance(axis.name, str): + raise TypeError( + "Native table identity requires string or absent axis names." + ) + series = [ + pd.Series(table.index), + *(table.iloc[:, i] for i in range(table.shape[1])), + ] + for values in series: + if values.dtype == object and not all( + isinstance(x, str) for x in values.dropna() + ): + raise TypeError("Native object dtype may contain only strings or nulls.") + identity = { + "schema": "microcosm.native_table_identity.v1", + "values_sha256": table_values_sha256(table, normalize_strings=False), + "index": { + "type": type(table.index).__name__, + "name": table.index.name, + "dtype": _native_dtype(table.index.dtype), + }, + "column_axis": { + "type": type(table.columns).__name__, + "name": table.columns.name, + "dtype": _native_dtype(table.columns.dtype), + }, + "columns": [ + {"name": name, "dtype": _native_dtype(table[name].dtype)} + for name in table.columns + ], + } + identity["sha256"] = hashlib.sha256( + json.dumps( + identity, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return identity diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py new file mode 100644 index 000000000..dca569472 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py @@ -0,0 +1,143 @@ +"""Bounded, checksummed codec preserving target-only money source authority.""" + +import struct +from hashlib import sha256 + +from .asec_current_money import ( + _STATE_TOKEN, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + AuthenticatedAsecSource, + MoneyBindings, + MoneyField, + ReadyCurrentMoney, + RestatedAsecMoney, + SyntheticReadyCurrentMoney, + _parse, + _require, + _validate_money, + require_complete_current_money, +) + +MAGIC = b"MCAM2024\x01" +_CONTENT_DOMAIN = b"microcosm/asec-current-money-content/1\0" +# 32 person fields, one household observation, 8+1+1+1 bytes per field/row. +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 +) + + +def _ready_parts( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> tuple[bytes, ...]: + """One canonical serialization shared by transport and content identity.""" + _require( + type(ready) in (SyntheticReadyCurrentMoney, ReadyCurrentMoney), + "SYNTHETIC_READY_REQUIRED", + ) + _validate_money(ready, ready.bindings.spec, nominal=False) + # Recheck completeness even for a directly constructed or replaced value. + require_complete_current_money( + RestatedAsecMoney(ready.bindings, ready.fields, _token=_STATE_TOKEN), + ready.bindings.spec, + production=type(ready) is ReadyCurrentMoney, + ) + parts = [MAGIC, struct.pack(" str: + """Bind the full canonical header and field body, apart from transport checksum.""" + digest = sha256(_CONTENT_DOMAIN) + for part in _ready_parts(ready): + digest.update(part) + return digest.hexdigest() + + +def encode_current_money( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> bytes: + """Encode immutable target amounts and evidence, including field-scoped origins. + + Schema 1 retains legacy zero provenance. Schema 2 binds the restored-source + PTOTVAL origin policy through the exact header/spec; the byte widths and + checksum framing are unchanged. Per-field validation rejects swapped codes. + """ + parts = _ready_parts(ready) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_current_money( + payload: bytes, + expected_bindings: MoneyBindings, + *, + expected: ReadyCurrentMoney | None = None, +) -> SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Authenticate replay content against issued readiness before state creation. + + Verified source bindings alone cannot authorize a restated amount body. + Synthetic replay retains its binding-only API and refuses an expected object. + """ + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(type(expected_bindings) is MoneyBindings, "EXPECTED_BINDINGS_REQUIRED") + expected_bindings.__post_init__() + authenticated = type(expected_bindings.spec.source) is AuthenticatedAsecSource + if authenticated: + _require(type(expected) is ReadyCurrentMoney, "EXPECTED_CONTENT_REQUIRED") + _require(expected.bindings == expected_bindings, "EXPECTED_CONTENT_BINDING") + _validate_money(expected, expected_bindings.spec, nominal=False) + else: + _require(expected is None, "EXPECTED_CONTENT_SYNTHETIC") + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + """Require genuine-summary fields, numeric domains and registered bands. + + This validates evidence shape and the fixed decision policy, not the source + identity or truth of the supplied measurements. Genuine summary functions + emit Python integers/floats and list-valued bands; immutable tuple bands are + also accepted for callers that freeze the same data in memory. + """ + prefix = f"Malformed {family}-input summary" + fields = { + "unique_counts", + *share_bands, + *(name for name, _ in share_bands.values()), + *invariants, + } + if not isinstance(summary, Mapping) or set(summary) != fields: + raise ValueError(f"{prefix}: expected the exact registered field inventory.") + counts = summary["unique_counts"] + if not isinstance(counts, Mapping) or set(counts) != set(outputs): + raise ValueError(f"{prefix}: expected a count for every registered output.") + for name, value in (*counts.items(), *((key, summary[key]) for key in invariants)): + if type(value) is not int or value < 0: + raise ValueError(f"{prefix}: {name!r} must be a nonnegative integer.") + for share_name, (band_name, registered) in share_bands.items(): + share = summary[share_name] + # The closed interval also refuses both infinities and NaN without + # coercing strings, bools, or oversized integer evidence to float64. + if type(share) not in (int, float) or not 0.0 <= share <= 1.0: + raise ValueError(f"{prefix}: {share_name!r} must be finite in [0, 1].") + band = summary[band_name] + if ( + not isinstance(band, (list, tuple)) + or len(band) != 2 + or any(type(value) not in (int, float) for value in band) + or tuple(band) != registered + ): + raise ValueError( + f"{prefix}: {band_name!r} differs from its registered band." + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py new file mode 100644 index 000000000..fc4928883 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py @@ -0,0 +1,203 @@ +"""Pure, unbound ACS housing-unit semantics; no source authentication. + +ACS TYPEHUGQ/NP observations do not supply the CPS interview-scope or +household-kind observations. Those output axes therefore remain source-specific +not-observed codes, even for a known occupied ACS housing unit. Invalid TEN +payloads are uninterpreted placeholders, never observed zeros. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +__all__ = ["ACSHousingUniverseRefusalError", "classify_acs_housing_universe"] + + +class ACSHousingUniverseRefusalError(ValueError): + """Refuse an invalid input contract without reporting source values or keys.""" + + +def _owned_index(index: pd.Index) -> pd.Index: + """Copy axis storage, reconstructing indexes with shared deep-copy caches.""" + if isinstance(index, pd.RangeIndex): + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + if isinstance(index, pd.MultiIndex): + return pd.MultiIndex( + levels=[_owned_index(level) for level in index.levels], + codes=[code.copy() for code in index.codes], + names=list(index.names), + sortorder=index.sortorder, + verify_integrity=False, + ) + if isinstance(index, pd.CategoricalIndex): + return pd.CategoricalIndex( + pd.Categorical.from_codes( + index.codes.copy(), + categories=_owned_index(index.categories), + ordered=index.ordered, + ), + name=index.name, + ) + return index.copy(deep=True) + + +def _header() -> dict: + """Fresh semantic metadata only: no source rows, keys, counts, or pins.""" + return { + "schema_version": 1, + "artifact_kind": "microcosm.acs_housing_universe_unbound.v1", + "source_authentication": "unbound_semantic_classifier", + "release_eligible": False, + "source_mapping": { + "TYPEHUGQ": { + "1": "housing_unit", + "2": "institutional_group_quarters", + "3": "noninstitutional_group_quarters", + }, + "NP": "nonnegative int64; no additional upper bound", + "TEN": "native ACS codes; see tenure_subtype codebook", + "occupied_hu": ( + "TYPEHUGQ=1 and NP>0 establishes occupancy; TYPEHUGQ=2/3 " + "is outside the HU universe; TYPEHUGQ=1 and NP=0 leaves " + "occupancy unresolved and does not assert vacancy" + ), + "hu_tenure_class": ( + "only known occupied HUs: valid TEN=1/2 owner, valid TEN=3/4 " + "renter, invalid TEN unclassified but retained in the HU total" + ), + "interview_scope": "CPS interview observation not supplied by ACS inputs", + "household_kind": "CPS household-kind observation not supplied by ACS inputs", + "ignored_columns": "all other columns, including model tenure_type", + "row_policy": "retain every row in original order, including all GQ rows", + }, + "codebooks": { + "interview_scope": {"0": "not_observed_in_these_ACS_inputs"}, + "physical_unit": {"1": "housing_unit", "2": "group_quarters"}, + "household_kind": {"0": "not_observed_in_these_ACS_inputs"}, + "tenure_subtype": { + "0": "unavailable", + "1": "owned_with_mortgage_or_loan", + "2": "owned_outright", + "3": "paying_rent", + "4": "no_rent_paid", + }, + "occupied_hu": { + "0": "occupancy_not_established", + "1": "occupied_housing_unit", + "2": "outside_housing_unit_universe", + }, + "hu_tenure_class": { + "0": "outside_or_unresolved_occupied_HU_universe", + "1": "owner", + "2": "renter_including_no_rent", + "3": "occupied_HU_tenure_unclassified", + }, + "unresolved_reasons": { + "0": "no_unresolved_reason", + "1": "housing_unit_occupancy_not_established", + "2": "occupied_housing_unit_tenure_unavailable", + }, + "TEN_valid": {"0": "unavailable", "1": "observed_native_TEN"}, + }, + "validity_placeholder_semantics": { + "input": "exact numpy int64 TYPEHUGQ/NP/TEN and positional bool ndarray mask", + "invalid_TEN": ( + "any int64 payload is an uninterpreted placeholder; never an " + "observed zero and never checked against the observed TEN domain" + ), + "output": "uint8 codes; tenure_subtype=0 iff TEN_valid=0 on every row", + "GQ": "unavailable TEN, including a blank, adds no conflict or reason", + "unresolved_reasons": "bit mask: 1 occupancy not established; 2 occupied HU TEN unavailable", + }, + "limits": [ + "no_authenticated_source_or_source_pins", + "no_CPS_interview_or_household_kind_observations", + "no_B19001_income_concept_bridge", + "no_scientific_certification", + "no_stacked_population_or_denominator_certification", + ], + } + + +def classify_acs_housing_universe( + households: pd.DataFrame, *, tenure_valid: np.ndarray +) -> tuple[pd.DataFrame, dict]: + """Classify supplied ACS observations without authenticating their source. + + Require an exact DataFrame with unique columns and native numpy int64 + TYPEHUGQ, NP, and TEN. TYPEHUGQ must be 1/2/3 and NP nonnegative. The + positional validity argument must be an exact one-dimensional bool ndarray + of matching length. Only valid TEN cells are observed and require codes + 1/2/3/4; invalid payloads can be any int64. No coercion or nullable path is + provided. Other columns, including tenure_type, are ignored. + + Return an owned uint8 DataFrame preserving every row and its index, plus a + fresh, data-independent semantic header. Both CPS-specific axes stay zero + (not observed). NP=0 does not establish vacancy. Unknown occupied-HU tenure + stays explicitly unclassified within the total; GQ tenure never changes + the HU universe. The classifier performs no I/O or source authentication. + """ + if type(households) is not pd.DataFrame: + raise ACSHousingUniverseRefusalError("households_must_be_exact_dataframe") + if not households.columns.is_unique: + raise ACSHousingUniverseRefusalError("household_columns_must_be_unique") + required = ("TYPEHUGQ", "NP", "TEN") + if not set(required).issubset(set(households.columns)): + raise ACSHousingUniverseRefusalError("required_ACS_columns_missing") + for column in required: + dtype = households[column].dtype + if not isinstance(dtype, np.dtype) or dtype != np.dtype(np.int64): + raise ACSHousingUniverseRefusalError("ACS_columns_must_be_numpy_int64") + if ( + type(tenure_valid) is not np.ndarray + or tenure_valid.dtype != np.dtype(np.bool_) + or tenure_valid.ndim != 1 + or len(tenure_valid) != len(households) + ): + raise ACSHousingUniverseRefusalError( + "TEN_validity_must_be_matching_bool_vector" + ) + + unit_type = households["TYPEHUGQ"].to_numpy(copy=False) + persons = households["NP"].to_numpy(copy=False) + tenure = households["TEN"].to_numpy(copy=False) + if np.any((unit_type < 1) | (unit_type > 3)): + raise ACSHousingUniverseRefusalError("TYPEHUGQ_outside_domain") + if np.any(persons < 0): + raise ACSHousingUniverseRefusalError("NP_must_be_nonnegative") + observed_tenure = tenure[tenure_valid] + if np.any((observed_tenure < 1) | (observed_tenure > 4)): + raise ACSHousingUniverseRefusalError("valid_TEN_outside_domain") + + housing_unit = unit_type == 1 + occupied = housing_unit & (persons > 0) + occupancy_unknown = housing_unit & (persons == 0) + tenure_subtype = np.zeros(len(households), dtype=np.uint8) + tenure_subtype[tenure_valid] = observed_tenure + occupied_hu = np.zeros(len(households), dtype=np.uint8) + occupied_hu[occupied] = 1 + occupied_hu[~housing_unit] = 2 + hu_tenure_class = np.zeros(len(households), dtype=np.uint8) + hu_tenure_class[occupied & tenure_valid & (tenure_subtype <= 2)] = 1 + hu_tenure_class[occupied & tenure_valid & (tenure_subtype >= 3)] = 2 + hu_tenure_class[occupied & ~tenure_valid] = 3 + reasons = np.zeros(len(households), dtype=np.uint8) + reasons[occupancy_unknown] |= 1 + reasons[occupied & ~tenure_valid] |= 2 + + output = pd.DataFrame( + { + "interview_scope": np.zeros(len(households), dtype=np.uint8), + "physical_unit": np.where(housing_unit, 1, 2).astype(np.uint8), + "household_kind": np.zeros(len(households), dtype=np.uint8), + "tenure_subtype": tenure_subtype, + "occupied_hu": occupied_hu, + "hu_tenure_class": hu_tenure_class, + "unresolved_reasons": reasons, + "TEN_valid": tenure_valid.astype(np.uint8), + }, + index=_owned_index(households.index), + copy=True, + ) + return output, _header() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py new file mode 100644 index 000000000..0e0a39f1a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py @@ -0,0 +1,931 @@ +"""Closed ACS source observations, with a same-snapshot native Frame bridge. + +Source projections retain vacancies, both GQ classes and unknown occupied tenure. +Their graph transport is separate; source authentication requires reconstruction +from the two fixed Census archives. No source or model download occurs here. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import stat +import tempfile +import zipfile +from contextlib import contextmanager, suppress +from dataclasses import InitVar, dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token +from microcosm.graph.store import ContentStore + +from .acs_housing_universe import classify_acs_housing_universe +from .acs_inputs import map_acs_native_inputs +from .acs_pums import AcsPumsSource, _validate_person_counts, build_acs_pums_unit_frame +from .acs_sources import load_acs_source_manifest +from .graph_implementation import implementation_hash +from .operator_boundary import assert_operator_free_source_frame +from .source_csv_builtin import capture_csv_reader + +ACS_HU_STAGE = "acs_housing_universe_2024" +ACS_HU_CODEC = "us-acs-housing-universe-2024-v1" +ACS_HU_SOURCE_MAX_BYTES = 8 * 1024**3 +ACS_HU_RECEIPT_MAX_BYTES = 1024**2 +_CHUNK = 1024**2 +_MEMBER_MAX = 8 * 1024**3 +_EXPANDED_MAX = 16 * 1024**3 +_MEMBER_COUNT_MAX = 64 +_RECORD_MAX = 1024**2 +_FIELD_MAX = 64 * 1024 +_STATES = frozenset( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 21 22 23 24 25 " + "26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 " + "49 50 51 53 54 55 56".split() +) +_H_FIELDS = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +_P_FIELDS = ("SERIALNO", "SPORDER", "PWGTP") +_LINEAGE = ("source_member", "source_row_ordinal") +# No source/resource I/O merely to register the callable beside legacy codecs. +# Invented tests replace this private slot; public callers have no pin argument. +_ARCHIVE_PINS = None +_TOKEN = object() + + +class ACSHousingSourceError(ValueError): + """Sanitized source refusal; never print source keys or cells.""" + + +def _require(condition, code): + if not condition: + raise ACSHousingSourceError(code) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value) -> bytes: + return canonical_json(value) + + +def _parse_json(payload: bytes, cap: int) -> dict: + _require(type(payload) is bytes and len(payload) <= cap, "JSON_SIZE") + try: + value = json.loads(payload) + _require(type(value) is dict and _json(value) == payload, "CANONICAL_JSON") + except RecursionError: + raise ACSHousingSourceError("JSON_DEPTH") from None + return value + + +def _definition() -> tuple[dict, str]: + raw = Path(__file__).with_name("acs_2024_housing_universe.json").read_bytes() + return json.loads(raw), _sha(raw) + + +def _implementation() -> str: + _require(capture_csv_reader(csv) is not None, "SOURCE_CSV_READER_CHANGED") + return implementation_hash(ACS_HU_STAGE) + + +def _pins(): + if _ARCHIVE_PINS is not None: + return _ARCHIVE_PINS + return tuple( + (a.role, a.filename, a.sha256, a.size_bytes) + for a in load_acs_source_manifest().artifacts + ) + + +def _identity(value: os.stat_result): + return tuple( + getattr(value, key) + for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _path_components(path: Path): + path = path.absolute() + for parent in reversed((path, *path.parents)): + value = parent.lstat() + _require(not stat.S_ISLNK(value.st_mode), "SYMLINK_PATH") + return Path(os.path.abspath(path)) + + +def _directory(path: Path): + path = _path_components(path) + _require(stat.S_ISDIR(path.lstat().st_mode), "DIRECTORY_REQUIRED") + return path + + +def _copy(source: Path, destination: Path, cap: int, *, exact_size=None): + source = _path_components(source) + fd = os.open(source, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE_REQUIRED") + _require(before.st_size <= cap, "FILE_TOO_LARGE") + if exact_size is not None: + _require(before.st_size == exact_size, "SOURCE_SIZE") + digest, count = hashlib.sha256(), 0 + with destination.open("xb") as output: + while chunk := stream.read(min(_CHUNK, cap - count + 1)): + count += len(chunk) + _require(count <= cap, "FILE_GREW") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + _require( + count == before.st_size + and _identity(before) == _identity(os.fstat(stream.fileno())) + and _identity(before) == _identity(source.lstat()), + "FILE_CHANGED", + ) + _path_components(source) + destination.chmod(0o400) + return digest.hexdigest() + + +def _persisted_sha(path, size): + digest, count = hashlib.sha256(), 0 + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and before.st_size == size, "SNAPSHOT_SIZE" + ) + while chunk := stream.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "SNAPSHOT_GREW") + digest.update(chunk) + _require( + count == size and _identity(before) == _identity(os.fstat(stream.fileno())), + "SNAPSHOT_CHANGED", + ) + return digest.hexdigest() + + +def _write(path: Path, data: bytes, cap: int): + _require(len(data) <= cap, "OUTPUT_SIZE") + with path.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + path.chmod(0o400) + _require(_persisted_sha(path, len(data)) == _sha(data), "PERSISTED_OUTPUT") + + +@contextmanager +def _capture(source_dir, snapshot_root): + root = _directory(Path(snapshot_root)) + source = _directory(Path(source_dir)) + _require(not root.is_relative_to(source), "SNAPSHOT_SOURCE_OVERLAP") + pins = _pins() + expected = {p[1] for p in pins} + _require(expected == {"csv_hus.zip", "csv_pus.zip"}, "SOURCE_ROSTER") + _require({p.name for p in source.iterdir()} == expected, "SOURCE_ROSTER") + # Full private projection and selected publication/candidate capture can + # coexist. Existing captures already reduce free space and are not deducted + # a second time. This bound is operational, not a source-size prediction. + required = ( + sum(p[3] for p in pins) + + 2 * ACS_HU_SOURCE_MAX_BYTES + + 2 * ACS_HU_RECEIPT_MAX_BYTES + + 1024**3 + ) + _require(shutil.disk_usage(root).free >= required, "INSUFFICIENT_DISK") + private = Path(tempfile.mkdtemp(prefix="acs-hu-capture-", dir=root)) + paths = {} + try: + for role, name, digest, size in pins: + target = private / name + actual = _copy(source / name, target, size, exact_size=size) + _require(actual == digest, "SOURCE_SHA256") + _require(_persisted_sha(target, size) == digest, "PERSISTED_SOURCE_SHA256") + paths[role] = target + yield private, paths, pins + _require(pins == _pins(), "SOURCE_AUTHORITY_CHANGED") + except Exception as error: + # Captures belong exclusively to this invocation. Keep failed evidence; + # no rename can replace an earlier artifact or caller input. + code = ( + str(error) + if isinstance(error, ACSHousingSourceError) + else "SOURCE_PREPARATION_REFUSED" + ) + # Recording is best effort and strictly subordinate: if the record + # cannot be written, the refusal it was recording still propagates. + with suppress(Exception): + failure = _json({"status": "failed", "reason": code}) + if not (private / "failure.json").exists(): + _write(private / "failure.json", failure, ACS_HU_RECEIPT_MAX_BYTES) + raise + + +def _csv_record(raw: bytes) -> list[str]: + csv_reader = capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require(0 < len(raw) <= _RECORD_MAX, "CSV_RECORD_SIZE") + _require(not raw.startswith(b"\xef\xbb\xbf"), "CSV_BOM") + if raw.endswith(b"\n"): + raw = raw[:-1] + if raw.endswith(b"\r"): + raw = raw[:-1] + _require( + bool(raw) and b"\x00" not in raw and b"\r" not in raw and b"\n" not in raw, + "CSV_PHYSICAL_RECORD", + ) + values = next( + csv_reader( + [raw.decode("utf-8", errors="strict")], + strict=True, + quoting=csv.QUOTE_MINIMAL, + ) + ) + _require( + all(len(v.encode("utf-8")) <= _FIELD_MAX for v in values), "CSV_FIELD_SIZE" + ) + return values + + +def _archive(path: Path, role: str): + """Validate the complete central directory before resolving any member name.""" + prefix = "psam_hus" if role == "household" else "psam_pus" + required = _H_FIELDS if role == "household" else _P_FIELDS + inventory, rows, role_header, fields, total_bytes = [], [], None, None, 0 + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + _require(0 < len(members) <= _MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require( + len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER" + ) + for member in members: + name = member.filename + mode = member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and "/" not in name + and "\\" not in name + and "\x00" not in name, + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and (stat.S_IFMT(mode) in (0, stat.S_IFREG)), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= _MEMBER_MAX, "ZIP_MEMBER_SIZE") + total_bytes += member.file_size + _require(total_bytes <= _EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) + or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + selected_bytes = 0 + for member in sorted(members, key=lambda m: m.filename): + applicable = member.filename.casefold().startswith(prefix) + digest, count, header_raw, header, record_count = ( + hashlib.sha256(), + 0, + b"", + None, + 0, + ) + with archive.open(member) as stream: + while True: + raw = ( + stream.readline( + min(_RECORD_MAX + 1, member.file_size - count + 1) + ) + if applicable + else stream.read(min(_CHUNK, member.file_size - count + 1)) + ) + if not raw: + break + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + if not applicable: + continue + tokens = _csv_record(raw) + if header is None: + header, header_raw = tokens, raw + _require( + all(header) and len(set(header)) == len(header), + "CSV_HEADER", + ) + _require(set(required) <= set(header), "CSV_REQUIRED_FIELDS") + if role == "household": + _require( + bool({"ST", "STATE"} & set(header)), + "CSV_STATE_REQUIRED", + ) + if role_header is None: + role_header = header + fields = ( + ( + *required, + *(n for n in ("ST", "STATE") if n in header), + ) + if role == "household" + else required + ) + _require(header == role_header, "CSV_ROLE_HEADER_MISMATCH") + positions = [header.index(f) for f in fields] + continue + _require(len(tokens) == len(header), "CSV_ROW_WIDTH") + record_count += 1 + row = [ + *(tokens[p] for p in positions), + member.filename, + record_count, + ] + selected_bytes += len(_json(row)) + _require( + selected_bytes <= ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_TOO_LARGE", + ) + rows.append(row) + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "compressed_bytes": member.compress_size, + "bytes": count, + "crc32": member.CRC, + "sha256": digest.hexdigest(), + "is_data": applicable, + "header_hex": header_raw.hex(), + "header_sha256": _sha(header_raw), + "header": header, + "rows": record_count, + } + ) + _require(role_header is not None, "ZIP_ROLE_MISSING") + return list((*fields, *_LINEAGE)), rows, inventory + + +def _integer(token, low, high, label): + _require( + type(token) is str + and re.fullmatch(r"[0-9]+", token, flags=re.ASCII) is not None + and len(token) <= 5, + label, + ) + value = int(token) + _require(low <= value <= high, label) + return value + + +def _tables(projection): + _require( + set(projection) + == {"format", "household_columns", "person_columns", "households", "persons"} + and projection["format"] == "acs-housing-lexical-projection/1", + "PROJECTION_SCHEMA", + ) + hcols, pcols = projection["household_columns"], projection["person_columns"] + _require( + hcols + in [ + list((*_H_FIELDS, *state, *_LINEAGE)) + for state in (("ST",), ("STATE",), ("ST", "STATE")) + ] + and pcols == list((*_P_FIELDS, *_LINEAGE)), + "PROJECTION_COLUMNS", + ) + for cols, name in ((hcols, "households"), (pcols, "persons")): + _require(type(projection[name]) is list, "PROJECTION_ROWS") + for row in projection[name]: + _require( + type(row) is list + and len(row) == len(cols) + and all(type(v) is str for v in row[:-1]) + and type(row[-1]) is int + and row[-1] > 0, + "PROJECTION_ROW", + ) + household = pd.DataFrame(projection["households"], columns=hcols) + person = pd.DataFrame(projection["persons"], columns=pcols) + _require(not household.empty, "EMPTY_HOUSEHOLD_SOURCE") + for table in (household, person): + _require( + all( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", value, flags=re.ASCII) + and int(value[-7:]) > 0 + for value in table.SERIALNO + ), + "SERIALNO", + ) + _require(not household.SERIALNO.duplicated().any(), "DUPLICATE_HOUSEHOLD") + _require(set(person.SERIALNO) <= set(household.SERIALNO), "ORPHAN_PERSON") + typed = pd.DataFrame(index=household.index) + for name, low, high in (("TYPEHUGQ", 1, 3), ("NP", 0, 20), ("WGTP", 0, 9999)): + typed[name] = np.asarray( + [_integer(v, low, high, name) for v in household[name]], dtype="int64" + ) + valid = household.TEN.to_numpy() != "" + typed["TEN"] = np.asarray( + [ + _integer(v, 1, 4, "TEN") if present else 0 + for v, present in zip(household.TEN, valid, strict=True) + ], + dtype="int64", + ) + lines = np.asarray( + [ + _integer(v, 1, 20, "SPORDER") + if len(v) <= 2 + else _integer("", 1, 20, "SPORDER") + for v in person.SPORDER + ], + dtype="int64", + ) + pw = np.asarray( + [_integer(v, 1, 9999, "PWGTP") for v in person.PWGTP], dtype="int64" + ) + _require( + not pd.MultiIndex.from_arrays([person.SERIALNO, lines]).duplicated().any(), + "DUPLICATE_PERSON", + ) + state = "ST" if "ST" in household else "STATE" + _require(set(household[state]) <= _STATES, "STATE_SCOPE") + if "ST" in household and "STATE" in household: + _require(household.ST.equals(household.STATE), "STATE_CONFLICT") + for v in household.PUMA: + _require(len(v) == 5, "PUMA") + _integer(v, 100, 81003, "PUMA") + gq = typed.TYPEHUGQ.to_numpy() != 1 + vacant = ~gq & (typed.NP.to_numpy() == 0) + _require( + np.array_equal(household.SERIALNO.str.startswith("2024GQ").to_numpy(), gq), + "SERIALNO_TYPE", + ) + _require(bool((typed.NP.to_numpy()[gq] == 1).all()), "GQ_NP") + _require( + bool((typed.WGTP.to_numpy()[gq] == 0).all()) + and bool((typed.WGTP.to_numpy()[~gq] > 0).all()), + "WGTP_SCOPE", + ) + _require(not bool(valid[gq | vacant].any()), "TEN_NIU_SCOPE") + full_h = typed.assign(SERIALNO=household.SERIALNO) + _validate_person_counts(full_h, person) + codes, _header = classify_acs_housing_universe( + typed.loc[:, ["TYPEHUGQ", "NP", "TEN"]], tenure_valid=valid.astype(bool) + ) + return household, person, typed, lines, pw, codes + + +def _select(projection, serialnos): + household, person, typed, lines, _pw, _codes = _tables(projection) + if serialnos is None: + chosen = tuple(sorted(household.SERIALNO)) + else: + _require( + type(serialnos) is tuple + and bool(serialnos) + and all(type(v) is str for v in serialnos) + and len(set(serialnos)) == len(serialnos), + "SELECTION_KEYS", + ) + _require(set(serialnos) <= set(household.SERIALNO), "SELECTION_UNKNOWN") + chosen = tuple(sorted(serialnos)) + horder = household.sort_values("SERIALNO", kind="stable").index + porder = ( + person.assign(_line=lines) + .sort_values(["SERIALNO", "_line"], kind="stable") + .index + ) + chosen_set = set(chosen) + selected = { + **projection, + "households": [ + projection["households"][i] + for i in horder + if household.SERIALNO[i] in chosen_set + ], + "persons": [ + projection["persons"][i] for i in porder if person.SERIALNO[i] in chosen_set + ], + } + full_counts = { + "households": len(household), + "persons": len(person), + "vacant": int(((typed.TYPEHUGQ == 1) & (typed.NP == 0)).sum()), + "institutional_gq": int((typed.TYPEHUGQ == 2).sum()), + "noninstitutional_gq": int((typed.TYPEHUGQ == 3).sum()), + } + return selected, chosen, full_counts + + +@dataclass(frozen=True) +class AuthenticatedACSHousingSource: + """Owned immutable source bytes; DataFrame access returns independent copies.""" + + projection_json: bytes + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "AUTHENTICATED_SOURCE_CONSTRUCTOR") + + @property + def households(self): + return _tables(json.loads(self.projection_json))[0] + + @property + def persons(self): + return _tables(json.loads(self.projection_json))[1] + + @property + def codes(self): + return _tables(json.loads(self.projection_json))[5] + + +def _reconstruct(private, paths, pins, serialnos, implementation): + hc, h, hi = _archive(paths["household"], "household") + pc, p, pi = _archive(paths["person"], "person") + full = { + "format": "acs-housing-lexical-projection/1", + "household_columns": hc, + "person_columns": pc, + "households": h, + "persons": p, + } + full_bytes = _json(full) + _write(private / "full-projection.json", full_bytes, ACS_HU_SOURCE_MAX_BYTES) + selected, chosen, counts = _select(full, serialnos) + projection = _json(selected) + _require(len(projection) <= ACS_HU_SOURCE_MAX_BYTES, "PROJECTION_TOO_LARGE") + definition, definition_sha = _definition() + receipt = { + "format": "microcosm.acs_housing_universe_source.v1", + "release_eligible": False, + "vintage": 2024, + "definition_sha256": definition_sha, + "source_definitions": [ + definition["vacancy_definition"], + definition["gq_definition"], + ], + "implementation_sha256": implementation, + "parser_profile": { + "engine": "stdlib_csv_reader", + "encoding": "utf-8", + "quoting": "QUOTE_MINIMAL", + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + "field_max_bytes": _FIELD_MAX, + "record_max_bytes": _RECORD_MAX, + }, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} for r, n, d, s in pins + ], + "members": {"household": hi, "person": pi}, + "full_projection_sha256": _sha(full_bytes), + "full_counts": counts, + "selection": "all" if serialnos is None else "exact_serialnos", + "selected_serialnos_sha256": _sha(_json(chosen)), + "selected_counts": { + "households": len(selected["households"]), + "persons": len(selected["persons"]), + }, + "projection_sha256": _sha(projection), + "projection_bytes": len(projection), + "source_weight_kind": "design", + "calibrated_descendants_approved": False, + } + receipt_bytes = _json(receipt) + _require(len(receipt_bytes) <= ACS_HU_RECEIPT_MAX_BYTES, "RECEIPT_TOO_LARGE") + return AuthenticatedACSHousingSource(projection, receipt_bytes, _token=_TOKEN) + + +def _run_source(source_dir, snapshot_root, serialnos, output_dir, readback): + try: + original = _directory(Path(source_dir)) + destination = Path(os.path.abspath(output_dir)) + _require(not destination.is_relative_to(original), "OUTPUT_SOURCE_OVERLAP") + if readback: + _directory(destination) + else: + _directory(destination.parent) + _require(not destination.exists(), "OUTPUT_EXISTS") + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + result = _reconstruct(private, paths, pins, serialnos, implementation) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + if readback: + _directory(destination) + _require( + {p.name for p in destination.iterdir()} + == {"projection.json", "receipt.json"}, + "OUTPUT_ROSTER", + ) + for name, expected, cap in ( + ( + "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ), + ("receipt.json", result.receipt_json, ACS_HU_RECEIPT_MAX_BYTES), + ): + target = private / ("candidate-" + name) + # SOURCE_SIZE and FILE_TOO_LARGE are the pinned-archive + # codes. A candidate of the wrong length is a different + # fact, and an operator has to be able to tell them apart, + # so only this call's size refusals are renamed. + try: + digest = _copy( + destination / name, target, cap, exact_size=len(expected) + ) + except ACSHousingSourceError as error: + if str(error) not in ("SOURCE_SIZE", "FILE_TOO_LARGE"): + raise + raise ACSHousingSourceError("RECONSTRUCTION_SIZE") from None + _require( + digest == _sha(expected) + and _persisted_sha(target, len(expected)) == digest, + "RECONSTRUCTION_MISMATCH", + ) + else: + _directory(destination.parent) + destination.mkdir(mode=0o700, exist_ok=False) + _write( + destination / "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ) + _write( + destination / "receipt.json", + result.receipt_json, + ACS_HU_RECEIPT_MAX_BYTES, + ) + return result + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("SOURCE_PREPARATION_REFUSED") from None + + +def produce_acs_housing_source( + source_dir, *, snapshot_root, output_dir, serialnos=None +): + """Authenticate full parents, retain whole selected keys, publish exclusively.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, False) + + +def load_acs_housing_source(source_dir, output_dir, *, snapshot_root, serialnos=None): + """Fresh full-parent reconstruction; candidate files never supply authority.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, True) + + +def verify_acs_frame_projection(frame: Frame, projection: dict): + """Check a Frame against source observations, without granting source authority.""" + household, person, typed, lines, pw, _codes = _tables(projection) + h = frame.table("household") + p = frame.person + expected_h = household.loc[typed.NP > 0].reset_index(drop=True) + _require( + list(h.SERIALNO.astype(str)) == list(expected_h.SERIALNO), + "FRAME_HOUSEHOLD_ORIGIN", + ) + indexed = household.set_index("SERIALNO") + observed = indexed.loc[h.SERIALNO.astype(str)] + # The unchanged Frame builder retains TYPEHUGQ/NP/TEN but not raw WGTP. + # WGTP stays lexical in the source artifact and binds the typed design + # weights below, including the distinct GQ PWGTP substitution. + for field in ("TYPEHUGQ", "NP"): + _require( + np.array_equal(h[field].to_numpy(), observed[field].map(int).to_numpy()), + "FRAME_RAW_CONTROLS", + ) + valid = observed.TEN.to_numpy() != "" + _require(np.array_equal(h.TEN.notna().to_numpy(), valid), "FRAME_TEN_VALIDITY") + _require( + np.array_equal( + h.TEN.to_numpy()[valid], observed.TEN[valid].map(int).to_numpy() + ), + "FRAME_TEN_VALUES", + ) + state = "ST" if "ST" in observed else "STATE" + _require( + list(h.ST.astype(str)) == list(observed[state]) + and list(h.PUMA.astype(str)) == list(observed.PUMA), + "FRAME_GEOGRAPHY", + ) + ids = dict(zip(h.SERIALNO.astype(str), h.household_id, strict=True)) + _require(len(p) == len(person), "FRAME_PERSON_COUNT") + expected_ids = np.asarray([ids[s] for s in person.SERIALNO], dtype="int64") + _require( + np.array_equal(p.person_household_id.to_numpy(), expected_ids) + and np.array_equal(p.source_household_id.to_numpy(), expected_ids) + and np.array_equal(p.SPORDER.to_numpy(), lines) + and list(p.source_person_id.astype(str)) == [str(v) for v in lines], + "FRAME_PERSON_ORIGIN", + ) + _require( + np.array_equal(p.PWGTP.to_numpy(), pw) + and bool((p.source_year.to_numpy() == 2024).all()), + "FRAME_PERSON_SOURCE", + ) + _require( + frame.weighted_entities == ("household",) + and frame.weights_for("household").kind.value == "design", + "FRAME_WEIGHT_KIND", + ) + by_serial = dict(zip(person.SERIALNO, pw, strict=True)) + expected_weights = np.asarray( + [ + by_serial[serial] if int(weight) == 0 else int(weight) + for serial, weight in zip(observed.index, observed.WGTP, strict=True) + ], + dtype="float64", + ) + _require( + frame.weights_for("household").values.tobytes() == expected_weights.tobytes(), + "FRAME_DESIGN_WEIGHT", + ) + + +def frame_content_sha256(frame: Frame): + """Typed storage identity of every declared cell, axis, stratum and weight.""" + digest = hashlib.sha256(b"microcosm.acs-frame-storage/1\0") + + def series(values): + digest.update(_json({"dtype": str(values.dtype), "name": values.name})) + mask = values.isna().to_numpy(dtype=bool) + digest.update(mask.tobytes()) + array = values.to_numpy(copy=False) + if isinstance(values.dtype, np.dtype) and not values.dtype.hasobject: + digest.update(np.ascontiguousarray(array).tobytes()) + else: + for value, missing in zip(array, mask, strict=True): + item = ( + None + if missing + else value.item() + if isinstance(value, np.generic) + else value + ) + raw = _json(item) + digest.update(len(raw).to_bytes(8, "little")) + digest.update(raw) + + _require(frame.schema == US_SCHEMA, "FRAME_SCHEMA") + digest.update(_json(dict(frame.metadata))) + _require(not frame.mass_log, "FRAME_MASS_LOG") + for entity in frame.entities: + table = frame.table(entity) + digest.update(_json([entity, list(table.columns)])) + series(pd.Series(table.index.to_numpy(), name=table.index.name)) + for column in table: + series(table[column]) + series(frame.strata) + series(pd.Series(frame.strata.index.to_numpy(), name=frame.strata.index.name)) + for entity in frame.weighted_entities: + digest.update(_json([entity, frame.weights_for(entity).kind.value])) + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +@dataclass(frozen=True) +class PreparedACSHousingPopulation: + frame: Frame + source: AuthenticatedACSHousingSource + receipt_json: bytes + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +def prepare_acs_housing_population(source_dir, *, snapshot_root, serialnos=None): + """Validate the full lexical source; construct only exact selected native rows.""" + try: + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + source = _reconstruct(private, paths, pins, serialnos, implementation) + projection = json.loads(source.projection_json) + _require(bool(projection["persons"]), "EMPTY_GRAPH_POPULATION") + raw, _builder_receipt = build_acs_pums_unit_frame( + AcsPumsSource( + paths["household"], + paths["person"], + vintage=2024, + max_households=None, + ), + serialnos=serialnos, + ) + mapped = map_acs_native_inputs(raw) + frame = mapped.frame + assert_operator_free_source_frame( + frame, + label="ACS HU authenticated source", + native_inputs=mapped.native_inputs, + ) + verify_acs_frame_projection(frame, projection) + before = frame_content_sha256(frame) + transitions = [] + original_dtypes = { + (e, c): repr(frame.table(e)[c].dtype) + for e in frame.entities + for c in frame.table(e) + } + canonicalize_frame_string_dtypes( + frame, boundary="ACS HU graph storage", in_place=True + ) + for entity in frame.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + _require( + original.isna().equals(promoted.isna()) + and np.array_equal( + original[original.notna()].to_numpy(), + promoted[promoted.notna()].to_numpy(), + ), + "GRAPH_STRING_PROMOTION", + ) + table[column] = promoted + if original_dtypes[(entity, column)] != repr(table[column].dtype): + transitions.append( + { + "entity": entity, + "column": column, + "from": original_dtypes[(entity, column)], + "to": repr(table[column].dtype), + } + ) + verify_acs_frame_projection(frame, projection) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + receipt = { + "format": "microcosm.acs_housing_preparation.v2", + "release_eligible": False, + "source_receipt_sha256": _sha(source.receipt_json), + "projection_sha256": _sha(source.projection_json), + "implementation_sha256": implementation, + "same_snapshot_frame_and_observations": True, + "full_source_frame_before_selection": serialnos is None, + "full_source_lexical_projection": True, + "native_selection_before_person_accumulation": serialnos is not None, + "selection_kind": "all" if serialnos is None else "engineering_exact_keys", + "requested_serialnos": serialnos, + "full_source_inclusion_probability": None, + "pre_promotion_frame_sha256": before, + "frame_sha256": frame_content_sha256(frame), + "dtype_transitions": transitions, + "entity_rows": {e: frame.n(e) for e in frame.entities}, + "weight_kind": "design", + "HU_columns": "artifact_only", + } + return PreparedACSHousingPopulation(frame, source, _json(receipt)) + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("FRAME_PREPARATION_REFUSED") from None + + +def load_graph_acs_housing_universe(path, *, store=None): + """Frame codec with an explicit store and a dedicated private capture child.""" + _require(type(store) is ContentStore, "EXPLICIT_CONTENT_STORE_REQUIRED") + root = _directory(store.root) / "acs-hu-source-captures-v1" + root.mkdir(mode=0o700, exist_ok=True) + return prepare_acs_housing_population(path, snapshot_root=root).frame diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py new file mode 100644 index 000000000..9d3daf1cc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -0,0 +1,565 @@ +"""Closed real ACS preparation plus literal coverage issuance, without admission. + +Private process ownership is required in addition to immutable evidence bytes. +This is a native population successor, not a decoder or a graph attachment. +""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import sys +import tempfile +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import CodeType, FunctionType +from weakref import WeakKeyDictionary + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as coverage +from . import graph_implementation as implementation + +PROTOCOL = "microcosm.acs-native-coverage-binding.v2" +MAX_ARCHIVE_BYTES = 8 * 1024**3 # combined compressed bytes, before capture +MAX_EXPANDED_BYTES = 16 * 1024**3 # combined, before opening any member +MAX_SOURCE_ROWS = 6_000_000 # per role, before full-source construction +MAX_EVIDENCE_BYTES = 2 * 1024**2 +# Exact accepted direct AGEP -> A_AGE and AGEP -> age implementation, and owners. +# A new transform/owner version requires explicit review of this successor. +_ACCEPTED = { + "acs_pums.py": "3a80468c947775826516acebb51c28fba1bd35261cc9668ca72fdafe481a6690", + "acs_inputs.py": "aa4a8aeaba63dfef2f3e04fb89de59766deb088ed7f4d290aeba0425739916da", + "acs_housing_universe_source.py": "dfc23754587523edb68fb38d050852184b9b841559f15f1adc706f77b5aa9abc", + "acs_person_coverage_authentication.py": "475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff", +} +_TOKEN = object() +_ISSUED = WeakKeyDictionary() + + +class ACSNativeCoverageBindingError(ValueError): + """Static refusal without source values, paths or exception chains.""" + + +def _require(condition, code): + if not condition: + raise ACSNativeCoverageBindingError(code) + + +def _live_code(module, compiled): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ + path = getattr(module, "__file__", None) + if path is None: + return + tree = ast.parse(Path(path).read_bytes()) + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + value = getattr(module, node.name, None) + _require( + value is not None and value.__module__ == module.__name__, + "LOADED_PRODUCER", + ) + elif isinstance(node, ast.ImportFrom) and node.module != "__future__": + origin_name = ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), module.__package__ + ) + if node.level + else node.module + ) + origin = sys.modules.get(origin_name) + for alias in node.names: + if alias.name != "*" and origin is not None: + _require( + getattr(module, alias.asname or alias.name, None) + is getattr(origin, alias.name, None), + "LOADED_PRODUCER", + ) + + def check(function): + if not isinstance(function, FunctionType): + return + origin = sys.modules.get(function.__globals__.get("__name__")) + path = getattr(origin, "__file__", None) + _require(path is not None, "LOADED_PRODUCER") + if function.__code__.co_filename == "": + return # dataclass-generated methods + if path not in compiled: + codes = {} + + def visit(code): + codes[code.co_qualname] = code + for value in code.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(compile(Path(path).read_bytes(), path, "exec", dont_inherit=True)) + compiled[path] = codes + _require( + function.__globals__ is vars(origin) + and function.__code__ == compiled[path].get(function.__code__.co_qualname), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) + + for value in vars(module).values(): + if isinstance(value, FunctionType): + check(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + for method in vars(value).values(): + if isinstance(method, (classmethod, staticmethod)): + method = method.__func__ + if isinstance(method, property): + check(method.fget) + check(method.fset) + else: + check(method) + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-native-coverage-binding.v2", "NATIVE_PROTOCOL") + for name, expected in _ACCEPTED.items(): + _require( + coverage._sha(Path(__file__).with_name(name).read_bytes()) == expected, + "UNREVIEWED_PREPARATION", + ) + manifest = implementation.implementation_manifest(housing.ACS_HU_STAGE) + # Reuse the reviewed complete preparation closure, without changing its + # graph inventory or historical hashes. Include source-only coverage's own + # closure, C _csv provider (libpython where builtin), and this successor. + names = { + "microcosm.build.us_runtime.acs_person_coverage_authentication", + "microcosm.build.us_runtime.acs_person_coverage_columns", + __name__, + } + for item in manifest["modules"]: + package, relative = item.split("/", 1) + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: + _live_code(module, compiled) + return { + "preparation": manifest, + "coverage": coverage._producer(), + "issuer_sha256": coverage._sha(Path(__file__).read_bytes()), + "limits": [ + MAX_ARCHIVE_BYTES, + MAX_EXPANDED_BYTES, + MAX_SOURCE_ROWS, + MAX_EVIDENCE_BYTES, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_HOUSEHOLDS, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_PERSON_ROWS, + ], + "accepted_age_transform": "literal_numeric_identity_AGEP_to_AGEP_A_AGE_age", + } + + +def _archives(pins): + _require( + len(pins) == 2 and {p[0] for p in pins} == {"household", "person"}, + "SOURCE_ROLES", + ) + return [ + {"role": role, "filename": name, "sha256": digest, "bytes": size} + for role, name, digest, size in pins + ] + + +def _members_equal(left, right): + # Owners intentionally have distinct parser profiles and inventory shapes. + # Compare common captured byte/header/row identities, never parser claims. + fields = ("name", "bytes", "sha256", "compressed_bytes", "crc32", "rows") + for role in ("household", "person"): + a, b = left[role], right[role] + _require(len(a) == len(b), "SOURCE_MEMBER_IDENTITY") + for x, y in zip(a, b, strict=True): + _require(all(x[k] == y[k] for k in fields), "SOURCE_MEMBER_IDENTITY") + if y["applicable"]: + _require( + x["header_sha256"] == y["header_sha256"], "SOURCE_MEMBER_IDENTITY" + ) + + +def _preflight(paths, serialnos=None): + # Directory/expanded-byte gates precede member opening. Streaming lexical + # gates then precede preparation's full projection and pandas construction. + expanded = 0 + for role, path in paths.items(): + with coverage.zipfile.ZipFile(path) as archive: + members, _prefix = coverage._members(archive, role) + expanded += sum(member.file_size for member in members) + _require(expanded <= MAX_EXPANDED_BYTES, "EXPANDED_BUDGET") + inventories = {} + if serialnos is None: + # Preserve the whole-source route and its pre-construction row ceiling. + for role, path in paths.items(): + inventories[role], _empty = coverage._inventory(path, role, frozenset()) + _require(sum(m["rows"] for m in inventories[role]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + if role == "person": + _require(sum(m["rows"] for m in inventories[role]) <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") + return inventories, None + selected = frozenset(serialnos) + inventories["household"], households = coverage._inventory(paths["household"], "household", selected) + _require(sum(m["rows"] for m in inventories["household"]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + _require(set(households) == selected, "SELECTION_UNKNOWN") + _require(all(0 <= n <= 20 for n in households.values()), "SELECTED_NP") + expected_rows = sum(households.values()) + _require(expected_rows > 0, "EMPTY_SELECTED_POPULATION") + _require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") + # _inventory bounds the actual selected row/body accumulation, independently + # of reported NP. Every source member is still streamed and hashed. + inventories["person"], roster = coverage._inventory(paths["person"], "person", selected) + _require(sum(m["rows"] for m in inventories["person"]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + counts = dict.fromkeys(selected, 0) + for serial, _line in roster: + counts[serial] += 1 + _require(counts == households, "SELECTED_ROSTER") + _require(len(roster) == expected_rows, "SELECTED_ROSTER") + return inventories, (households, roster) + + +@dataclass(frozen=True, slots=True) +class _Owned: + frame: Frame + payload: bytes + prepared: housing.PreparedACSHousingPopulation + literal: coverage.AuthenticatedACSPersonCoverage + snapshots: tuple + pins: tuple + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSNativeCoverage: + """Process-owned immutable evidence; the live Frame is checked on every borrow.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(_owned(self).payload) + + @property + def frame(self): + verify_acs_native_coverage(self) + return _owned(self).frame + + @property + def coverage_table(self): + return _owned(self).literal.table + + @property + def source_coverage(self): + """The unchanged predecessor still has an ungranted native_binding.""" + return _owned(self).literal + + +def _owned(issuance): + _require(type(issuance) is AuthenticatedACSNativeCoverage, "ISSUANCE_TYPE") + owned = _ISSUED.get(issuance) + _require( + owned is not None + and type(issuance.payload) is bytes + and issuance.payload == owned.payload, + "ISSUANCE_NOT_OWNED", + ) + return owned + + +def _verify_sources(snapshots, pins): + for paths in snapshots: + for role, _name, digest, size in pins: + path = paths[role] + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + + +def _frame_sha256(frame): + # Supplement the unchanged owner's typed-cell identity with pandas axis + # descriptors/attrs and every entity's resolved original DESIGN weights. + details = {"storage": housing.frame_content_sha256(frame), "entities": {}} + for entity in frame.entities: + table = frame.table(entity) + weights = frame.resolve_weights(entity) + details["entities"][entity] = { + "index_type": type(table.index).__name__, + "index_dtype": repr(table.index.dtype), + "column_axis_name": table.columns.name, + "column_axis_dtype": repr(table.columns.dtype), + "dtypes": [repr(dtype) for dtype in table.dtypes], + "attrs": table.attrs, + "allows_duplicate_labels": table.flags.allows_duplicate_labels, + "weight_kind": weights.kind.value, + "weight_dtype": str(weights.values.dtype), + "weight_shape": list(weights.values.shape), + "weight_sha256": coverage._sha(weights.values.tobytes()), + } + details["strata_attrs"] = frame.strata.attrs + details["strata_index_type"] = type(frame.strata.index).__name__ + details["strata_allows_duplicate_labels"] = ( + frame.strata.flags.allows_duplicate_labels + ) + _require(not frame._link_tables, "UNEXPECTED_LINK_TABLES") + return coverage._sha(coverage._json(details, MAX_EVIDENCE_BYTES)) + + +def _verify_prepared_frame(prepared): + recorded = json.loads(prepared.receipt_json)["frame_sha256"] + _require( + housing.frame_content_sha256(prepared.frame) == recorded, + "PREPARATION_FRAME_CHANGED", + ) + return recorded + + +def _verify_frame(owned): + receipt = json.loads(owned.payload) + _require( + housing.frame_content_sha256(owned.frame) == receipt["frame_sha256"] + and _frame_sha256(owned.frame) == receipt["exact_frame_sha256"], + "FRAME_CHANGED", + ) + _require( + _verify_prepared_frame(owned.prepared) == receipt["frame_sha256"], + "PREPARATION_FRAME_CHANGED", + ) + + +def _verify(owned, producer): + receipt = json.loads(owned.payload) + frame, prepared, literal = owned.frame, owned.prepared, owned.literal + _require(receipt["protocol"] == PROTOCOL, "NATIVE_PROTOCOL") + _require(receipt["producer"] == producer, "PRODUCER_CHANGED") + _require( + prepared.frame is frame + and coverage._sha(prepared.receipt_json) + == receipt["preparation_receipt_sha256"] + and coverage._sha(prepared.source.receipt_json) + == receipt["housing_receipt_sha256"] + and coverage._sha(prepared.source.projection_json) + == receipt["projection_sha256"] + and coverage._sha(literal.payload) == receipt["coverage_payload_sha256"], + "EVIDENCE_CHANGED", + ) + _verify_frame(owned) + housing.verify_acs_frame_projection( + frame, json.loads(prepared.source.projection_json) + ) + consistency = coverage.verify_acs_coverage_native_consistency( + literal, frame + ).receipt + _require( + consistency["original_age"]["relation"] == "numeric_identity" + and consistency["original_age"]["matching_rows"] == frame.n("person"), + "ORIGINAL_AGE_IDENTITY", + ) + _verify_sources(owned.snapshots, owned.pins) + _verify_frame(owned) + + +def verify_acs_native_coverage(issuance, frame=None): + """Authenticate this issuance and exact live Frame, sources and producer. + + A separately built equal Frame is not this issuance. Any borrowed mutable + Frame must be verified again at consumption; retaining a borrow bypasses no + checks here but Python cannot intercept arbitrary downstream table reads. + Callers must hold exclusive access to mutable tables during verification + and consumption. Final checks detect changes during archive/producer work; + they do not make pandas tables an atomic concurrent snapshot. + """ + try: + owned = _owned(issuance) + _require(frame is None or frame is owned.frame, "FRAME_NOT_ISSUED") + producer = _producer() + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + return issuance + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_VERIFICATION_REFUSED") from None + + +def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, candidate_path=None): + """Run real closed owners with optional exact engineering household selection. + + The complete selected roster is bounded before native construction. Full + source validation and lexical projection remain. Candidate bytes, if given, + are compared only after independent real reconstruction and never decoded. + """ + try: + serialnos = housing.AcsPumsSource.snapshot_serialnos(serialnos) + # Snapshot path-like inputs once too, before producer/capture work. + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + candidate_path = None if candidate_path is None else Path(candidate_path).absolute() + coverage._json(serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES)) + pins = housing._pins() + archives = _archives(pins) + _require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, "ARCHIVE_BUDGET") + producer = _producer() + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventory, selected_roster = _preflight(paths, serialnos) + # These roots are outside the archive-only captured source directory. + roots = [ + Path(tempfile.mkdtemp(prefix=prefix, dir=snapshot_root)) + for prefix in ("acs-native-preparation-", "acs-native-coverage-") + ] + prepared = housing.prepare_acs_housing_population( + private, snapshot_root=roots[0], serialnos=serialnos + ) + _verify_prepared_frame(prepared) + _require(prepared.receipt["format"] == "microcosm.acs_housing_preparation.v2", "PREPARATION_PROTOCOL") + _require(prepared.receipt["requested_serialnos"] == (None if serialnos is None else list(serialnos)), "PREPARATION_SELECTION") + keys, _roster_sha = coverage._native_roster(prepared.frame) + if selected_roster is not None: + households, roster = selected_roster + _require( + {s: n for s, n in households.items() if n > 0} + == keys.groupby("SERIALNO").size().to_dict() + and set(roster) == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SELECTED_NATIVE_ROSTER", + ) + literal = coverage.load_authenticated_acs_person_coverage( + private, snapshot_root=roots[1], frame=prepared.frame + ) + hreceipt, creceipt = ( + json.loads(prepared.source.receipt_json), + literal.receipt, + ) + _require( + hreceipt["archives"] == creceipt["archives"] == archives, + "SOURCE_ARCHIVE_IDENTITY", + ) + _members_equal(hreceipt["members"], creceipt["members"]) + _require(creceipt["members"] == inventory, "SOURCE_MEMBER_IDENTITY") + relation = literal.native_binding.receipt["original_age"] + _require( + relation["relation"] == "numeric_identity", "ORIGINAL_AGE_IDENTITY" + ) + snapshots = [ + { + role: Path(source_dir).absolute() / name + for role, name, _d, _s in pins + }, + paths, + ] + for root in roots: + children = tuple(root.iterdir()) + _require(len(children) == 1 and children[0].is_dir(), "CAPTURE_ROSTER") + snapshots.append( + {role: children[0] / name for role, name, _d, _s in pins} + ) + receipt = { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "vintage": 2024, + "selection": { + "kind": "all" if serialnos is None else "engineering_exact_keys", + "requested_serialnos": serialnos, + "complete_selected_roster": True, + "person_rows": prepared.frame.n("person"), + "raw_person_keys_sha256": coverage._sha(coverage._json(sorted(zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True)), coverage.MAX_BODY_BYTES)), + "vacant_serialnos": None if selected_roster is None else sorted(s for s, n in selected_roster[0].items() if n == 0), + "vacancy_status": "source_record_without_population_rows", + "before_person_accumulation_and_unit_assignment": serialnos is not None, + "full_source_inclusion_probability": None, + "representative_sample": False, + }, + "original_literal_authentication": { + "source_authenticated": True, + "fields": ["AGEP", "MIL", "ESR"], + "raw_literals_preserved": True, + "predecessor_native_binding_authenticated": False, + }, + "native_preparation_issuance": { + "population_binding_authenticated": True, + "producer_executed_by_issuer": True, + "complete_original_nonvacant_roster": serialnos is None, + "complete_selected_original_roster": True, + "source_aliases_and_vintage_checked": True, + "age_relationship": "literal_AGEP_to_native_AGEP_A_AGE_to_model_age_identity", + "matching_age_rows": relation["matching_rows"], + "native_roster_sha256": creceipt["native_consistency"][ + "native_roster_sha256" + ], + }, + "design_anchors": { + "authenticated": True, + "kind": "design", + "source": "WGTP; single-person PWGTP for WGTP=0 GQ placeholders", + "renormalized": False, + "original_weight_literals_projection_sha256": coverage._sha(prepared.source.projection_json), + "weight_sha256": coverage._sha( + prepared.frame.weights_for("household").values.tobytes() + ), + }, + "assumptions": { + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "graph_attachment": "requires_reviewed_typed_successor", + }, + "capture": { + "owners_share_physical_snapshot": False, + "owners_separately_capture_same_pins": True, + "issuer_preflight_capture_count": 1, + "owner_capture_count": 2, + "full_source_construction_peak_remains": True, + "selected_only_memory": False, + }, + "frame_sha256": housing.frame_content_sha256(prepared.frame), + "exact_frame_sha256": _frame_sha256(prepared.frame), + "preparation_receipt_sha256": coverage._sha(prepared.receipt_json), + "housing_receipt_sha256": coverage._sha(prepared.source.receipt_json), + "projection_sha256": coverage._sha(prepared.source.projection_json), + "coverage_payload_sha256": coverage._sha(literal.payload), + } + payload = coverage._json(receipt, MAX_EVIDENCE_BYTES) + owned = _Owned( + prepared.frame, payload, prepared, literal, tuple(snapshots), pins + ) + if candidate_path is not None: + target = private / "candidate.native-coverage" + digest = housing._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + digest == coverage._sha(payload) + and housing._persisted_sha(target, len(payload)) == digest, + "CANDIDATE_MISMATCH", + ) + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + # Capture's exit authority check must complete before minting the token. + _verify_frame(owned) + result = AuthenticatedACSNativeCoverage(payload, _token=_TOKEN) + _ISSUED[result] = owned + return result + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_ISSUANCE_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py new file mode 100644 index 000000000..fe0346500 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py @@ -0,0 +1,667 @@ +"""Closed ACS coverage source issuance; native consistency grants no identity. + +The existing prepared ACS container is publicly constructible. Until a closed +prepared/native authority exists, even a consistent Frame receives only an +UngrantedACSNativeBinding. Nothing here attaches columns or prepares a population. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import io +import json +import platform +import re +import stat +import sys +import sysconfig +import zipfile +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import BuiltinFunctionType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as custody +from . import acs_person_coverage_columns as literal +from .acs_pums import AcsPumsSource +from .source_csv_builtin import csv_reader_bound + +PROTOCOL = "microcosm.acs-person-coverage-authentication.v1" +MAX_HEADER_BYTES = 1024**2 +MAX_BODY_BYTES = 64 * 1024**2 +MAX_RECORD_BYTES = 400_000 +MAX_CSV_HEADER_BYTES = 64 * 1024 +MAX_TOKEN_BYTES = 64 * 1024 +_MAGIC = b"ACS-COVERAGE/1\n" +_TOKEN = object() +_CSV_READER = _csv.reader +_COLUMNS = (*literal.READ_COLUMNS, "MIL_state", "ESR_state") +_IMPLEMENTATION_FILES = ( + "acs_person_coverage_authentication.py", + "acs_person_coverage_columns.py", + "acs_housing_universe_source.py", + "acs_sources.py", + "acs_pums.py", + "source_csv_builtin.py", + "acs_2024_1yr_sources.json", +) + + +class ACSCoverageAuthenticationError(ValueError): + """Static refusal code, without source paths, keys, tokens or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSCoverageAuthenticationError(code) + + +def _sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def _json(value, cap=MAX_HEADER_BYTES): + # ASCII JSON escapes preserve CR/LF/HT and distinguish null from "". Values + # originate in byte-bounded records or the fixed, bounded header inventory. + # Count exact escaped bytes before an encoder can allocate a whole token. + count = 0 + + def charge(size): + nonlocal count + count += size + _require(count <= cap, "CANONICAL_SIZE") + + def visit(item): + if isinstance(item, str): + charge(2) + for char in item: + code = ord(char) + charge( + 2 + if char in '\\"\b\f\n\r\t' + else 1 + if 32 <= code <= 126 + else 6 + if code <= 0xFFFF + else 12 + ) + elif item is None: + charge(4) + elif type(item) is bool: + charge(4 if item else 5) + elif type(item) is int: + charge(len(str(item))) + elif isinstance(item, (list, tuple)): + charge(2 + max(0, len(item) - 1)) + for part in item: + visit(part) + elif type(item) is dict: + charge(2 + max(0, len(item) - 1) + len(item)) + for key, part in item.items(): + _require(type(key) is str, "CANONICAL_KEY") + visit(key) + visit(part) + else: + _require(False, "CANONICAL_TYPE") + + visit(value) + parts, count = [], 0 + for part in json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).iterencode(value): + count += len(part) + _require(count <= cap, "CANONICAL_SIZE") + parts.append(part.encode("ascii")) + return b"".join(parts) + + +def _producer(): + # Provider bytes alone cannot attest the callable used by either parsing + # pass. Preserve the real C binding across issuance/verification, including + # refusal when both public aliases were replaced before this owner imported. + _require( + type(_CSV_READER) is BuiltinFunctionType + and _CSV_READER.__module__ == "_csv" + and _CSV_READER.__name__ == "reader" + and _CSV_READER.__self__ is _csv + and csv.reader is _csv.reader is _CSV_READER + and literal.csv is custody.csv is csv + and csv_reader_bound(csv), + "CSV_READER_CHANGED", + ) + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + # This runtime builds _csv into libpython. Other builds ship an + # extension or link the parser into the executable itself. + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "protocol": PROTOCOL, + "files": { + name: _sha(Path(__file__).with_name(name).read_bytes()) + for name in _IMPLEMENTATION_FILES + }, + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + "csv_binary_sha256": _sha(Path(parser_binary).read_bytes()), + "stdlib": { + name: _sha(Path(module.__file__).read_bytes()) + for name, module in ( + ("csv", csv), + ("zipfile", zipfile), + ("json", json), + ) + }, + "parser": { + "reader": "_csv.reader:verified_builtin_identity", + "encoding": "utf-8-sig", + "newline": "", + "csv_field_size_limit": csv.field_size_limit(), + "literal_record_chars": literal.MAX_CSV_RECORD_CHARS, + "source_rows": literal.MAX_ROWS, + "selected_rows": literal.MAX_SELECTED_ROWS, + }, + "ceilings": { + "header": MAX_HEADER_BYTES, + "body": MAX_BODY_BYTES, + "record": MAX_RECORD_BYTES, + "csv_header": MAX_CSV_HEADER_BYTES, + "token": MAX_TOKEN_BYTES, + "member": custody._MEMBER_MAX, + "expanded_archive": custody._EXPANDED_MAX, + "member_count": custody._MEMBER_COUNT_MAX, + }, + } + + +def _records(stream): + """Fence raw logical records/tokens before UTF-8 decoding or CSV allocation. + + Quote parity only locates boundaries; the unchanged strict literal parser + subsequently owns CSV validity. CR, LF and CRLF are preserved, including + inside quotes. The conservative token ceiling includes raw CSV quoting. + """ + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + _require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + _require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + _require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +def _decode_record(raw, *, first): + # BOM is only consumed at member start, exactly as in the literal reader. + with io.TextIOWrapper( + io.BytesIO(raw), encoding="utf-8-sig" if first else "utf-8", newline="" + ) as text: + reader = literal._literal_csv_records(text) + result = next(reader, None) + _require(result is not None and next(reader, None) is None, "CSV_RECORD") + return result + + +def _members(archive, role): + # These are the closed housing owner's complete central-directory checks, + # kept local because _archive also invokes its physical-record parser. + # Editing/extracting that owner would change historical custody producers. + prefix = "psam_hus" if role == "household" else "psam_pus" + members = archive.infolist() + _require(0 < len(members) <= custody._MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require(len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER") + expanded = 0 + for member in members: + name, mode = member.filename, member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and not any(c in name for c in ("/", "\\", "\x00")), + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= custody._MEMBER_MAX, "ZIP_MEMBER_SIZE") + expanded += member.file_size + _require(expanded <= custody._EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + _require(any(n.casefold().startswith(prefix) for n in names), "ZIP_ROLE_MISSING") + return sorted(members, key=lambda m: m.filename), prefix + + +def _inventory(path, role, serialnos): + inventory, selected, rows, selected_budget = [], {}, 0, 0 + required = ("SERIALNO", "NP") if role == "household" else literal.READ_COLUMNS + with zipfile.ZipFile(path) as archive: + members, prefix = _members(archive, role) + for member in members: + applicable = member.filename.casefold().startswith(prefix) + digest, count, row_count, header, header_sha = ( + hashlib.sha256(), + 0, + 0, + None, + None, + ) + with archive.open(member) as stream: + if not applicable: + while block := stream.read(min(4096, member.file_size - count + 1)): + count += len(block) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(block) + else: + for raw in _records(stream): + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + values = _decode_record(raw, first=header is None) + if header is None: + header, header_sha = values, _sha(raw) + _require( + all(header) + and len(set(header)) == len(header) + and set(required) <= set(header), + "CSV_HEADER", + ) + positions = [header.index(c) for c in required] + continue + _require(len(values) == len(header), "CSV_ROW_WIDTH") + rows += 1 + row_count += 1 + _require(rows <= literal.MAX_ROWS, "SOURCE_ROWS") + cells = [values[p] for p in positions] + if cells[0] not in serialnos: + continue + # Upper bound on escaped row + statuses + lineage before + # the low-level reader allocates its selected DataFrame. + selected_budget += 6 * len(raw) + 1024 + _require( + selected_budget <= MAX_BODY_BYTES, "SELECTED_BODY_BUDGET" + ) + if role == "household": + key = cells[0] + _require(re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_NP") + value = int(cells[1]) + else: + _require( + re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_SPORDER" + ) + key = (cells[0], int(cells[1])) + _require(1 <= key[1] <= 20, "SOURCE_SPORDER") + value = [member.filename, row_count] + _require(key not in selected, "SOURCE_DUPLICATE_KEY") + _require( + len(selected) < literal.MAX_SELECTED_ROWS, "SELECTED_ROWS" + ) + selected[key] = value + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "bytes": count, + "sha256": digest.hexdigest(), + "compressed_bytes": member.compress_size, + "crc32": member.CRC, + "applicable": applicable, + "header_sha256": header_sha, + "rows": row_count, + } + ) + return inventory, selected + + +def _native_roster(frame): + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + p, h = frame.person, frame.table("household") + _require(0 < len(h) <= len(p) <= literal.MAX_SELECTED_ROWS, "NATIVE_ROWS") + _require(p.columns.is_unique and h.columns.is_unique, "NATIVE_COLUMNS") + _require( + { + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_person_id", + "source_row_id", + "AGEP", + "A_AGE", + "age", + } + <= set(p) + and {"household_id", "SERIALNO", "NP"} <= set(h), + "NATIVE_COLUMNS", + ) + for table, names in ( + ( + p, + ( + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_row_id", + ), + ), + (h, ("household_id", "NP")), + ): + for name in names: + _require( + pd.api.types.is_integer_dtype(table[name].dtype) + and not table[name].isna().any(), + "NATIVE_INTEGER", + ) + _require( + not p.person_id.duplicated().any() + and not h.household_id.duplicated().any() + and not p.source_row_id.duplicated().any() + and bool((p.source_row_id >= 0).all()), + "NATIVE_IDS", + ) + _require( + not h.SERIALNO.isna().any() and not h.SERIALNO.duplicated().any(), + "NATIVE_SERIALNO", + ) + _require(set(p.person_household_id) == set(h.household_id), "NATIVE_MEMBERSHIP") + household = h.set_index("household_id") + serials = p.person_household_id.map(household.SERIALNO) + _require(bool((p.source_year == 2024).all()), "NATIVE_VINTAGE") + _require( + np.array_equal(p.SPORDER, p.A_LINENO) + and np.array_equal(p.person_household_id, p.source_household_id) + and list(p.source_person_id) == [str(v) for v in p.SPORDER], + "NATIVE_ALIASES", + ) + keys = literal._keys( + pd.DataFrame({"SERIALNO": serials, "SPORDER": p.SPORDER}) + ).reset_index(drop=True) + counts = p.groupby("person_household_id", sort=False).size() + _require(np.array_equal(counts.loc[h.household_id], h.NP), "NATIVE_HOUSEHOLD_COUNT") + roster = ( + [int(pid), int(hid), serial, int(order), int(row)] + for pid, hid, serial, order, row in zip( + p.person_id, + p.person_household_id, + keys.SERIALNO, + keys.SPORDER, + p.source_row_id, + strict=True, + ) + ) + digest = hashlib.sha256() + for row in roster: + digest.update(_json(row, MAX_RECORD_BYTES - 1) + b"\n") + return keys, digest.hexdigest() + + +def _age_relation(frame, table): + matched, unresolved, mismatch = 0, 0, 0 + for raw, ag, aa, age in zip( + table.AGEP, frame.person.AGEP, frame.person.A_AGE, frame.person.age, strict=True + ): + if re.fullmatch(r"[0-9]{1,2}", raw) is None: + unresolved += 1 + elif all( + isinstance(v, (int, float, np.integer, np.floating)) + and not isinstance(v, (bool, np.bool_)) + and np.isfinite(v) + and v == int(raw) + for v in (ag, aa, age) + ): + matched += 1 + else: + mismatch += 1 + return { + "matching_rows": matched, + "unresolved_literal_rows": unresolved, + "mismatching_rows": mismatch, + "relation": "numeric_identity" + if not unresolved and not mismatch + else "unproven", + "preparation_provenance_authenticated": False, + } + + +@dataclass(frozen=True, slots=True) +class UngrantedACSNativeBinding: + """Owned consistency evidence, explicitly lacking prepared population proof.""" + + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BINDING_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedACSPersonCoverage: + """Immutable source envelope; candidate bytes cannot construct this type.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + + def _parts(self): + start = len(_MAGIC) + 4 + size = int.from_bytes(self.payload[len(_MAGIC) : start], "big") + return self.payload[start : start + size], self.payload[start + size :] + + @property + def receipt(self): + return json.loads(self._parts()[0]) + + @property + def table(self): + rows = [json.loads(row) for row in self._parts()[1].splitlines()] + result = pd.DataFrame([r[: len(_COLUMNS)] for r in rows], columns=_COLUMNS) + for column in _COLUMNS: + result[column] = result[column].astype( + "int64" if column == "SPORDER" else "string" + ) + return result + + @property + def native_binding(self): + return UngrantedACSNativeBinding( + _json( + { + **self.receipt["native_consistency"], + "source_payload_sha256": _sha(self.payload), + "population_binding_authenticated": False, + "missing_authority": "closed_prepared_acs_native_population", + }, + MAX_HEADER_BYTES, + ), + _token=_TOKEN, + ) + + +def verify_acs_coverage_native_consistency(coverage, frame): + """Check the exact live parent; never upgrade consistency to population proof.""" + try: + _require(type(coverage) is AuthenticatedACSPersonCoverage, "SOURCE_TYPE") + _require(coverage.receipt["producer"] == _producer(), "PRODUCER_CHANGED") + _keys, roster = _native_roster(frame) + evidence = coverage.native_binding.receipt + _require( + roster == evidence["native_roster_sha256"] + and custody.frame_content_sha256(frame) == evidence["frame_sha256"], + "NATIVE_FRAME_CHANGED", + ) + _require( + _age_relation(frame, coverage.table) == evidence["original_age"], + "NATIVE_AGE_CHANGED", + ) + return coverage.native_binding + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("NATIVE_CONSISTENCY_REFUSED") from None + + +def load_authenticated_acs_person_coverage( + source_dir, *, snapshot_root, frame, candidate_path=None +): + """Reconstruct from closed default archive pins, then compare optional bytes. + + ``frame`` supplies a checked selection, never source or population authority. + No manifest/pin/preparation-receipt argument and no publishing API exists. + Successful and failed private captures retain the housing owner's policy. + """ + try: + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + before = custody.frame_content_sha256(frame) + keys, roster_sha = _native_roster(frame) + producer = _producer() + with custody._capture(source_dir, snapshot_root) as (private, paths, pins): + _require( + {r for r, _n, _d, _s in pins} == {"household", "person"} + and len(pins) == 2, + "SOURCE_ROLES", + ) + members = {} + members["household"], households = _inventory( + paths["household"], "household", set(keys.SERIALNO) + ) + members["person"], lineage = _inventory( + paths["person"], "person", set(keys.SERIALNO) + ) + _require( + households == keys.groupby("SERIALNO").size().to_dict(), + "SOURCE_HOUSEHOLD_ROSTER", + ) + _require( + set(lineage) == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SOURCE_PERSON_ROSTER", + ) + table, original_receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(paths["household"], paths["person"], vintage=2024), + person_keys=keys, + chunksize=min(1000, len(keys)), + ) + body, size = [], 0 + for row in table.itertuples(index=False, name=None): + raw = ( + _json([*row, *lineage[(row[0], row[1])]], MAX_RECORD_BYTES - 1) + + b"\n" + ) + size += len(raw) + _require(size <= MAX_BODY_BYTES, "BODY_SIZE") + body.append(raw) + body = b"".join(body) + header = { + "protocol": PROTOCOL, + "encoding": "ascii-escaped-json-header-and-ndjson-v1", + "producer": producer, + "source_authenticated": True, + "vintage": 2024, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in pins + ], + "members": members, + "field_contract": literal.coverage_field_contract(), + "original_literal_receipt": original_receipt, + "columns": [*_COLUMNS, "source_member", "source_row_ordinal"], + "body_bytes": len(body), + "body_sha256": _sha(body), + "rows": len(table), + "native_consistency": { + "frame_sha256": before, + "native_roster_sha256": roster_sha, + "complete_households_checked": True, + "source_aliases_checked": True, + "original_age": _age_relation(frame, table), + }, + } + raw_header = _json(header, MAX_HEADER_BYTES) + payload = _MAGIC + len(raw_header).to_bytes(4, "big") + raw_header + body + # No candidate header, digest, DataFrame or claimed producer is parsed. + if candidate_path is not None: + target = private / "candidate.coverage" + actual = custody._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + actual == _sha(payload) + and custody._persisted_sha(target, len(payload)) == actual, + "CANDIDATE_MISMATCH", + ) + for role, _name, digest, size in pins: + _require( + custody._persisted_sha(paths[role], size) == digest, + "SNAPSHOT_CHANGED", + ) + _require(_producer() == producer, "PRODUCER_CHANGED") + _require(_native_roster(frame)[1] == roster_sha, "NATIVE_FRAME_CHANGED") + _require( + custody.frame_content_sha256(frame) == before, "NATIVE_FRAME_CHANGED" + ) + # _capture's authority-stability check must complete before issuance. + return AuthenticatedACSPersonCoverage(payload, _token=_TOKEN) + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("SOURCE_RECONSTRUCTION_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py new file mode 100644 index 000000000..53b692dad --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py @@ -0,0 +1,295 @@ +"""Additive, literal ACS person coverage fields for an exact native roster. + +This reader preserves MIL and ESR separately, including their different age +universes. It performs no population-domain allocation and authenticates no +source file. A source owner must bind the archive and requested native keys +before these observations may support a genuine graph. The existing ACS +loader, its columns and its implementation identity are unchanged. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import re +from pathlib import Path +from zipfile import ZipFile + +import pandas as pd + +from .acs_pums import AcsPumsSource +from .source_csv_builtin import capture_csv_reader + +PROTOCOL = "microcosm.acs-person-coverage-columns.v1" +DICTIONARY_URL = ( + "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/" + "PUMS_Data_Dictionary_2024.pdf" +) +DICTIONARY_SHA256 = "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff" +KEYS = ("SERIALNO", "SPORDER") +READ_COLUMNS = (*KEYS, "AGEP", "MIL", "ESR") +MAX_ROWS = 6_000_000 +MAX_SELECTED_ROWS = 1_000_000 +MAX_CSV_RECORD_CHARS = 100_000 +_NON_CSV_CONTROLS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + + +def coverage_field_contract() -> dict: + """A fresh source-definition document, suitable for provenance consumers.""" + return { + "protocol": PROTOCOL, + "survey": "ACS", + "survey_year": 2024, + "record_grain": "person", + "read_columns": list(READ_COLUMNS), + "dictionary": {"url": DICTIONARY_URL, "sha256": DICTIONARY_SHA256}, + "fields": { + "MIL": { + "pdf_page": 41, + "minimum_age": 17, + "blank_meaning": "outside age universe", + "codes": { + "1": "active_duty", + "2": "past_active_duty", + "3": "training_only", + "4": "never_served", + }, + }, + "ESR": { + "pdf_page": 56, + "minimum_age": 16, + "blank_meaning": "outside age universe", + "codes": { + "1": "civilian_working", + "2": "civilian_job_absent", + "3": "unemployed", + "4": "armed_forces_working", + "5": "armed_forces_job_absent", + "6": "outside_labor_force", + }, + }, + }, + "cross_survey_residence_equivalence_established": False, + } + + +def _json(value) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _keys(table: pd.DataFrame) -> pd.DataFrame: + """Canonical native identities; no row-position join or key imputation.""" + if ( + not isinstance(table, pd.DataFrame) + or not table.columns.is_unique + or not set(KEYS).issubset(table) + ): + raise ValueError("ACS coverage requires native person keys") + result = table.loc[:, list(KEYS)].copy() + if result.isna().any().any(): + raise ValueError("ACS coverage native keys are missing") + serial = result.SERIALNO + if not serial.map( + lambda x: ( + isinstance(x, str) and re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", x) is not None + ) + ).all(): + raise ValueError("ACS coverage native serial is not a 2024 HU/GQ identity") + order = result.SPORDER.astype("string") + if not order.str.fullmatch(r"[0-9]{1,2}").all(): + raise ValueError("ACS coverage native person order is not an integer") + result["SPORDER"] = order.astype("int64") + if not result.SPORDER.between(1, 20).all(): + raise ValueError("ACS coverage native person order is outside 1..20") + if result.duplicated(list(KEYS)).any(): + raise ValueError("ACS coverage native person keys repeat") + return result + + +def _field_state(age: str, raw: str, *, minimum_age: int, codes: dict) -> str: + if not re.fullmatch(r"[0-9]{1,2}", age): + return "age_unresolved" + if int(age) < minimum_age: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _literal_csv_records(stream): + """Check controls before CSV parsing, bounding each complete logical record.""" + csv_reader = capture_csv_reader(csv) + if csv_reader is None: + raise ValueError("SOURCE_CSV_READER_CHANGED") + record_chars = 0 + + def lines(): + nonlocal record_chars + while line := stream.readline(MAX_CSV_RECORD_CHARS - record_chars + 1): + record_chars += len(line) + if record_chars > MAX_CSV_RECORD_CHARS: + raise ValueError("ACS coverage CSV record character bound exceeded") + # HT is literal whitespace; CR/LF delimit records or remain literal + # inside quotes. newline="" prevents universal-newline rewriting. + if _NON_CSV_CONTROLS.search(line): + raise ValueError("ACS coverage CSV contains a forbidden control") + yield line + + reader = csv_reader(lines(), strict=True) + try: + while True: + record_chars = 0 + record = next(reader, None) + if record is None: + return + yield record + except (csv.Error, UnicodeError) as exc: + raise ValueError("ACS coverage invalid literal CSV") from exc + + +def _scan_acs_person_coverage(source, consume_row): + """Exhaust the literal source once before returning its member/row counts. + + The private consumer receives only exact READ_COLUMNS strings. It cannot + signal successful early completion: all members, records and widths are + checked unless an exception refuses the scan. Source custody and requested + roster checks remain the caller's responsibility. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + rows = 0 + with ZipFile(source.person_zip) as archive: + members = sorted( + name + for name in archive.namelist() + if Path(name).name.lower().startswith("psam_pus") + and name.lower().endswith(".csv") + ) + if not members or len(members) != len(set(members)): + raise ValueError( + "ACS coverage person member roster is missing or duplicated" + ) + for name in members: + with ( + archive.open(name) as member, + io.TextIOWrapper(member, encoding="utf-8-sig", newline="") as stream, + ): + reader = _literal_csv_records(stream) + header = next(reader, []) + if len(header) != len(set(header)) or not set(READ_COLUMNS).issubset( + header + ): + raise ValueError( + "ACS coverage source columns are missing or duplicated" + ) + positions = [header.index(column) for column in READ_COLUMNS] + for record in reader: + rows += 1 + if rows > MAX_ROWS: + raise ValueError("ACS coverage source row bound exceeded") + if len(record) != len(header): + raise ValueError( + "ACS coverage CSV record width differs from header" + ) + consume_row(tuple(record[position] for position in positions)) + return members, rows + + +def read_acs_person_coverage_columns( + source: AcsPumsSource, + *, + person_keys: pd.DataFrame, + chunksize: int = 100_000, +) -> tuple[pd.DataFrame, dict]: + """Stream an additive person projection and align exact selected households. + + ``person_keys`` must include every person in each requested source + household. The read refuses missing, duplicate or extra native person keys + in those households, including differences discovered in later members. + Other households are streamed past. The separate max_households option on + the legacy source is irrelevant here: the exact native roster is authority. + + AGEP, MIL and ESR remain literal strings, including empty Census cells. + Each member is parsed once, with full record widths checked before column + or household selection. Non-CSV C0/C1 controls (including NUL) refuse; + tabs and quoted CR/LF remain literal. Records, including their physical + line endings, are bounded to MAX_CSV_RECORD_CHARS decoded characters. + State columns distinguish printed codes, out-of-universe blanks, missing + in-universe values, unexpected codes and age conflicts. They are evidence + states, not a combined military predicate or survey membership decision. + + Only invented archives have been executed in development. Genuine use + requires source authentication and a separately bounded source-read plan. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + if type(chunksize) is not int or not 0 < chunksize <= 100_000: + raise ValueError("ACS coverage chunksize must be in 1..100000") + expected = _keys(person_keys).reset_index(drop=True) + if not 0 < len(expected) <= MAX_SELECTED_ROWS: + raise ValueError("ACS coverage selected person count is outside the bound") + retained = frozenset(expected.SERIALNO) + pieces, batch, selected_rows = [], [], 0 + + def consume_row(cells): + nonlocal batch, selected_rows + if cells[0] not in retained: + return + selected_rows += 1 + if selected_rows > len(expected): + raise ValueError("ACS coverage source has extra selected-household people") + batch.append(cells) + if len(batch) == chunksize: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + batch = [] + + members, rows = _scan_acs_person_coverage(source, consume_row) + if batch: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + if not pieces: + raise ValueError("ACS coverage requested persons are missing") + observed = pd.concat(pieces, ignore_index=True) + observed_keys = _keys(observed) + observed["SPORDER"] = observed_keys.SPORDER + wanted = pd.MultiIndex.from_frame(expected) + actual = pd.MultiIndex.from_frame(observed_keys) + if set(wanted) != set(actual): + raise ValueError("ACS coverage source and requested native roster differ") + result = observed.set_index(list(KEYS)).loc[wanted].reset_index() + contract = coverage_field_contract() + for field, definition in contract["fields"].items(): + result[field + "_state"] = pd.Series( + [ + _field_state( + age, + raw, + minimum_age=definition["minimum_age"], + codes=definition["codes"], + ) + for age, raw in zip(result.AGEP, result[field], strict=True) + ], + dtype="string", + ) + receipt = { + "protocol": PROTOCOL, + "contract_sha256": hashlib.sha256(_json(contract)).hexdigest(), + "source_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "member_names": members, + "source_rows_streamed": rows, + "selected_person_rows": len(result), + "native_roster_sha256": hashlib.sha256( + _json(expected.to_dict("records")) + ).hexdigest(), + "projection_sha256": hashlib.sha256( + _json(result.to_dict("records")) + ).hexdigest(), + "legacy_max_households_applied": False, + "read_columns": list(READ_COLUMNS), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py new file mode 100644 index 000000000..44d97c96f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py @@ -0,0 +1,579 @@ +"""Closed raw ACS catalogue before native population or unit construction. + +The full housing lexical projection and raw catalogue remain in memory. Literal +persons are scanned once and records are assembled in whole-household batches; +this does not make national preparation memory-bounded. No domain is classified +here. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import weakref +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import FunctionType + +from . import acs_housing_universe_source as housing +from . import acs_native_coverage_binding as native +from . import acs_person_coverage_columns as literal +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.acs-source-catalogue.v1" +_ARCHIVE_BYTES = 8 * 1024**3 +_EXPANDED_BYTES = 16 * 1024**3 +_MEMBERS = 64 +_SOURCE_ROWS = 6_000_000 +_BATCH_PEOPLE = 100_000 +_CATALOGUE_BYTES = 2 * 1024**3 +_RECEIPT_BYTES = 1024**2 +_TOKEN = object() +_ISSUED = {} + + +class ACSSourceCatalogueError(ValueError): + """Static refusal, without source identities, paths or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSSourceCatalogueError(code) + + +def _local_bytes(): + return { + module.__name__: housing._sha(Path(module.__file__).read_bytes()) + for module in (sys.modules[__name__], domains) + } + + +def _local_runtime(): + """Capture exact loaded code, including generated raw dataclass methods.""" + result = {} + for module in (sys.modules[__name__], domains): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = value.__code__ + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = function.__code__ + return result + + +def _immutable_pins(pins): + _require( + type(pins) is tuple + and len(pins) == 2 + and all( + type(pin) is tuple + and len(pin) == 4 + and all(type(value) is str for value in pin[:3]) + and type(pin[3]) is int + for pin in pins + ), + "SOURCE_AUTHORITY_TYPE", + ) + return pins + + +def _final_seals(pins, pin_authority): + # Pure comparisons after the last producer/source I/O, including capture + # exit. Another read would reopen the same interval this seal closes. + # _pins() loads the code manifest when the private override is None. Bind + # that exact authority mode and any override here without reopening I/O. + current = housing._ARCHIVE_PINS + if current is not None: + _immutable_pins(current) + _require( + current == pin_authority and (current is None or current == pins), + "SOURCE_AUTHORITY_CHANGED", + ) + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-source-catalogue.v1", "PROTOCOL_CHANGED") + _require( + literal.KEYS == ("SERIALNO", "SPORDER") + and literal.READ_COLUMNS == ("SERIALNO", "SPORDER", "AGEP", "MIL", "ESR"), + "LITERAL_CONTRACT_CHANGED", + ) + limits = ( + _ARCHIVE_BYTES, + _EXPANDED_BYTES, + _MEMBERS, + _SOURCE_ROWS, + _BATCH_PEOPLE, + _CATALOGUE_BYTES, + _RECEIPT_BYTES, + ) + ceilings = (8 * 1024**3, 16 * 1024**3, 64, 6_000_000, 100_000, 2 * 1024**3, 1024**2) + _require( + all( + type(v) is int and 0 < v <= cap + for v, cap in zip(limits, ceilings, strict=True) + ), + "LIMITS", + ) + checked = native._producer() + compiled = {} + for module in (domains, sys.modules[__name__]): + native._live_code(module, compiled) + code = _local_bytes() + _require(code == _BYTE_AUTHORITY, "PRODUCER_CODE_CHANGED") + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "native_source_closure": checked, + "catalogue_sha256": code[__name__], + "raw_domain_types_sha256": code[domains.__name__], + "limits": list(limits), + } + + +def _preflight(paths): + """Keep full-source gates separate from native's 1M selected-row gate.""" + expanded = 0 + for role, path in paths.items(): + with native.coverage.zipfile.ZipFile(path) as archive: + members, _prefix = native.coverage._members(archive, role) + _require(len(members) <= _MEMBERS, "MEMBER_LIMIT") + expanded += sum(member.file_size for member in members) + _require(expanded <= _EXPANDED_BYTES, "EXPANDED_LIMIT") + inventories = {} + for role, path in paths.items(): + inventory, empty = native.coverage._inventory(path, role, frozenset()) + _require(not empty, "UNEXPECTED_SELECTED_ROWS") + _require( + sum(member["rows"] for member in inventory) <= _SOURCE_ROWS, + "SOURCE_ROW_LIMIT", + ) + inventories[role] = inventory + return inventories + + +def _source_checks(source_dir, paths, pins): + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + for role, name, digest, size in pins: + for path in (source_dir / name, paths[role]): + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + + +def _charge(record, digest, size): + raw = native.coverage._json(record, max(0, _CATALOGUE_BYTES - size - 1)) + b"\n" + size += len(raw) + _require(size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + digest.update(raw) + return size + + +def _collect(projection, paths): + # Validate the actual persisted full projection, independently of the + # publicly constructible returned housing capsule's mutable attributes. + housing._tables(projection) + hcols = {name: i for i, name in enumerate(projection["household_columns"])} + pcols = {name: i for i, name in enumerate(projection["person_columns"])} + by_household = {} + positions = {} + people = [None] * len(projection["persons"]) + for index, row in enumerate(projection["persons"]): + serial, raw_order = row[pcols["SERIALNO"]], row[pcols["SPORDER"]] + key = (serial, int(raw_order)) + _require(key not in positions, "DUPLICATE_PERSON") + positions[key] = index + by_household.setdefault(serial, []).append(index) + household_keys, started_households = set(), set() + prospective_size = 0 + + def reserve(value, punctuation): + # An empty household's [] gains exactly the encoded person lengths and + # one comma between people. Charge before retaining literal fields. + nonlocal prospective_size + raw = native.coverage._json( + value, max(0, _CATALOGUE_BYTES - prospective_size - punctuation) + ) + prospective_size += len(raw) + punctuation + _require(prospective_size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + + def household_record(row, members): + return ( + row[hcols["SERIALNO"]], + row[hcols["TYPEHUGQ"]], + row[hcols["NP"]], + row[hcols["WGTP"]], + row[hcols["source_member"]], + row[hcols["source_row_ordinal"]], + members, + ) + + for row in projection["households"]: + serial, count = row[hcols["SERIALNO"]], int(row[hcols["NP"]]) + _require(serial not in household_keys, "DUPLICATE_HOUSEHOLD") + household_keys.add(serial) + _require(count <= _BATCH_PEOPLE, "HOUSEHOLD_EXCEEDS_BATCH") + reserve(household_record(row, ()), 1) + _require(set(by_household) <= household_keys, "GLOBAL_PERSON_KEYS") + contract = literal.coverage_field_contract() + + def consume_row(cells): + serial, order, age, mil, esr = cells + _require( + all(type(v) is str for v in cells) + and literal.re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", serial) is not None + and literal.re.fullmatch(r"[0-9]{1,2}", order) is not None + and 1 <= int(order) <= 20, + "LITERAL_PERSON_KEY", + ) + key = (serial, int(order)) + _require(key in positions, "UNEXPECTED_LITERAL_PERSON") + index = positions[key] + _require(people[index] is None, "DUPLICATE_LITERAL_PERSON") + states = { + field: literal._field_state( + age, + raw, + minimum_age=contract["fields"][field]["minimum_age"], + codes=contract["fields"][field]["codes"], + ) + for field, raw in (("MIL", mil), ("ESR", esr)) + } + _require(all(type(v) is str for v in states.values()), "LITERAL_TYPES") + original = projection["persons"][index] + person = ( + original[pcols["SPORDER"]], + age, + esr, + states["ESR"], + mil, + states["MIL"], + original[pcols["PWGTP"]], + original[pcols["source_member"]], + original[pcols["source_row_ordinal"]], + ) + reserve(person, int(serial in started_households)) + people[index] = person + started_households.add(serial) + + if positions: + _members, rows = literal._scan_acs_person_coverage( + housing.AcsPumsSource(paths["household"], paths["person"], vintage=2024), + consume_row, + ) + _require(rows == len(positions), "GLOBAL_PERSON_KEYS") + _require(all(person is not None for person in people), "GLOBAL_PERSON_KEYS") + records, vacancies, batch = [], [], [] + digest, size, batches, batch_people = hashlib.sha256(), 0, 0, 0 + + def consume(): + nonlocal size, batches + count = sum(len(by_household.get(row[hcols["SERIALNO"]], ())) for row in batch) + if count: + # Preserve the prior selected-reader ceiling on each assembly + # batch, even though no selected DataFrame is allocated here. + _require( + count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS), + "BATCH_PERSON_LIMIT", + ) + batches += 1 + for row in batch: + serial = row[hcols["SERIALNO"]] + members = tuple(people[index] for index in by_household.get(serial, ())) + record = household_record(row, members) + _require(len(members) == int(record[2]), "HOUSEHOLD_COMPLETENESS") + size = _charge(record, digest, size) + ( + vacancies if int(record[1]) == 1 and int(record[2]) == 0 else records + ).append(record) + + for row in projection["households"]: + count = int(row[hcols["NP"]]) + if batch and batch_people + count > _BATCH_PEOPLE: + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += count + if batch: + consume() + _require(size == prospective_size, "CATALOGUE_BYTE_ACCOUNTING") + return ( + tuple(records), + tuple(vacancies), + { + "households": len(household_keys), + "people": len(positions), + "occupied_hu": sum(int(r[1]) == 1 for r in records), + "institutional_gq": sum(int(r[1]) == 2 for r in records), + "noninstitutional_gq": sum(int(r[1]) == 3 for r in records), + "vacancies": len(vacancies), + "literal_batches": batches, + "canonical_record_bytes": size, + "canonical_record_sha256": digest.hexdigest(), + }, + ) + + +def _household(record): + serial, kind, count, weight, _member, _ordinal, people = record + key = domains.HouseholdKey(domains.Source.ACS, 2024, 2024, serial) + return domains.AcsHousehold( + key, + kind, + count, + weight, + tuple( + domains.AcsPerson(order, age, esr, esr_state, mil, mil_state, pwgtp, key) + for order, age, esr, esr_state, mil, mil_state, pwgtp, _pmember, _pordinal in people + ), + ) + + +@dataclass(frozen=True, slots=True) +class _Owned: + receipt: bytes + records: tuple + vacancies: tuple + source_dir: Path + paths: tuple + pins: tuple + pin_authority: tuple | None + producer: bytes + projection_path: Path + projection_bytes: int + projection_sha256: str + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSSourceCatalogue: + """Process-issued compact receipt; raw views are independent frozen values.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def validate(self): + return verify_acs_source_catalogue(self) + + def to_bytes(self): + return _checked(self).receipt + + @property + def receipt(self): + return json.loads(self.to_bytes()) + + @property + def households(self): + return tuple(_household(record) for record in _checked(self).records) + + @property + def exclusion_ledger(self): + return tuple(_household(record) for record in _checked(self).vacancies) + + @property + def lineage(self): + owned = _checked(self) + return { + r[0]: { + "household": (r[4], r[5]), + "persons": tuple((p[0], p[7], p[8]) for p in r[6]), + } + for group in (owned.records, owned.vacancies) + for r in group + } + + +def _lookup(value): + _require(type(value) is AuthenticatedACSSourceCatalogue, "ISSUANCE_TYPE") + entry = _ISSUED.get(id(value)) + _require(entry is not None and entry[0]() is value, "ISSUANCE_NOT_OWNED") + owned = entry[1] + _require( + type(value.payload) is bytes and value.payload == owned.receipt, + "ISSUANCE_CHANGED", + ) + return owned + + +def _checked(value): + try: + owned = _lookup(value) + before = native.coverage._json(_producer(), _RECEIPT_BYTES) + _require(before == owned.producer, "PRODUCER_CHANGED") + _source_checks(owned.source_dir, dict(owned.paths), owned.pins) + _require( + housing._persisted_sha(owned.projection_path, owned.projection_bytes) + == owned.projection_sha256, + "PROJECTION_CHANGED", + ) + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == before, + "PRODUCER_CHANGED", + ) + _final_seals(owned.pins, owned.pin_authority) + _require(_lookup(value) is owned, "ISSUANCE_CHANGED") + return owned + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_VERIFICATION_REFUSED") from None + + +def verify_acs_source_catalogue(value): + _checked(value) + return value + + +def issue_acs_source_catalogue(source_dir, *, snapshot_root, candidate=None): + """Execute closed real owners; candidate bytes never supply source authority.""" + try: + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _RECEIPT_BYTES), + "CANDIDATE_TYPE_SIZE", + ) + pin_authority = housing._ARCHIVE_PINS + if pin_authority is not None: + _immutable_pins(pin_authority) + pins = _immutable_pins(housing._pins()) + archives = native._archives(pins) + _require(sum(p[3] for p in pins) <= _ARCHIVE_BYTES, "ARCHIVE_LIMIT") + producer = _producer() + producer_bytes = native.coverage._json(producer, _RECEIPT_BYTES) + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventories = _preflight(paths) + source = housing._reconstruct( + private, paths, pins, None, housing._implementation() + ) + source_receipt = json.loads(source.receipt_json) + native._members_equal(source_receipt["members"], inventories) + projection_path = private / "full-projection.json" + with projection_path.open("rb") as stream: + projection_raw = stream.read(housing.ACS_HU_SOURCE_MAX_BYTES + 1) + _require( + len(projection_raw) <= housing.ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_SIZE", + ) + projection_sha256 = housing._sha(projection_raw) + _require( + projection_sha256 == source_receipt["full_projection_sha256"], + "PROJECTION_CHANGED", + ) + projection_bytes = len(projection_raw) + projection = json.loads(projection_raw) + del projection_raw + records, vacancies, counts = _collect(projection, paths) + _require( + counts["households"] == sum(m["rows"] for m in inventories["household"]) + and counts["people"] == sum(m["rows"] for m in inventories["person"]), + "GLOBAL_SOURCE_COMPLETENESS", + ) + receipt = native.coverage._json( + { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "members": inventories, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "domain_assignment_authenticated": False, + "selection_performed": False, + "source_year": 2024, + "survey_year": 2024, + "scope": "all_occupied_hu_and_both_gq_with_separate_vacancies", + "field_contract": literal.coverage_field_contract(), + "complete_original_membership": True, + "original_anchors_preserved": True, + "housing_projection_sha256": projection_sha256, + "counts": counts, + "canonical_record_schema": [ + "SERIALNO", + "TYPEHUGQ", + "NP", + "WGTP", + "household_member", + "household_row", + [ + "SPORDER", + "AGEP", + "ESR", + "ESR_state", + "MIL", + "MIL_state", + "PWGTP", + "person_member", + "person_row", + ], + ], + "canonical_record_order": "housing_source_member_then_row_ordinal", + "execution": { + "native_frame_constructed": False, + "unit_assignment_executed": False, + "full_housing_projection_retained": True, + "full_person_scan_per_literal_batch": False, + "literal_full_scans": int(bool(counts["people"])), + "national_efficiency_claimed": False, + }, + }, + _RECEIPT_BYTES, + ) + if candidate is not None: + _require(candidate == receipt, "CANDIDATE_MISMATCH") + _source_checks(source_dir, paths, pins) + _require( + housing._persisted_sha(projection_path, projection_bytes) + == projection_sha256, + "PROJECTION_CHANGED", + ) + owned = _Owned( + receipt, + records, + vacancies, + source_dir, + tuple(paths.items()), + pins, + pin_authority, + producer_bytes, + projection_path, + projection_bytes, + projection_sha256, + ) + # Capture exit can read the default source manifest. Recheck producer + # bytes/live code after that I/O, then finish with pure final seals. + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == producer_bytes, + "PRODUCER_CHANGED", + ) + _final_seals(pins, pin_authority) + result = AuthenticatedACSSourceCatalogue(receipt, _token=_TOKEN) + identity = id(result) + + def cleanup(reference): + entry = _ISSUED.get(identity) + if entry is not None and entry[0] is reference: + del _ISSUED[identity] + + _ISSUED[identity] = (weakref.ref(result, cleanup), owned) + return result + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_ISSUANCE_REFUSED") from None + + +# Authority begins at this import, rather than accepting replacement source and +# matching live code as a newly approved implementation on first issuance. +_BYTE_AUTHORITY = _local_bytes() +_RUNTIME_AUTHORITY = _local_runtime() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py new file mode 100644 index 000000000..7cad3394d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -0,0 +1,652 @@ +"""Issue selected 2024 ASEC observations with original household DESIGN anchors. + +The closed parent and person-coverage owners still authenticate all three source +cohorts. Only the selected 2024 observations become this new population. Exact +key selection is an engineering operation with no inclusion-probability claim. +No pooling coefficient, age adjustment, coverage classification or allocation is +performed here. Source evidence stays outside the selected Frame's metadata. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import sys +import weakref +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +import numpy as np +import pandas as pd + +from microcosm.build.cd_benchmark import origin +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import bundle as frame_owner +from microcosm.frame import weights as weight_owner + +from . import asec_coverage_authentication as coverage_owner +from . import asec_current_money_source as parent_owner +from . import asec_household_coverage_fields as field_owner +from . import asec_original_household_weights as anchor_owner +from . import asec_person_income_source as restoration +from . import graph_context + +ARTIFACT_KIND = "microcosm.us.asec-2024-native-population.v1" +_MAX_HOUSEHOLDS = 100_000 +_MAX_PERSONS = 600_000 +_MAX_EVIDENCE_BYTES = 128 * 1024**2 +_MAX_CHECKPOINT_BYTES = 8 * 1024**3 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecNativePopulationError(ValueError): + """Static refusal codes; no source paths, native keys or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecNativePopulationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + result = bytearray() + for part in json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ).iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_EVIDENCE_BYTES, "EVIDENCE_SIZE") + result.extend(part) + return bytes(result) + + +def _modules(): + return ( + sys.modules[__name__], + parent_owner, + restoration, + coverage_owner, + anchor_owner, + field_owner, + graph_context, + origin, + frame_owner, + weight_owner, + ) + + +def _implementation(): + runtime = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} + _require(runtime == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()}, + "parent": parent_owner._verification_identity(), + "restoration": restoration._implementation(), + "coverage": coverage_owner._implementation(), + "anchors": anchor_owner._implementation(), + "household_fields": field_owner._producer(), + "limits": [ + _MAX_HOUSEHOLDS, + _MAX_PERSONS, + _MAX_EVIDENCE_BYTES, + _MAX_CHECKPOINT_BYTES, + ], + "periods": [2024, 2025, 2024], + } + + +def _path(value): + _require(type(value) is str or isinstance(value, Path), "SOURCE_PATH") + return Path(value).absolute() + + +def _inputs(parent, household, restored, persons, member, selected, candidate): + # Snapshot every caller-owned lookup before fingerprinting or source I/O. + _require(isinstance(persons, Mapping) and len(persons) == 3, "PERSON_PATHS") + paths = {} + for year in persons: + _require( + type(year) is int + and year in (2022, 2023, 2024) + and year not in paths + and len(paths) < 3, + "PERSON_PATHS", + ) + paths[year] = _path(persons[year]) + _require(set(paths) == {2022, 2023, 2024} and len(persons) == 3, "PERSON_PATHS") + if selected is not None: + _require( + type(selected) is tuple and 0 < len(selected) <= _MAX_HOUSEHOLDS, + "SELECTION", + ) + for key in selected: + _require( + type(key) is tuple + and len(key) == 2 + and all(type(v) is int for v in key) + and key[0] == 2024 + and 1 <= key[1] <= 99999, + "SELECTION", + ) + _require(len(set(selected)) == len(selected), "SELECTION") + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_EVIDENCE_BYTES), + "CANDIDATE", + ) + return ( + _path(parent), + _path(household), + _path(restored), + paths, + _path(member), + selected, + candidate, + ) + + +def _file_identity(path, expected, maximum, size=None): + """Check original input bytes on every borrow, with bounded regular-file I/O.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= maximum, + "SOURCE_FILE_BOUNDS", + ) + _require(size is None or before.st_size == size, "SOURCE_FILE_SIZE") + digest, count = hashlib.sha256(), 0 + while block := os.read(descriptor, min(1024**2, before.st_size - count + 1)): + count += len(block) + _require(count <= before.st_size, "SOURCE_FILE_CHANGED") + digest.update(block) + after = os.fstat(descriptor) + keys = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == before.st_size + and digest.hexdigest() == expected + and all(getattr(before, k) == getattr(after, k) for k in keys), + "SOURCE_FILE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _frame_identity(frame): + _require( + type(frame) is Frame and frame.schema == US_SCHEMA and not frame.links, + "FRAME_SHAPE", + ) + _require( + set(frame._tables) == set(US_SCHEMA.entities) + and not frame._link_tables + and set(frame._weights) == set(frame.weighted_entities), + "FRAME_STORAGE", + ) + # The parent signature binds typed cells, ordered axes, metadata, mass log, + # schema and stored weights. Bind additional mutable pandas/ndarray state. + decorations = [] + for name in frame.entities: + table = frame.table(name) + _require(type(table) is pd.DataFrame, "FRAME_TABLE_TYPE") + decorations.append([name, table.attrs, table.flags.allows_duplicate_labels]) + decorations.append( + ["strata", frame.strata.attrs, frame.strata.flags.allows_duplicate_labels] + ) + weights = [] + for entity in frame.weighted_entities: + vector = frame.weights_for(entity) + _require( + type(vector) is Weights and type(vector.values) is np.ndarray, + "FRAME_WEIGHT_TYPE", + ) + weights.append( + [ + entity, + vector.kind.value, + str(vector.values.dtype), + list(vector.values.shape), + vector.values.flags.writeable, + _sha(vector.values.tobytes()), + ] + ) + return _sha( + _encode( + { + "base": parent_owner._frame_signature(frame), + "decorations": decorations, + "weights": weights, + } + ) + ) + + +def _roster(parent, coverage, anchors, fields, selected): + projection = coverage.table() + people = parent.frame.person + _require(len(projection) == len(people), "PARENT_ROSTER") + for column in coverage_owner._PARENT_COLUMNS: + _require( + projection[column].tolist() == people[column].tolist(), "PARENT_ROSTER" + ) + anchor_doc, field_doc = anchors.document, fields.document + _require( + anchor_doc["members"] == field_doc["members"] + and len(anchor_doc["members"]) == 1, + "HOUSEHOLD_MEMBER_BINDING", + ) + member = anchor_doc["members"][0] + source = next(s for s in coverage.receipt["sources"] if s["source_year"] == 2024) + _require( + member["income_year"] == 2024 + and member["survey_year"] == 2025 + and source["survey_year"] == 2025 + and member["archive_sha256"] == source["archive_sha256"], + "COHORT_ARCHIVE_BINDING", + ) + anchor_rows = {r["native_household_id"]: r for r in anchor_doc["records"]} + field_rows = {r["native_household_id"]: r for r in field_doc["records"]} + _require( + len(anchor_rows) == len(anchor_doc["records"]) + and len(field_rows) == len(field_doc["records"]) + and anchor_rows.keys() == field_rows.keys(), + "HOUSEHOLD_ROSTER", + ) + native_to_receiving, receiving_to_native, person_keys = {}, {}, set() + counts = {} + for row in projection.itertuples(index=False): + key = (int(row.source_year), int(row.source_household_id)) + receiving = int(row.person_household_id) + _require( + native_to_receiving.setdefault(key, receiving) == receiving + and receiving_to_native.setdefault(receiving, key) == key, + "HOUSEHOLD_PARTITION", + ) + person_key = (key[0], row.PERIDNUM, int(row.A_LINENO)) + _require(person_key not in person_keys, "PERSON_ROSTER") + person_keys.add(person_key) + counts[key] = counts.get(key, 0) + 1 + all_keys = {key for key in native_to_receiving if key[0] == 2024} + _require(bool(all_keys), "EMPTY_COHORT") + for native, row in field_rows.items(): + anchor = anchor_rows[native] + for column in ( + "income_year", + "survey_year", + "member_id", + "member_sha256", + "member_row_1based", + "origin_key", + "H_SEQ", + "H_HHTYPE", + ): + _require(row[column] == anchor[column], "HOUSEHOLD_PROJECTION_BINDING") + _require( + row["income_year"] == 2024 and row["survey_year"] == 2025, + "HOUSEHOLD_COHORT", + ) + key = (2024, native) + if row["H_HHTYPE"] == "1": + _require( + key in all_keys and row["reported_person_count"] == counts[key], + "HOUSEHOLD_MEMBER_COUNT", + ) + else: + _require(key not in all_keys, "HOUSEHOLD_INTERVIEW_STATUS") + _require(all(k[1] in field_rows for k in all_keys), "MISSING_HOUSEHOLD") + chosen = all_keys if selected is None else set(selected) + _require(chosen <= all_keys and len(chosen) <= _MAX_HOUSEHOLDS, "SELECTION_UNKNOWN") + mask = np.array( + [receiving_to_native[int(h)] in chosen for h in people.person_household_id], + dtype=bool, + ) + _require(0 < int(mask.sum()) <= _MAX_PERSONS, "SELECTED_PERSON_BOUNDS") + selected_ids = set(people.loc[mask, "person_household_id"].tolist()) + ordered_ids = [ + int(h) + for h in parent.frame.table("household").household_id + if h in selected_ids + ] + ordered_keys = tuple(receiving_to_native[h] for h in ordered_ids) + _require(set(ordered_keys) == chosen, "RECEIVING_HOUSEHOLDS") + exact = anchors.original_weight_fractions(ordered_keys) + rows = [] + for receiving, key, fraction in zip(ordered_ids, ordered_keys, exact, strict=True): + row = anchor_rows[key[1]] + expected_origin = origin.origin_key( + origin.AsecHouseholdOrigin( + origin.SourceMember( + member["canonical_member_id"], member["member_sha256"] + ), + 2024, + key[1], + ) + ) + _require(row["origin_key"] == expected_origin, "ORIGIN_BINDING") + rows.append( + { + "household_id": receiving, + "native_key": list(key), + "origin_key": expected_origin, + "HSUP_WGT": row["HSUP_WGT"], + "numerator": row["weight_integer_units"], + "denominator": 100, + "fraction": [fraction.numerator, fraction.denominator], + "source": row, + "household_fields": field_rows[key[1]], + } + ) + roster = projection.loc[mask].to_dict(orient="records") + return mask, np.array([float(f) for f in exact], dtype=np.float64), rows, roster + + +def _descendant(parent, mask, anchors): + """Select observations without using legacy weight slices as source anchors.""" + frame = parent.frame + _require(frame.schema == US_SCHEMA and not frame.links, "PARENT_SCHEMA") + person = frame.person.loc[mask].copy(deep=True) + tables = {"person": person} + for entity in US_SCHEMA.group_entities: + column = US_SCHEMA.membership_column(entity) + ids = set(person[column].tolist()) + # Referenced groups cannot silently retain only part of their members. + _require(not frame.person.loc[~mask, column].isin(ids).any(), "SPLIT_GROUP") + table = frame.table(entity) + tables[entity] = table.loc[ + table[US_SCHEMA.id_column(entity)].isin(ids) + ].reset_index(drop=True) + descendant = Frame( + tables, + US_SCHEMA, + {"household": Weights(anchors, WeightKind.DESIGN)}, + frame.strata.loc[mask].copy(deep=True), + metadata={}, + mass_log=(), + ) + parent_owner._detach_frame_axes(descendant) + return descendant + + +class _State(NamedTuple): + """Tuple storage keeps the external seal immutable even to object.__setattr__.""" + + frame: Frame + parent: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + parent_identity: str + frame_identity: str + context: bytes + attached_evidence: tuple + + +def _attached_evidence(parent, coverage, anchors, fields): + """Capture immutable issued bytes; the parent authority also has identity.""" + values = ( + parent.source._validate(), + coverage._header, + coverage._body, + anchor_owner._checked_capsule_payload(anchors), + anchor_owner._checked_capsule_payload(fields), + ) + _require(all(type(value) is bytes for value in values), "ATTACHED_EVIDENCE_CHANGED") + return (parent.source, *values) + + +def _validate_state(state): + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require(_frame_identity(state.frame) == state.frame_identity, "FRAME_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + state.parent.validate() + coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + anchor_owner.verify_asec_household_weights_source(state.anchors) + field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + _file_identity(path, expected, maximum, size) + # File checks/foreign owner calls may yield. Recheck the actual receiving + # frame and producer after them, not only before final source verification. + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + _require( + _frame_identity(state.frame) == state.frame_identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "FRAME_CHANGED", + ) + # These final seal checks perform no source/resource I/O. The immutable + # tuple belongs to the external population seal, not mutable capsule state. + current = _attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsec2024NativePopulation: + """Process-issued population; every accessor verifies its full live binding.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsec2024NativePopulation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecNativePopulationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecNativePopulationError("NATIVE_BINDING_REFUSAL") from None + + def _state(self): + return self._checked()[2] + + def validate(self): + self._state() + + @property + def frame(self): + return self._state().frame + + @property + def context(self): + return self._state().context + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +def load_authenticated_asec_2024_native_population( + parent_path, + household_attachment_path, + person_income_attachment_path, + *, + person_member_paths, + household_member_path, + selected_households=None, + candidate=None, +): + """Reconstruct from closed source paths; candidate bytes never grant authority. + + ``selected_households`` is a nonempty unique tuple of ``(2024, H_SEQ)`` + integer keys, or None for the complete 2024 source cohort. This is not a + probability sample. The authenticated parent/coverage still read all cohorts. + """ + try: + ( + parent_path, + household_path, + restored_path, + persons, + member, + selection, + candidate, + ) = _inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + selected_households, + candidate, + ) + producer = _encode(_implementation()) + parent = restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = _frame_identity(parent.frame) + coverage = coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = anchor_owner.load_authenticated_asec_household_weights({2024: member}) + fields = field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + mask, weights, rows, roster = _roster( + parent, coverage, anchors, fields, selection + ) + frame = _descendant(parent, mask, weights) + context = graph_context.encode_us_frame_context(frame) + identity = json.loads(parent.source.identity) + source_files = [ + (parent_path, identity["parent_sha256"], _MAX_CHECKPOINT_BYTES, None), + ( + household_path, + identity["attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ] + for source in coverage.receipt["sources"]: + source_files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + source_files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + state = _State( + frame, + parent, + coverage, + anchors, + fields, + tuple(source_files), + producer, + parent_identity, + _frame_identity(frame), + context, + _attached_evidence(parent, coverage, anchors, fields), + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "income_year": 2024, + "survey_year": 2025, + "analysis_year": 2024, + "selection": { + "kind": "complete_2024_cohort" + if selection is None + else "exact_engineering_keys", + "keys": [row["native_key"] for row in rows], + "inclusion_probabilities_known": False, + }, + "households": rows, + "persons": roster, + "raw_age_relation": "original_A_AGE_identity_no_adjustment", + "original_weight_conversion": "float64(Fraction(HSUP_WGT_integer,100))", + "weight_kind": "design", + "frame_sha256": state.frame_identity, + "context_sha256": _sha(context), + "producer_sha256": _sha(producer), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "population_cohorts": [2024], + "parent_identity": identity, + "frame_sha256": parent_identity, + }, + "person_coverage": coverage.receipt, + "household_weights_sha256": _sha(anchors.payload), + "household_fields_sha256": _sha(fields.payload), + "coverage_classification_performed": False, + "source_share_allocation_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsec2024NativePopulation(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecNativePopulationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecNativePopulationError("NATIVE_SOURCE_REFUSAL") from None + + +# Pins can be substituted explicitly by invented tests. Callable implementation +# changes cannot silently establish a new producer merely by rebuilding receipts. +_RUNTIME_AUTHORITY = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py new file mode 100644 index 000000000..5641051e3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py @@ -0,0 +1,754 @@ +"""Closed, source-only PRPERTYP evidence for an exact current-money parent. + +No attachment, public pin override, population preparation or publication API. +The low-level reader remains non-authoritative inside this owned envelope. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import stat +import struct +import sys +import tempfile +import weakref +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import CodeType, FunctionType + +import pandas as pd + +from . import asec_current_money_source as money +from . import asec_person_coverage_source as literal +from . import source_csv_builtin as _csv_builtin +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +ARTIFACT_KIND = "microcosm.asec_coverage_authentication.v1" +MAGIC = b"MCASCOV\x01" +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_HEADER_MAX = 262_144 +_BODY_MAX = 268_435_456 +_ROW_MAX = 1_048_576 +_TOKEN_MAX = 65_536 +_CSV_HEADER_MAX = 65_536 +_MAX_PERSONS = 600_000 +_CHUNK = 65_536 +_INT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +_STRING_COLUMNS = ("PERIDNUM", "PRPERTYP", "PRPERTYP_state") +COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", +) +_PARENT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", +) +_NUMBERS = struct.Struct("<6q") +_LENGTH = struct.Struct("= 0, "COVERAGE_BODY_BYTES") + if self.quoted: + if byte == 34: + self.quoted, self.after_quote = False, True + elif self.after_quote and byte == 34: + self.quoted, self.after_quote = True, False + elif byte in (10, 13): + if self.header: + self._end_header() + self.row = self.field = 0 + self.column = 0 + self.header = self.after_quote = False + self.start = True + self.after_cr = byte == 13 + elif byte == 44: + self.field = 0 + self.column += 1 + self.start, self.after_quote = True, False + else: + self.quoted = self.start and byte == 34 + self.start = self.after_quote = False + + +def _capture(path, destination, *, size, digest, budget): + """The student-controls exact-size, nonblocking private-capture pattern. + + Add raw record/token limits during that same copy, plus a private inode + identity retained until parsing finishes. No second coordinate parse. + """ + _require( + type(size) is int and 0 < size <= literal.MAX_MEMBER_BYTES, + "COVERAGE_MEMBER_BYTES", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_SOURCE_FILE_KIND") + _require(before.st_size == size, "COVERAGE_SOURCE_SIZE") + count, hashed, guard = 0, hashlib.sha256(), _CsvBounds(budget) + with ( + os.fdopen(descriptor, "rb", buffering=0, closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "COVERAGE_SOURCE_SIZE") + guard.feed(chunk) + hashed.update(chunk) + dst.write(chunk) + dst.flush() + captured = _identity(os.fstat(dst.fileno())) + _require( + count == size and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_SOURCE_CHANGED", + ) + _require(hashed.hexdigest() == digest, "COVERAGE_SOURCE_DIGEST") + return captured + finally: + os.close(descriptor) + + +def _parent_binding(source): + _require( + type(source) is money.AuthenticatedCurrentMoneySource, + "COVERAGE_AUTHENTICATED_PARENT", + ) + source.validate() + return { + "authority_type": "AuthenticatedCurrentMoneySource", + "source_identity_sha256": _sha(source.source.identity), + "frame_sha256": money._frame_signature(source.frame), + "scope_sha256": _sha(source.scope.identity), + } + + +def _partitions(person): + published_to_parent, parent_to_published = {}, {} + for year, native, parent in person[ + ["source_year", "source_household_id", "person_household_id"] + ].itertuples(index=False, name=None): + published = (year, native) + _require(parent > 0, "COVERAGE_HOUSEHOLD_COORDINATE") + _require( + published_to_parent.setdefault(published, parent) == parent, + "COVERAGE_HOUSEHOLD_SPLIT", + ) + _require( + parent_to_published.setdefault(parent, published) == published, + "COVERAGE_HOUSEHOLD_MERGE", + ) + return { + "relation": "bidirectional_partition_bijection", + "published_key": ["source_year", "source_household_id"], + "parent_key": "person_household_id", + "households": len(parent_to_published), + "rows": len(person), + } + + +def _utf8_size(token): + _require(type(token) is str and len(token) <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + size = sum( + 1 if ord(c) < 128 else 2 if ord(c) < 2048 else 3 if ord(c) < 65536 else 4 + for c in token + ) + _require(size <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + return size + + +def _body(table): + _require(0 < len(table) <= _MAX_PERSONS, "COVERAGE_ROWS") + result = bytearray() + for row in table[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", + ] + ].itertuples(index=False, name=None): + sizes = [_utf8_size(token) for token in row[6:]] + size = _NUMBERS.size + 3 * _LENGTH.size + sum(sizes) + _require(size <= _ROW_MAX, "COVERAGE_ROW_BYTES") + _require(len(result) + _LENGTH.size + size <= _BODY_MAX, "COVERAGE_BODY_BYTES") + result.extend(_LENGTH.pack(size)) + result.extend(_NUMBERS.pack(*row[:6])) + for token, length in zip(row[6:], sizes, strict=True): + result.extend(_LENGTH.pack(length)) + result.extend(token.encode("utf-8")) + return bytes(result) + + +def _decode_rows(body, rows): + _require(type(body) is bytes and len(body) <= _BODY_MAX, "COVERAGE_BODY_BYTES") + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "COVERAGE_ROWS") + view, offset = memoryview(body), 0 + for _ in range(rows): + _require(offset + 4 <= len(view), "COVERAGE_ROW_BYTES") + size = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + end = offset + size + _require( + _NUMBERS.size + 12 <= size <= _ROW_MAX and end <= len(view), + "COVERAGE_ROW_BYTES", + ) + values = list(_NUMBERS.unpack_from(view, offset)) + offset += _NUMBERS.size + for _ in _STRING_COLUMNS: + _require(offset + 4 <= end, "COVERAGE_TOKEN_BYTES") + length = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + _require( + length <= _TOKEN_MAX and offset + length <= end, "COVERAGE_TOKEN_BYTES" + ) + values.append(bytes(view[offset : offset + length]).decode("utf-8")) + offset += length + _require(offset == end, "COVERAGE_ROW_BYTES") + yield values + _require(offset == len(view), "COVERAGE_BODY_BYTES") + + +def _bounded_header(header): + # Count ASCII JSON bytes without first allocating escaped strings or a + # serialized header. This vocabulary is owned metadata, never row tokens. + remaining = _HEADER_MAX + + def charge(size): + nonlocal remaining + remaining -= size + _require(remaining >= 0, "COVERAGE_HEADER_BYTES") + + def visit(value, depth=0): + _require(depth <= 16, "COVERAGE_HEADER_DEPTH") + if isinstance(value, str): + charge(2) + for c in value: + charge( + 2 + if c in '\\"\b\f\n\r\t' + else 6 + if ord(c) < 32 or 127 <= ord(c) <= 65535 + else 12 + if ord(c) > 65535 + else 1 + ) + elif isinstance(value, (list, tuple, dict)): + charge(2 + max(0, len(value) - 1)) + if isinstance(value, dict): + for key, item in value.items(): + _require(type(key) is str, "COVERAGE_HEADER_TYPE") + visit(key, depth + 1) + charge(1) + visit(item, depth + 1) + else: + for item in value: + visit(item, depth + 1) + else: + _require( + value is None + or type(value) is bool + or (type(value) is int and -(2**63) <= value < 2**63), + "COVERAGE_HEADER_TYPE", + ) + charge(len(_json(value))) + + visit(header) + return _json(header) + + +@dataclass(frozen=True, eq=False, slots=True, weakref_slot=True) +class AuthenticatedAsecCoverage: + """Issued immutable bytes, with fresh receipt and table views on demand.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "COVERAGE_CONSTRUCTOR_UNAVAILABLE") + _ISSUED[self] = (_sha(self._header), _sha(self._body)) + + def validate(self): + try: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "COVERAGE_HEADER_BYTES", + ) + _require( + type(self._body) is bytes and len(self._body) <= _BODY_MAX, + "COVERAGE_BODY_BYTES", + ) + _require( + _ISSUED.get(self) == (_sha(self._header), _sha(self._body)), + "COVERAGE_ISSUED_CONTENT_CHANGED", + ) + header = json.loads(self._header) + _require( + _json(header["implementation"]) == _json(_implementation()), + "COVERAGE_IMPLEMENTATION_CHANGED", + ) + _require(0 < header["rows"] <= _MAX_PERSONS, "COVERAGE_ROWS") + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_CONTENT_REFUSAL") from None + + @property + def receipt(self): + self.validate() + return json.loads(self._header) + + @property + def content_sha256(self): + self.validate() + return _sha(self._header + self._body) + + def to_bytes(self): + self.validate() + payload = MAGIC + _LENGTH.pack(len(self._header)) + self._header + self._body + return payload + hashlib.sha256(payload).digest() + + def table(self): + self.validate() + table = pd.DataFrame( + _decode_rows(self._body, self.receipt["rows"]), columns=COLUMNS + ) + for name in _STRING_COLUMNS: + table[name] = pd.array(table[name], dtype="string") + return table + + +def verify_asec_coverage_parent(coverage, source): + """Verify the closed artifact against the exact named live receiving parent.""" + try: + _require( + type(coverage) is AuthenticatedAsecCoverage, + "COVERAGE_AUTHENTICATED_ARTIFACT", + ) + coverage.validate() + _require( + coverage.receipt["parent"] == _parent_binding(source), + "COVERAGE_PARENT_CHANGED", + ) + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_PARENT_REFUSAL") from None + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + before = _implementation() + parent = _parent_binding(source) + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(year) is int for year in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "COVERAGE_MEMBER_PATHS", + ) + _require( + tuple(p[0] for p in _MEMBER_PINS) == (2022, 2023, 2024) + and len({p[1] for p in _MEMBER_PINS}) == 3, + "COVERAGE_PIN_COHORTS", + ) + person = source.frame.person + _require(0 < len(person) <= _MAX_PERSONS, "COVERAGE_ROWS") + _require( + set(_INT_COLUMNS + ("PERIDNUM",)) <= set(person), "COVERAGE_PARENT_COLUMNS" + ) + _require( + all(str(person[name].dtype) == "int64" for name in _INT_COLUMNS), + "COVERAGE_PARENT_DTYPE", + ) + # Build only from the validated parent's owned scope/native columns; never + # from model age or a caller's DataFrame/hash/receipt. + person = person[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + ] + ].copy(deep=True) + _require( + tuple(person.person_id) == source.scope.person_ids + and tuple(person.person_household_id) == source.scope.person_household_ids + and tuple(person.source_year) == source.scope.person_years, + "COVERAGE_PARENT_COORDINATES", + ) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + roster = person[list(literal.ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require( + tuple(roster.PERIDNUM) == source.scope.person_native_keys, + "COVERAGE_PARENT_NATIVE_KEYS", + ) + membership = _partitions(person) + with tempfile.TemporaryDirectory(prefix="microcosm-asec-coverage-") as directory: + paths, identities, sources = {}, {}, [] + budget = [_BODY_MAX] + pins = {pin[0]: pin for pin in _MEMBER_PINS} + for year in (2022, 2023, 2024): + _, member, archive, digest, rows, size = pins[year] + _require( + type(rows) is int + and 0 < rows <= _MAX_PERSONS + and int((roster.source_year == year).sum()) == rows, + "COVERAGE_COHORT_ROWS", + ) + path = Path(directory) / member + identities[year] = _capture( + member_paths[year], path, size=size, digest=digest, budget=budget + ) + paths[year] = path + sources.append( + { + "source_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive, + "member_sha256": digest, + "member_bytes": size, + "rows": rows, + } + ) + projection, receipt = literal.read_asec_person_coverage_source( + paths, person_roster=roster + ) + for year, path in paths.items(): + _require( + _identity(path.stat(follow_symlinks=False)) == identities[year], + "COVERAGE_PRIVATE_SOURCE_CHANGED", + ) + for name in _INT_COLUMNS[:2]: + projection[name] = person[name].to_numpy(copy=True) + body = _body(projection) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + _require( + _json(before) == _json(_implementation()), "COVERAGE_IMPLEMENTATION_CHANGED" + ) + header = { + "artifact_kind": ARTIFACT_KIND, + "encoding": "length_prefixed_utf8_fixed_int64_v1", + "columns": COLUMNS, + "rows": len(projection), + "body_sha256": _sha(body), + "parent": parent, + "sources": sources, + "household_partitions": membership, + "source_authenticated": True, + "named_parent_binding_authenticated": True, + "archive_bytes_read": False, + "source_scope": "all_three_complete_original_person_cohorts", + "age_relation": { + "relation": "original_csv_A_AGE_equals_parent_A_AGE", + "comparison": "strict_unsigned_decimal_int64_identity", + "evidence": "authenticated_private_csv_read_in_this_reconstruction", + "compared_rows": len(projection), + "conflicts": 0, + "model_age_used": False, + }, + "coverage_status": "literal_source_fields_only", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + "domain_authority": False, + "original_design_weight_semantics_established": False, + "release_eligible": False, + "literal_reader_receipt": receipt, + "implementation": before, + } + return AuthenticatedAsecCoverage(_bounded_header(header), body, _token=_TOKEN) + + +def _compare_candidate(path, payload): + # No candidate header, token, checksum or self-declared source is decoded. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_CANDIDATE_FILE_KIND") + _require(before.st_size == len(payload), "COVERAGE_CANDIDATE_BYTES") + offset = 0 + with os.fdopen(descriptor, "rb", buffering=0, closefd=False) as handle: + while chunk := handle.read(min(_CHUNK, len(payload) - offset + 1)): + _require( + chunk == memoryview(payload)[offset : offset + len(chunk)], + "COVERAGE_CANONICAL_BYTES", + ) + offset += len(chunk) + _require( + offset == len(payload) + and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_CANDIDATE_CHANGED", + ) + finally: + os.close(descriptor) + + +def authenticate_asec_coverage(source, *, member_paths, candidate_path=None): + """Independently reconstruct before comparing optional candidate bytes.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + _compare_candidate(candidate_path, expected.to_bytes()) + verify_asec_coverage_parent(expected, source) + return expected + except AsecCoverageAuthenticationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + csv.Error, + ): + raise AsecCoverageAuthenticationError("COVERAGE_SOURCE_REFUSAL") from None + + +# The imported literal parser is code authority, not a caller-selectable +# decoder. Refuse monkeypatched parsing/status/field code even before issuance. +_LITERAL_AUTHORITY = _runtime_code(literal) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py new file mode 100644 index 000000000..e1b4c99bc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py @@ -0,0 +1,1099 @@ +"""Pure ASEC current-money decoding and price restatement. + +No resource discovery or source authentication happens here. Only the separate +resource loader reads installed code. Authenticated authority can only come from +the separate verified checkpoint loader; synthetic identities remain distinct. +""" + +import json +import struct +import weakref +from dataclasses import InitVar, dataclass +from dataclasses import fields as dataclass_fields +from enum import IntEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +RECIPE = "us_asec_current_money_ccpiu_2024_v1" +ZERO_POLICY = "frozen_fillna_zero_origin_unresolved" +RESTORED_SOURCE_KIND = "asec_v4_with_household_and_person_income_observations_v1" +ENCODED_ZERO_POLICY = "authenticated_census_csv_encoded_zero_v1" +RESTORED_FIELD_COLUMNS = MappingProxyType({"PTOTVAL": "asec_PTOTVAL"}) +RESTORED_FIELD_ZERO_POLICY = MappingProxyType({"PTOTVAL": ENCODED_ZERO_POLICY}) + + +class ZeroOrigin(IntEnum): + """Source-encoding provenance; no code establishes a respondent answer.""" + + NOT_ZERO = 0 + FROZEN_FILLNA_UNRESOLVED = 1 + AUTHENTICATED_CENSUS_ENCODED = 2 + + +FIELDS = tuple( + "ANN_VAL CAP_VAL CHSP_VAL CSP_VAL DIS_VAL1 DIS_VAL2 DIV_VAL DST_VAL1 DST_VAL1_YNG DST_VAL2 DST_VAL2_YNG ED_VAL FRSE_VAL HTOTVAL INT_VAL OI_VAL PHIP_VAL PMED_VAL PNSN_VAL POTC_VAL PTOTVAL RETCB_VAL RNT_VAL SEMP_VAL SPM_CAPHOUSESUB SPM_CHILDCAREXPNS SPM_ENGVAL SSI_VAL SS_VAL UC_VAL VET_VAL WC_VAL WSAL_VAL".split() +) +RESOURCE_PINS = ( + "09cefd4d08968cf1dc4dcdc787deef88c77b84549dfb0baa6bf9cbd9e3ce7352", + "88cec207a3c73e5de93d677acf16e8db9ddae9761c644c06da92cacf0feb3f63", + "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7", +) +MICROUNIT_VERSION = "0.1.0" +MICROUNIT_SHA256 = "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e" +RESOURCE_MAX_BYTES = 512 * 1024 +HEADER_MAX_BYTES = 64 * 1024 +MAX_PERSONS = 1_000_000 +MAX_HOUSEHOLDS = 400_000 +_STATE_TOKEN = object() +_SOURCE_TOKEN = object() +# Keep issuance outside publicly writable evidence objects. Keys use identity, +# not dataclass value equality: independently reconstructed evidence stays equal +# without lending its authority to an unissued copy. Weak references retire +# entries with their objects, and retained bytes are bounded by HEADER_MAX_BYTES. +_SOURCE_ISSUANCE: dict[int, tuple[weakref.ReferenceType, bytes]] = {} + + +class MoneyRefusalError(ValueError): + """Sanitized field/reason refusal; never includes row identifiers or values.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field if field in FIELDS else "contract" + super().__init__(f"{self.field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise MoneyRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(value: bytes, limit: int = RESOURCE_MAX_BYTES) -> dict: + _require(type(value) is bytes and 0 < len(value) <= limit, "JSON_SIZE_OR_TYPE") + + def pairs(items): + result = {} + for key, item in items: + _require(key not in result, "DUPLICATE_JSON_KEY") + result[key] = item + return result + + def constant(_): + raise MoneyRefusalError("NONFINITE_JSON") + + try: + result = json.loads(value, object_pairs_hook=pairs, parse_constant=constant) + except MoneyRefusalError: + raise + except (ValueError, RecursionError) as exc: + raise MoneyRefusalError("MALFORMED_JSON") from exc + _require(type(result) is dict, "JSON_OBJECT_REQUIRED") + return result + + +def _digest(value: str) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +@dataclass(frozen=True) +class AuthenticatedAsecSource: + """Immutable evidence issued only by the actual checkpoint verification loader.""" + + evidence: bytes = b"" + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SOURCE_TOKEN, "AUTHENTICATED_SOURCE_UNAVAILABLE") + self._validate_content() + key = id(self) + + def discard(reference): + issued = _SOURCE_ISSUANCE.get(key) + if issued is not None and issued[0] is reference: + del _SOURCE_ISSUANCE[key] + + _SOURCE_ISSUANCE[key] = (weakref.ref(self, discard), self.evidence) + + def _validate(self): + issued = _SOURCE_ISSUANCE.get(id(self)) + evidence = self.evidence + _require( + type(self) is AuthenticatedAsecSource + and issued is not None + and issued[0]() is self + and type(evidence) is bytes + and evidence == issued[1], + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + return evidence + + def _validate_content(self): + data = _parse(self.evidence, HEADER_MAX_BYTES) + restored = data.get("source_kind") == RESTORED_SOURCE_KIND + extra = ( + { + "person_income_attachment_sha256", + "field_source_columns", + "field_zero_origin_policy", + } + if restored + else set() + ) + _require( + set(data) + == extra + | { + "schema_version", + "source_kind", + "source_authentication", + "parent_sha256", + "attachment_sha256", + "cohorts", + "sidecars", + "field_roster", + "zero_origin_policy", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + }, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + data["schema_version"] == (2 if restored else 1) + and data["source_kind"] + == ( + RESTORED_SOURCE_KIND + if restored + else "asec_v4_with_household_observations_v1" + ) + and data["source_authentication"] == "checkpoint_bytes_verified" + and data["field_roster"] == list(FIELDS) + and data["zero_origin_policy"] == ZERO_POLICY, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + all( + _digest(data[name]) + for name in ( + "parent_sha256", + "attachment_sha256", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + ) + ), + "SOURCE_PIN", + ) + if restored: + _require( + _digest(data["person_income_attachment_sha256"]) + and data["field_source_columns"] == RESTORED_FIELD_COLUMNS + and data["field_zero_origin_policy"] == RESTORED_FIELD_ZERO_POLICY, + "RESTORED_SOURCE_SCHEMA", + ) + _require(self.evidence == _json(data), "AUTHENTICATED_SOURCE_SCHEMA") + + @property + def identity(self) -> bytes: + # Return the same immutable bytes checked, not a second mutable lookup. + return self._validate() + + +def _restored_source(source) -> bool: + return ( + type(source) is AuthenticatedAsecSource + and _parse(source.identity)["source_kind"] == RESTORED_SOURCE_KIND + ) + + +def _origin_code(spec, field: str) -> ZeroOrigin: + return ( + ZeroOrigin.AUTHENTICATED_CENSUS_ENCODED + if field == "PTOTVAL" and _restored_source(spec.source) + else ZeroOrigin.FROZEN_FILLNA_UNRESOLVED + ) + + +def _header_provenance(spec) -> dict: + if _restored_source(spec.source): + return { + "schema_version": 2, + "field_zero_origin_policy": dict(RESTORED_FIELD_ZERO_POLICY), + } + return {"schema_version": 1} + + +def _authentication(source) -> str: + return ( + "checkpoint_bytes_verified" + if type(source) is AuthenticatedAsecSource + else "synthetic_unverified" + ) + + +@dataclass(frozen=True) +class SyntheticAsecSource: + """Explicit invented pins, never promoted to authenticated source authority.""" + + parent_sha256: str + attachment_sha256: str + cohort_sha256: tuple[tuple[int, str], ...] + sidecar_sha256: tuple[tuple[str, str], ...] + selection_sha256: str + source_kind: str = "asec_v4_with_household_observations_v1" + field_roster: tuple[str, ...] = FIELDS + zero_origin_policy: str = ZERO_POLICY + + def __post_init__(self): + _require( + all( + _digest(v) + for v in ( + self.parent_sha256, + self.attachment_sha256, + self.selection_sha256, + ) + ), + "SOURCE_PIN", + ) + _require( + type(self.cohort_sha256) is tuple and len(self.cohort_sha256) == 3, + "COHORT_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and type(p[0]) is int and _digest(p[1]) + for p in self.cohort_sha256 + ), + "COHORT_PINS", + ) + _require( + tuple(p[0] for p in self.cohort_sha256) == (2022, 2023, 2024), + "SOURCE_YEAR_MAPPING", + ) + _require( + type(self.sidecar_sha256) is tuple and len(self.sidecar_sha256) == 2, + "SIDECAR_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and _digest(p[1]) + for p in self.sidecar_sha256 + ), + "SIDECAR_PINS", + ) + _require( + tuple(p[0] for p in self.sidecar_sha256) + == ("ED_VAL.archive", "ED_VAL.member"), + "SIDECAR_PINS", + ) + _require( + self.source_kind == "asec_v4_with_household_observations_v1", + "UNSUPPORTED_PARENT_KIND", + ) + _require( + type(self.field_roster) is tuple and self.field_roster == FIELDS, + "FIELD_ROSTER", + ) + _require(self.zero_origin_policy == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + + @property + def identity(self) -> bytes: + return _json( + { + **{f.name: getattr(self, f.name) for f in dataclass_fields(self)}, + "source_authentication": "synthetic_unverified", + } + ) + + +@dataclass(frozen=True) +class CurrentMoneyResources: + """Explicit immutable resource bytes and inspected-code/runtime claims. + + Pure compilation checks these claims, not installed files. Use the separate + loader for installed verification. Source authentication belongs to the + separate closed checkpoint loader. + """ + + domains: bytes + price: bytes + consumers: bytes + execution_identity: bytes + inspected_version: str + inspected_module_sha256: str + + def __post_init__(self): + _require( + all( + type(x) is bytes + for x in ( + self.domains, + self.price, + self.consumers, + self.execution_identity, + ) + ), + "RESOURCE_BYTES_REQUIRED", + ) + + +@dataclass(frozen=True) +class MoneyDomain: + name: str + entity: str + column: str + grain: str + minimum: int + maximum: int + zero_semantics: str + + +@dataclass(frozen=True) +class CurrentMoneySpec: + """Closed v1 resources and issued source evidence; no mutable user mappings.""" + + resources: CurrentMoneyResources + source: SyntheticAsecSource | AuthenticatedAsecSource + fields: tuple[MoneyDomain, ...] + factors: tuple[tuple[int, str, str], ...] + identity: bytes + + @property + def sha256(self) -> str: + return _sha(self.identity) + + +def compile_asec_current_money_spec( + resources: CurrentMoneyResources, + source_descriptor: SyntheticAsecSource | AuthenticatedAsecSource, +) -> CurrentMoneySpec: + """Compile pinned declarations; this does not change live nominal consumers.""" + _require( + type(resources) is CurrentMoneyResources + and type(source_descriptor) in (SyntheticAsecSource, AuthenticatedAsecSource), + "SYNTHETIC_SOURCE_REQUIRED", + ) + if type(source_descriptor) is AuthenticatedAsecSource: + source_descriptor._validate() + else: + source_descriptor.__post_init__() + resources.__post_init__() + documents = tuple( + _parse(v) for v in (resources.domains, resources.price, resources.consumers) + ) + # Immutable v1 pins close every nested key, domain, writer, year and address. + # A changed convention requires a reviewed resource/recipe revision. + _require( + tuple( + _sha(v) for v in (resources.domains, resources.price, resources.consumers) + ) + == RESOURCE_PINS, + "RESOURCE_FINGERPRINT", + ) + _require( + resources.inspected_version == MICROUNIT_VERSION + and resources.inspected_module_sha256 == MICROUNIT_SHA256, + "MICROUNIT_CONTRACT_MISMATCH", + ) + execution = _parse(resources.execution_identity, HEADER_MAX_BYTES) + _require( + set(execution) == {"modules", "dependencies", "platform"}, "EXECUTION_IDENTITY" + ) + _require( + type(execution["modules"]) is dict + and set(execution["modules"]) + == { + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + }, + "EXECUTION_MODULES", + ) + _require( + all(_digest(v) for v in execution["modules"].values()), "EXECUTION_MODULES" + ) + _require( + type(execution["dependencies"]) is dict + and set(execution["dependencies"]) == {"python", "numpy", "pandas"}, + "EXECUTION_DEPENDENCIES", + ) + _require( + type(execution["platform"]) is dict + and set(execution["platform"]) + == {"system", "machine", "byteorder", "python_implementation"}, + "EXECUTION_PLATFORM", + ) + _require( + all( + type(v) is str and 0 < len(v) <= 128 + for group in (execution["dependencies"], execution["platform"]) + for v in group.values() + ), + "EXECUTION_IDENTITY", + ) + domains, price, _ = documents + fields = tuple( + MoneyDomain( + **{ + k: f[k] + for k in ( + "name", + "entity", + "column", + "grain", + "minimum", + "maximum", + "zero_semantics", + ) + } + ) + for f in domains["fields"] + ) + factors = tuple( + ( + y, + f"174.4/{price['cells'][str(y)]}", + struct.pack(" None: + _require(type(spec) is CurrentMoneySpec, "SPEC_TYPE") + _require( + spec == compile_asec_current_money_spec(spec.resources, spec.source), + "SPEC_BINDING", + ) + + +@dataclass(frozen=True) +class AsecMoneyScope: + """Ordered source coordinates supplied explicitly with projected tables. + + Pandas labels are representation only. IDs/native keys here belong to each + positional row, and must be projected in the same permutation as the table. + No original source authentication is inferred from this synthetic contract. + """ + + person_ids: tuple[int, ...] + household_ids: tuple[int, ...] + person_household_ids: tuple[int, ...] + person_spm_ids: tuple[int, ...] + person_years: tuple[int, ...] + household_years: tuple[int, ...] + person_native_keys: tuple[str, ...] + household_native_keys: tuple[str, ...] + + def __post_init__(self): + p, h = len(self.person_ids), len(self.household_ids) + _require(0 < p <= MAX_PERSONS and 0 < h <= MAX_HOUSEHOLDS, "SOURCE_SIZE") + for name in ( + "person_ids", + "household_ids", + "person_household_ids", + "person_spm_ids", + "person_years", + "household_years", + ): + vector = getattr(self, name) + _require( + type(vector) is tuple + and all(type(x) is int and -(2**63) <= x < 2**63 for x in vector), + "SCOPE_INTEGER_VECTOR", + ) + _require( + len(vector) == (h if name.startswith("household") else p), + "SCOPE_LENGTH", + ) + _require(len(set(self.person_ids)) == p, "DUPLICATE_PERSON_ID") + _require(len(set(self.household_ids)) == h, "DUPLICATE_HOUSEHOLD_ID") + _require( + set(self.person_household_ids) == set(self.household_ids), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + set(self.person_years) <= {2022, 2023, 2024} + and set(self.household_years) <= {2022, 2023, 2024}, + "SOURCE_YEAR_MAPPING", + ) + expected = dict(zip(self.household_ids, self.household_years, strict=True)) + _require( + all( + expected[g] == y + for g, y in zip( + self.person_household_ids, self.person_years, strict=True + ) + ), + "MIXED_HOUSEHOLD_YEAR", + ) + spm_years = {} + for spm, year in zip(self.person_spm_ids, self.person_years, strict=True): + _require(spm not in spm_years or spm_years[spm] == year, "MIXED_SPM_YEAR") + spm_years[spm] = year + for name, years, count in ( + ("person_native_keys", self.person_years, p), + ("household_native_keys", self.household_years, h), + ): + keys = getattr(self, name) + _require( + type(keys) is tuple + and len(keys) == count + and all(type(k) is str and 0 < len(k) <= 256 for k in keys), + "NATIVE_KEYS", + ) + _require( + len(set(zip(years, keys, strict=True))) == count, "DUPLICATE_NATIVE_KEY" + ) + + @property + def identity(self) -> bytes: + # Digests bound the header, while the active scope retains real coordinates. + return _json( + {f.name: _sha(_json(getattr(self, f.name))) for f in dataclass_fields(self)} + ) + + +@dataclass(frozen=True) +class AsecMoneyViews: + """Explicit raw projections plus their ordered coordinate association.""" + + person: pd.DataFrame + household: pd.DataFrame + scope: AsecMoneyScope + + +class CodebookStatus(IntEnum): + MISSING_NULL = 0 + AMOUNT_NONZERO = 1 + ZERO_NONE_OR_NIU = 2 + ZERO_DOLLARS_AS_CODED = 3 + DECLARED_NIU = 4 + + +_ZERO_STATUS = { + "none_or_niu_not_distinguishable_from_amount_alone": CodebookStatus.ZERO_NONE_OR_NIU, + "valid_zero_dollars": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "valid_zero_dollars_or_none_as_described": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "niu": CodebookStatus.DECLARED_NIU, +} +_DTYPES = frozenset( + ( + "int8", + "uint8", + "int16", + "uint16", + "int32", + "uint32", + "int64", + "uint64", + "float64", + "Int8", + "UInt8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float64", + ) +) + + +@dataclass(frozen=True) +class MoneyField: + """Immutable canonical little-endian amounts and three uint8 evidence axes.""" + + name: str + amount_bytes: bytes + status_bytes: bytes + validity_bytes: bytes + zero_origin_bytes: bytes + + @property + def amounts(self) -> np.ndarray: + return np.frombuffer(self.amount_bytes, dtype=" np.ndarray: + return np.frombuffer(self.status_bytes, dtype="u1") + + @property + def validity(self) -> np.ndarray: + return np.frombuffer(self.validity_bytes, dtype="u1") + + @property + def zero_origin(self) -> np.ndarray: + return np.frombuffer(self.zero_origin_bytes, dtype="u1") + + +def _validate_field( + field: MoneyField, + domain: MoneyDomain, + count: int, + *, + nominal: bool, + zero_origin_code: ZeroOrigin, +) -> None: + _require(type(field) is MoneyField and field.name == domain.name, "FIELD_IDENTITY") + buffers = ( + field.amount_bytes, + field.status_bytes, + field.validity_bytes, + field.zero_origin_bytes, + ) + _require( + all(type(v) is bytes for v in buffers) + and tuple(map(len, buffers)) == (8 * count, count, count, count), + "FIELD_BUFFER_SHAPE", + domain.name, + ) + a, s, v, z = field.amounts, field.statuses, field.validity, field.zero_origin + _require( + np.isfinite(a).all() + and np.isin(s, [0, 1, 2, 3, 4]).all() + and np.isin(v, [0, 1]).all() + and np.isin(z, [0, 1, 2]).all(), + "FIELD_ENCODING", + domain.name, + ) + _require( + np.array_equal(v == 0, s == CodebookStatus.MISSING_NULL), + "INVALID_SLOT", + domain.name, + ) + _require( + np.array_equal(a != 0, s == CodebookStatus.AMOUNT_NONZERO) + and not np.signbit(a[a == 0]).any(), + "AMOUNT_STATUS_ENCODING", + domain.name, + ) + expected_zero = _ZERO_STATUS[domain.zero_semantics] + allowed = { + CodebookStatus.MISSING_NULL, + CodebookStatus.AMOUNT_NONZERO, + expected_zero, + } + if domain.name == "ANN_VAL": + allowed.add(CodebookStatus.DECLARED_NIU) + _require(np.isin(s, list(allowed)).all(), "FIELD_STATUS", domain.name) + _require( + type(zero_origin_code) is ZeroOrigin + and zero_origin_code != ZeroOrigin.NOT_ZERO, + "ZERO_ORIGIN_POLICY", + domain.name, + ) + expected_origin = (s == expected_zero).astype("u1") * int(zero_origin_code) + _require(np.array_equal(z, expected_origin), "ZERO_ORIGIN_ENCODING", domain.name) + factor_max = 1.0 if nominal else float("174.4") / float("163.6") + lo = min(domain.minimum, domain.minimum * factor_max) + hi = max(domain.maximum, domain.maximum * factor_max) + _require(((a >= lo) & (a <= hi)).all(), "AMOUNT_DOMAIN", domain.name) + if domain.name == "ANN_VAL": + _require((a >= 0).all(), "AMOUNT_DOMAIN", domain.name) + if nominal: + _require((a == np.floor(a)).all(), "FRACTIONAL_SOURCE", domain.name) + + +@dataclass(frozen=True) +class MoneyBindings: + """Expected replay identity, including the actual raw input fingerprint.""" + + spec: CurrentMoneySpec + header: bytes + + def __post_init__(self): + _spec(self.spec) + data = _parse(self.header, HEADER_MAX_BYTES) + keys = { + "schema_version", + "recipe", + "country", + "semantic", + "target_year", + "source_authentication", + "state", + "spec_sha256", + "scope_sha256", + "input_sha256", + "person_rows", + "household_rows", + "fields", + "dtype", + "evidence_dtype", + "person_year_sha256", + "household_year_sha256", + } + provenance = _header_provenance(self.spec) + _require( + set(data) == keys | set(provenance) and self.header == _json(data), + "HEADER_SCHEMA", + ) + fixed = { + **provenance, + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(self.spec.source), + "state": "target_current", + "spec_sha256": self.spec.sha256, + "fields": list(FIELDS), + "dtype": " MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class RestatedAsecMoney: + bindings: MoneyBindings + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATE_TOKEN, "STATE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header(self) -> bytes: + return self.bindings.header + + def field(self, name: str) -> MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class SyntheticReadyCurrentMoney(RestatedAsecMoney): + """Complete invented fixture amounts. This is NOT production authority.""" + + +class ReadyCurrentMoney(RestatedAsecMoney): + """Complete amounts bound to verified source bytes; not release certification.""" + + def __new__(cls, *args, **kwargs): + _require( + kwargs.get("_token") is _STATE_TOKEN, "PRODUCTION_READINESS_UNAVAILABLE" + ) + return super().__new__(cls) + + +def _field(fields: tuple[MoneyField, ...], name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return fields[FIELDS.index(name)] + + +def _validate_money(value, spec: CurrentMoneySpec, *, nominal: bool) -> None: + _spec(spec) + _require( + type(value.bindings) is MoneyBindings and value.bindings.spec == spec, + "SPEC_MISMATCH", + ) + value.bindings.__post_init__() + header = _parse(value.bindings.header, HEADER_MAX_BYTES) + _require( + type(value.fields) is tuple and len(value.fields) == len(FIELDS), "FIELD_ROSTER" + ) + for domain, field in zip(spec.fields, value.fields, strict=True): + _validate_field( + field, + domain, + header[domain.entity + "_rows"], + nominal=nominal, + zero_origin_code=_origin_code(spec, domain.name), + ) + + +def _index_identity(index: pd.Index) -> bytes: + # An intentionally narrow, honest representation contract; duplicate/nondefault + # signed-int64 labels are supported positionally. No label-based assignment. + _require( + type(index) in (pd.Index, pd.RangeIndex) and str(index.dtype) == "int64", + "UNSUPPORTED_INDEX", + ) + _require(index.name is None or type(index.name) is str, "UNSUPPORTED_INDEX_NAME") + descriptor = { + "type": type(index).__name__, + "name": index.name, + "values_sha256": _sha(index.to_numpy(dtype=" DecodedAsecMoney: + """Decode explicit raw columns positionally, preserving codebook evidence.""" + _require(type(raw_views) is AsecMoneyViews, "TYPED_STATE") + _spec(spec) + _require( + type(source_scope) is AsecMoneyScope + and type(raw_views.scope) is AsecMoneyScope, + "SCOPE_TYPE", + ) + source_scope.__post_init__() + _require(raw_views.scope == source_scope, "SCOPE_MISMATCH") + p, h = len(source_scope.person_ids), len(source_scope.household_ids) + for table, count in ((raw_views.person, p), (raw_views.household, h)): + _require( + type(table) is pd.DataFrame + and len(table) == count + and table.columns.is_unique, + "VIEW_SHAPE", + ) + scope_identity = _json( + { + "coordinates": _parse(source_scope.identity), + "person_index": _parse(_index_identity(raw_views.person.index)), + "household_index": _parse(_index_identity(raw_views.household.index)), + } + ) + result = [] + inputs = [] + # Sorting native group IDs is only a temporary positional validation index. + spm = np.asarray(source_scope.person_spm_ids, dtype="= 10001) & (values <= 99999) & valid).any(), + "DISPUTED_CODEBOOK_RANGE", + domain.name, + ) + _require( + not (((values < domain.minimum) | (values > domain.maximum)) & valid).any(), + "OUTSIDE_CODEBOOK_RANGE", + domain.name, + ) + # Bind received numeric bytes (including -0) and validity before normalization. + received = values.copy() + received[~valid] = 0.0 + inputs.append( + { + "field": domain.name, + "entity": domain.entity, + "column": domain.column, + "dtype": str(series.dtype), + "values_sha256": _sha(received.tobytes()), + "validity_sha256": _sha(valid.astype("u1").tobytes()), + } + ) + status = np.full(len(values), CodebookStatus.AMOUNT_NONZERO, dtype="u1") + zeros = valid & (values == 0) + status[zeros] = _ZERO_STATUS[domain.zero_semantics] + if domain.name == "ANN_VAL": + niu = valid & (values == -1) + status[niu] = CodebookStatus.DECLARED_NIU + values[niu] = 0.0 + status[~valid] = CodebookStatus.MISSING_NULL + values[~valid | (values == 0)] = 0.0 + field = MoneyField( + domain.name, + values.tobytes(), + status.tobytes(), + valid.astype("u1").tobytes(), + (zeros.astype("u1") * int(_origin_code(spec, domain.name))).tobytes(), + ) + _validate_field( + field, + domain, + len(table), + nominal=True, + zero_origin_code=_origin_code(spec, domain.name), + ) + if domain.grain == "spm_unit_repeated_on_person": + for vector in (values, status, valid, zeros): + sorted_values = vector[order] + _require( + not ((sorted_values[1:] != sorted_values[:-1]) & same).any(), + "INCONSISTENT_SPM_AMOUNT", + domain.name, + ) + result.append(field) + header = _json( + { + **_header_provenance(spec), + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(spec.source), + "state": "target_current", + "spec_sha256": spec.sha256, + "scope_sha256": _sha(scope_identity), + "input_sha256": _sha(_json(inputs)), + "person_year_sha256": _sha( + np.asarray(source_scope.person_years, dtype=" RestatedAsecMoney: + """Restate once; target-year bytes and normalized zero bypass arithmetic.""" + _require(type(decoded) is DecodedAsecMoney, "TYPED_STATE") + _validate_money(decoded, spec, nominal=True) + header = _parse(decoded.bindings.header, HEADER_MAX_BYTES) + years = {} + for entity in ("person", "household"): + data = getattr(decoded, entity + "_year_bytes") + _require( + type(data) is bytes and len(data) == header[entity + "_rows"] * 8, + "YEAR_BUFFER", + ) + _require(_sha(data) == header[entity + "_year_sha256"], "YEAR_BINDING") + vector = np.frombuffer(data, dtype=" SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Require every amount; synthetic evidence cannot request verified readiness.""" + _require(type(production) is bool, "PRODUCTION_FLAG") + verified = type(spec.source) is AuthenticatedAsecSource + _require(not production or verified, "SYNTHETIC_NOT_PRODUCTION") + _require(not verified or production, "AUTHENTICATED_READINESS_REQUIRED") + _require(type(restated) is RestatedAsecMoney, "TYPED_STATE") + _validate_money(restated, spec, nominal=False) + for field in restated.fields: + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + ready_type = ReadyCurrentMoney if verified else SyntheticReadyCurrentMoney + return ready_type(restated.bindings, restated.fields, _token=_STATE_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py new file mode 100644 index 000000000..bab0c7b02 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py @@ -0,0 +1,34 @@ +"""Pinned additive consumer status, separate from the immutable money recipe. + +The source recipe's declarations and execution identity stay byte-for-byte +unchanged. This graph record reports the routes actually wired by this slice; +it grants no source authority and changes no source address or monetary rule. +""" + +from hashlib import sha256 +from importlib import resources + +from .asec_current_money import RESOURCE_PINS, _parse + +GRAPH_CONSUMERS_RESOURCE = "asec_current_money_graph_consumers_v1.json" +GRAPH_CONSUMERS_SHA256 = ( + "67432a10d05e11dd4b97a80cd4f04ed0eb6e56c4c8e21d5dffe87c5f67e27c66" +) + + +def load_graph_current_money_consumers() -> dict: + """Verify graph status and its unchanged source-declaration parent.""" + package = resources.files(__package__) + payload = package.joinpath(GRAPH_CONSUMERS_RESOURCE).read_bytes() + if sha256(payload).hexdigest() != GRAPH_CONSUMERS_SHA256: + raise ValueError("GRAPH_CONSUMERS_FINGERPRINT") + document = _parse(payload) + parent = document["source_declarations"] + if ( + parent["resource"] != "asec_current_money_consumers_v1.json" + or parent["sha256"] != RESOURCE_PINS[2] + or sha256(package.joinpath(parent["resource"]).read_bytes()).hexdigest() + != parent["sha256"] + ): + raise ValueError("GRAPH_CONSUMERS_PARENT_FINGERPRINT") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py new file mode 100644 index 000000000..66d8e6d10 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py @@ -0,0 +1,114 @@ +"""Explicit packaged-resource and installed microunit byte verification. + +This module never imports microunit or runs units. The lower-level packaged +function permits explicit inspected-contract injection for pure synthetic tests; +it does not certify an installed distribution or authenticate source data. +""" + +import platform +import sys +from importlib import metadata, resources + +import numpy as np +import pandas as pd + +from .asec_current_money import ( + MICROUNIT_SHA256, + MICROUNIT_VERSION, + CurrentMoneyResources, + MoneyRefusalError, + _json, + _require, + _sha, +) + +RESOURCE_NAMES = ( + "asec_current_money_domains_v1.json", + "asec_current_money_price_basis_v1.json", + "asec_current_money_consumers_v1.json", +) +MODULE_NAMES = ( + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", +) + + +def packaged_current_money_resources( + *, inspected_version: str, inspected_module_sha256: str +) -> CurrentMoneyResources: + """Read package resources with explicit claims, for synthetic fixture injection. + + Use load_current_money_resources to verify the actual installed reader. + Neither resource-reading function creates authenticated source authority. + """ + package = resources.files(__package__) + identity = _json( + { + "modules": { + name: _sha(package.joinpath(name).read_bytes()) for name in MODULE_NAMES + }, + "dependencies": { + "python": platform.python_version(), + "numpy": np.__version__, + "pandas": pd.__version__, + }, + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "byteorder": sys.byteorder, + "python_implementation": platform.python_implementation(), + }, + } + ) + return CurrentMoneyResources( + *(package.joinpath(name).read_bytes() for name in RESOURCE_NAMES), + identity, + inspected_version, + inspected_module_sha256, + ) + + +def load_current_money_resources( + *, microunit_search_path: tuple[str, ...] | None = None +) -> CurrentMoneyResources: + """Verify pinned installed version/module bytes without importing microunit. + + An explicit site-packages search path supports a separately pinned local + environment. Paths are operational and never enter the normative identity. + """ + try: + if microunit_search_path is None: + distribution = metadata.distribution("microunit") + else: + _require( + type(microunit_search_path) is tuple + and all(type(p) is str for p in microunit_search_path), + "MICROUNIT_SEARCH_PATH", + ) + matches = [ + d + for d in metadata.distributions(path=microunit_search_path) + if d.metadata.get("Name", "").lower() == "microunit" + ] + _require(len(matches) == 1, "MICROUNIT_DISTRIBUTION_COUNT") + distribution = matches[0] + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + relative = "microunit/tax_unit_construction.py" + _require( + distribution.files is not None + and any(str(f) == relative for f in distribution.files), + "MICROUNIT_MODULE_INVENTORY", + ) + path = distribution.locate_file(relative) + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_MODULE_SIZE") + module = path.read_bytes() + _require( + len(module) <= 1024 * 1024 and _sha(module) == MICROUNIT_SHA256, + "MICROUNIT_MODULE_SHA256", + ) + except (metadata.PackageNotFoundError, OSError) as exc: + raise MoneyRefusalError("MICROUNIT_UNAVAILABLE") from exc + return packaged_current_money_resources( + inspected_version=distribution.version, inspected_module_sha256=_sha(module) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py new file mode 100644 index 000000000..71e4010d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py @@ -0,0 +1,549 @@ +"""Typed subset of an authenticated ASEC current-money body. + +The authenticated ``ReadyCurrentMoney`` object never leaves the preparation +process. Its canonical encoded bytes travel as a typed graph artifact, and this +module parses those bytes into a *body* that carries amounts and evidence +without carrying source readiness. Selecting whole households from a body +produces a second typed artifact whose header names its parent and the exact +selected coordinates. + +Neither type subclasses ``RestatedAsecMoney``, so every existing +``type(x) is ReadyCurrentMoney`` check refuses them: a subset can never be +minted as full-source readiness, and full readiness can only originate in the +reviewed source loaders. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass +from hashlib import sha256 + +import numpy as np + +from microcosm.graph import ArtifactType + +from ._asec_current_money_codec import _CONTENT_DOMAIN +from ._asec_current_money_codec import MAGIC as BODY_MAGIC +from ._asec_current_money_codec import PAYLOAD_MAX_BYTES as BODY_PAYLOAD_MAX_BYTES +from .asec_current_money import ( + FIELDS, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + RECIPE, + MoneyField, + MoneyRefusalError, + _digest, + _json, + _parse, + _require, + _sha, +) + +US_ASEC_CURRENT_MONEY_BODY_TYPE = ArtifactType( + "microcosm.us.asec_current_money_body", 1 +) +US_ASEC_PREPARED_RECEIPT_TYPE = ArtifactType("microcosm.us.asec_prepared_receipt", 3) +US_ASEC_SELECTION_TYPE = ArtifactType("microcosm.us.asec_household_selection", 2) +US_ASEC_SELECTED_MONEY_TYPE = ArtifactType( + "microcosm.us.asec_selected_current_money", 1 +) + +SELECTED_MAGIC = b"MCASELM1\x01" +SELECTED_ARTIFACT_KIND = "microcosm.us.asec_selected_current_money.v1" +_ENTITIES = ("person", "household") +_BODY_TOKEN = object() +_SELECTED_TOKEN = object() +_SELECTED_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "recipe", + "target_year", + "state", + "semantic", + "source_authentication", + "parent_header_sha256", + "parent_content_sha256", + "prepared_receipt_sha256", + "selection_sha256", + "person_rows", + "household_rows", + "person_identity_sha256", + "household_identity_sha256", + "fields", + "field_entities", + "dtype", + "evidence_dtype", + } +) +# 32 person fields plus one household observation, four evidence axes each. +SELECTED_PAYLOAD_MAX_BYTES = ( + len(SELECTED_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) + + +def _entities(field_entities) -> tuple[str, ...]: + """Validate a producer-declared field/entity roster against the closed fields.""" + _require( + type(field_entities) is tuple and len(field_entities) == len(FIELDS), + "FIELD_ENTITY_ROSTER", + ) + names = [] + for item in field_entities: + _require( + type(item) is tuple + and len(item) == 2 + and type(item[0]) is str + and item[1] in _ENTITIES, + "FIELD_ENTITY_ROSTER", + ) + names.append(item[0]) + _require(tuple(names) == FIELDS, "FIELD_ENTITY_ROSTER") + return tuple(item[1] for item in field_entities) + + +def _decoded_entities(data: dict) -> tuple[str, ...]: + roster = data.get("field_entities") + _require( + type(roster) is list + and all(type(item) is list and len(item) == 2 for item in roster), + "FIELD_ENTITY_ROSTER", + ) + return _entities(tuple(tuple(item) for item in roster)) + + +def _positions(values: np.ndarray, *, rows: int, reason: str) -> np.ndarray: + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("int64") + and values.ndim == 1, + reason, + ) + _require(0 < len(values) <= rows, reason) + _require((values[:-1] < values[1:]).all() if len(values) > 1 else True, reason) + _require(int(values[0]) >= 0 and int(values[-1]) < rows, reason) + return values + + +def _split( + payload: bytes, cursor: int, entities, counts: dict[str, int] +) -> tuple[MoneyField, ...]: + fields = [] + for name, entity in zip(FIELDS, entities, strict=True): + count = counts[entity] + buffers = [] + for width in (8, 1, 1, 1): + buffers.append(payload[cursor : cursor + width * count]) + cursor += width * count + fields.append(MoneyField(name, *buffers)) + return tuple(fields) + + +def _field_shapes(fields, entities, counts: dict[str, int]) -> None: + for field, entity in zip(fields, entities, strict=True): + count = counts[entity] + widths = ( + len(field.amount_bytes), + len(field.status_bytes), + len(field.validity_bytes), + len(field.zero_origin_bytes), + ) + _require(widths == (8 * count, count, count, count), "FIELD_BUFFER_SHAPE") + amounts = field.amounts + _require(np.isfinite(amounts).all(), "FIELD_ENCODING", field.name) + _require( + np.isin(field.statuses, [0, 1, 2, 3, 4]).all() + and np.isin(field.validity, [0, 1]).all() + and np.isin(field.zero_origin, [0, 1, 2]).all(), + "FIELD_ENCODING", + field.name, + ) + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + + +@dataclass(frozen=True) +class BoundCurrentMoneyBody: + """Amount/evidence buffers bound to a producer-declared body identity. + + This is emphatically not readiness: it carries no spec, no source evidence + and no authenticated authority, only bytes that a named producer already + authenticated inside its own process. + """ + + header: bytes + content_sha256: str + field_entities: tuple[tuple[str, str], ...] + fields: tuple[MoneyField, ...] + person_rows: int + household_rows: int + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _BODY_TOKEN, "BODY_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.field_entities[FIELDS.index(name)][1] + + +@dataclass(frozen=True) +class SelectedCurrentMoney: + """Whole-household subset of a body, bound to its exact selected coordinates.""" + + header: bytes + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SELECTED_TOKEN, "SELECTED_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + @property + def person_rows(self) -> int: + return self.header_data["person_rows"] + + @property + def household_rows(self) -> int: + return self.header_data["household_rows"] + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.header_data["field_entities"][FIELDS.index(name)][1] + + def amounts(self, name: str) -> np.ndarray: + """Return a writable copy of one field's selected target-year amounts.""" + return self.field(name).amounts.copy() + + +def parse_current_money_body( + payload: bytes, + *, + expected_header_sha256: str, + expected_content_sha256: str, + field_entities: tuple[tuple[str, str], ...], +) -> BoundCurrentMoneyBody: + """Parse canonical money bytes without reconstructing replayable readiness. + + The caller supplies the producer's header and content digests; both must + match exactly. Digests, not header bytes, are what a producer receipt + carries across a graph edge, so binding on them keeps the consumer honest + without re-transporting the header. This deliberately cannot mint a + ``ReadyCurrentMoney``: the authenticated replay contract stays with + ``decode_current_money``. + """ + _require( + type(payload) is bytes + and len(BODY_MAGIC) + 4 + 32 < len(payload) <= BODY_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(_digest(expected_header_sha256), "EXPECTED_HEADER_DIGEST") + _require(_digest(expected_content_sha256), "EXPECTED_CONTENT_DIGEST") + entities = _entities(field_entities) + _require(payload.startswith(BODY_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" SelectedCurrentMoney: + """Slice a body to selected positions and bind the selected coordinates.""" + _require(type(body) is BoundCurrentMoneyBody, "TYPED_BODY_REQUIRED") + _require( + all( + _digest(value) + for value in ( + prepared_receipt_sha256, + selection_sha256, + person_identity_sha256, + household_identity_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + entities = _entities(body.field_entities) + positions = { + "person": _positions( + person_positions, rows=body.person_rows, reason="PERSON_POSITIONS" + ), + "household": _positions( + household_positions, + rows=body.household_rows, + reason="HOUSEHOLD_POSITIONS", + ), + } + selected = [] + for field, entity in zip(body.fields, entities, strict=True): + index = positions[entity] + selected.append( + MoneyField( + field.name, + field.amounts[index].tobytes(), + field.statuses[index].tobytes(), + field.validity[index].tobytes(), + field.zero_origin[index].tobytes(), + ) + ) + parent = body.header_data + header = _json( + { + "schema_version": 1, + "artifact_kind": SELECTED_ARTIFACT_KIND, + "release_eligible": False, + "recipe": RECIPE, + "target_year": 2024, + "state": "target_current", + "semantic": "annual_current_money", + "source_authentication": parent["source_authentication"], + "parent_header_sha256": body.header_sha256, + "parent_content_sha256": body.content_sha256, + "prepared_receipt_sha256": prepared_receipt_sha256, + "selection_sha256": selection_sha256, + "person_rows": len(positions["person"]), + "household_rows": len(positions["household"]), + "person_identity_sha256": person_identity_sha256, + "household_identity_sha256": household_identity_sha256, + "fields": list(FIELDS), + "field_entities": [list(item) for item in body.field_entities], + "dtype": " None: + data = _parse(selected.header, HEADER_MAX_BYTES) + _require(set(data) == _SELECTED_HEADER_KEYS, "SELECTED_HEADER_SCHEMA") + _require(selected.header == _json(data), "SELECTED_HEADER_SCHEMA") + _require( + data["schema_version"] == 1 + and data["artifact_kind"] == SELECTED_ARTIFACT_KIND + and data["release_eligible"] is False + and data["recipe"] == RECIPE + and data["target_year"] == 2024 + and data["state"] == "target_current" + and data["semantic"] == "annual_current_money" + and data["dtype"] == " tuple[bytes, ...]: + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + _validate_selected(selected) + parts = [ + SELECTED_MAGIC, + struct.pack(" bytes: + """Encode the selected subset with a transport checksum over its bytes.""" + parts = _selected_parts(selected) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_selected_current_money( + payload: bytes, + *, + expected_parent_header_sha256: str, + expected_parent_content_sha256: str, + expected_prepared_receipt_sha256: str, + expected_selection_sha256: str, +) -> SelectedCurrentMoney: + """Decode a selected subset only against its declared parent and selection.""" + _require( + type(payload) is bytes + and len(SELECTED_MAGIC) + 4 + 32 < len(payload) <= SELECTED_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require( + all( + _digest(value) + for value in ( + expected_parent_header_sha256, + expected_parent_content_sha256, + expected_prepared_receipt_sha256, + expected_selection_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + _require(payload.startswith(SELECTED_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" 0 for value in counts.values()), + "SELECTED_ROW_BOUND", + ) + expected_size = start + size + 11 * sum(counts[entity] for entity in entities) + 32 + _require(len(payload) == expected_size, "PAYLOAD_LENGTH") + _require( + sha256(memoryview(payload)[:-32]).digest() == payload[-32:], "PAYLOAD_CHECKSUM" + ) + fields = _split(payload, start + size, entities, counts) + value = SelectedCurrentMoney(header, fields, _token=_SELECTED_TOKEN) + _validate_selected(value) + return value + + +__all__ = [ + "BoundCurrentMoneyBody", + "MoneyRefusalError", + "SELECTED_ARTIFACT_KIND", + "SELECTED_MAGIC", + "SELECTED_PAYLOAD_MAX_BYTES", + "SelectedCurrentMoney", + "US_ASEC_CURRENT_MONEY_BODY_TYPE", + "US_ASEC_PREPARED_RECEIPT_TYPE", + "US_ASEC_SELECTED_MONEY_TYPE", + "US_ASEC_SELECTION_TYPE", + "decode_selected_current_money", + "encode_selected_current_money", + "parse_current_money_body", + "select_current_money", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py new file mode 100644 index 000000000..afe4fa1ae --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py @@ -0,0 +1,591 @@ +"""Authenticate the reviewed full ASEC v4 plus household-observation attachment. + +No public pin override, file discovery, download, source selection, or unit fit. +Verification is source authority, not evidence that all downstream consumers are +corrected or that a population is eligible for release. +""" + +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import Frame + +from . import asec_checkpoint, asec_household_observations, operator_boundary +from .asec_current_money import ( + _DTYPES, + _SOURCE_TOKEN, + FIELDS, + ZERO_POLICY, + AsecMoneyScope, + AsecMoneyViews, + AuthenticatedAsecSource, + CurrentMoneySpec, + MoneyRefusalError, + _index_identity, + _json, + _parse, + _require, + _sha, + classify_asec_money, + compile_asec_current_money_spec, + require_complete_current_money, + restate_asec_current_money, +) +from .asec_current_money_resources import load_current_money_resources +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +# Exact reviewed bytes from the accepted current-money source contract and the +# completed household-observation receipt. Tests monkeypatch this PRIVATE tuple +# only for invented files; production has no override/constructor for other pins. +_SOURCE_PINS = ( + "e2f2b7495bfcf1448dfb0acb0a17e93f86a6ab8bef8a70ec0a2981983028cbb5", + "f7f086e262d9a1d9ec6fc1a0b7c32e577128578a4fc5b3f14e13fb8711f7ec2d", + ( + (2022, "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e"), + (2023, "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88"), + (2024, "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d"), + ), +) +_LOAD_TOKEN = object() +_ATTACHMENT_COLUMNS = tuple( + "asec_" + c for c in asec_household_observations.ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) + + +def _stage_verified(path, expected, staging): + """Load only a verified private copy, closing same-path replacement windows.""" + digest = hashlib.sha256() + try: + with Path(path).open("rb") as src, staging.open("xb") as dst: + before = os.fstat(src.fileno()) + while chunk := src.read(1024 * 1024): + digest.update(chunk) + dst.write(chunk) + after = os.fstat(src.fileno()) + _require( + ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ), + "SOURCE_BYTES_CHANGED", + ) + _require(digest.hexdigest() == expected, "SOURCE_BYTES_MISMATCH") + except OSError: + raise MoneyRefusalError("SOURCE_BYTES_UNAVAILABLE") from None + + +def _series_digest(digest, series): + """Use the existing checkpoint's closed scalar vocabulary, one column at a time.""" + spec = checkpoint._series_spec(series, label="authenticated source") + digest.update(_json(spec)) + extension = series.array + data, mask = getattr(extension, "_data", None), getattr(extension, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + for array in (data, mask): + value = np.ascontiguousarray(array).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + elif isinstance(series.dtype, np.dtype) and not series.dtype.hasobject: + value = np.ascontiguousarray(series.to_numpy(copy=False)).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + else: + for value in series.to_numpy(dtype=object, copy=False): + encoded = checkpoint._encode_object_scalar(value) + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + + +def _frame_signature(frame): + digest = hashlib.sha256(b"microcosm/asec-current-money-source-frame/1\0") + digest.update(_json(checkpoint._checkpoint_metadata(frame, {}))) + digest.update(_json(_plain_metadata(frame.metadata))) + for _, table in checkpoint._frame_tables(frame): + digest.update( + _json( + { + "columns_dtype": repr(table.columns.dtype), + "columns_type": type(table.columns).__name__, + } + ) + ) + _series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + _series_digest(digest, table[column]) + _series_digest(digest, frame.strata) + digest.update(_json(frame.strata.name)) + digest.update(_json(checkpoint._index_spec(frame.strata.index, label="strata"))) + _series_digest( + digest, + pd.Series( + frame.strata.index.to_numpy(copy=False), dtype=frame.strata.index.dtype + ), + ) + for entity in frame.weighted_entities: + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +def _owned_index(index): + """Detach axes, including pandas' materialized RangeIndex cache.""" + if isinstance(index, pd.RangeIndex): + # RangeIndex.copy(deep=True) shares its cached ndarray. Reconstruct the + # same logical range so a child cannot write through to a sealed parent. + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + return index.copy(deep=True) + + +def _detach_frame_axes(frame): + """Detach axes of an already copied Frame; preserve its data and axis types.""" + for _, table in checkpoint._frame_tables(frame): + table.index = _owned_index(table.index) + table.columns = _owned_index(table.columns) + frame.strata.index = _owned_index(frame.strata.index) + + +def _plain_metadata(value): + if isinstance(value, Mapping): + return [ + "mapping", + [[key, _plain_metadata(item)] for key, item in sorted(value.items())], + ] + if isinstance(value, tuple): + return ["tuple", [_plain_metadata(item) for item in value]] + if isinstance(value, frozenset): + return [ + "frozenset", + sorted((_plain_metadata(item) for item in value), key=_json), + ] + return ["scalar", value] + + +def _verification_identity(): + package = resources.files(__package__) + names = ( + "asec_current_money.py", + "asec_current_money_source.py", + "asec_checkpoint.py", + "asec_household_observations.py", + "education_assistance_source.py", + "reported_coverage_source.py", + "operator_boundary.py", + ) + shared = resources.files("microcosm.build") + frame_package = resources.files("microcosm.frame") + files = {name: _sha(package.joinpath(name).read_bytes()) for name in names} + files.update( + { + "build/" + name: _sha(shared.joinpath(name).read_bytes()) + for name in ( + "frame_checkpoint.py", + "outer_stage_runtime.py", + "serialization_dtypes.py", + ) + } + ) + files.update( + { + "frame/" + path.name: _sha(path.read_bytes()) + for path in frame_package.iterdir() + if path.name.endswith(".py") + } + ) + # This exact live projection is consumed by the existing source-only guard. + projection = { + family: {entity: sorted(columns) for entity, columns in by_entity.items()} + for family, by_entity in operator_boundary.PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES.items() + } + return _sha( + _json( + { + "modules": files, + "operator_owned": projection, + "pins": _SOURCE_PINS, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "h5py") + }, + } + ) + ) + + +def _scope(frame): + person = frame.person + keys = asec_household_observations._household_keys(frame) + + def ints(column): + values = person[column] + _require( + values.dtype == np.dtype("int64") and not values.isna().any(), + "SOURCE_COORDINATE_DTYPE", + ) + return tuple(int(v) for v in values.to_numpy()) + + years = ints("source_year") + _require(set(years) == {2022, 2023, 2024}, "SOURCE_YEAR_MAPPING") + return AsecMoneyScope( + ints("person_id"), + tuple(int(v) for v in keys.household_id), + ints("person_household_id"), + ints("person_spm_unit_id"), + years, + tuple(int(v) for v in keys.source_year), + tuple( + value.decode("utf-8") if isinstance(value, bytes) else value + for value in person.PERIDNUM.tolist() + ), + tuple(str(int(v)) for v in keys.H_SEQ), + ) + + +def _views(frame, scope, *, restored=False): + person = frame.person.loc[:, [f for f in FIELDS if f != "HTOTVAL"]].copy() + if restored: + person["PTOTVAL"] = frame.person["asec_PTOTVAL"].to_numpy(copy=True) + return AsecMoneyViews( + person, + frame.table("household").loc[:, ["asec_HTOTVAL"]].copy(), + scope, + ) + + +def _input_binding(views): + scope_sha = _sha( + _json( + { + "coordinates": _parse(views.scope.identity), + "person_index": _parse(_index_identity(views.person.index)), + "household_index": _parse(_index_identity(views.household.index)), + } + ) + ) + inputs = [] + for entity, table in (("person", views.person), ("household", views.household)): + for column, series in table.items(): + field = "HTOTVAL" if entity == "household" else column + _require(str(series.dtype) in _DTYPES, "UNSUPPORTED_DTYPE", field) + values = series.to_numpy(dtype="= 0 + and row["joined_rows"] == int((keys.source_year == year).sum()) + and row["unreferenced_source_rows"] + == row["source_rows"] - row["joined_rows"] + >= 0, + "ATTACHMENT_SOURCE_COVERAGE", + ) + return cohorts + + +def _source_verification_identity(evidence): + from .asec_current_money import RESTORED_SOURCE_KIND + + if evidence["source_kind"] == RESTORED_SOURCE_KIND: + from .asec_person_income_source import _implementation + + return _implementation() + return _verification_identity() + + +@dataclass(frozen=True) +class AuthenticatedCurrentMoneySource: + frame: Frame + scope: AsecMoneyScope + source: AuthenticatedAsecSource + spec: CurrentMoneySpec + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _LOAD_TOKEN, "SOURCE_CONSTRUCTOR_UNAVAILABLE") + + def validate(self): + _require( + type(self.source) is AuthenticatedAsecSource, + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + self.source._validate() + evidence = _parse(self.source.identity) + _require( + _source_verification_identity(evidence) == evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + _require( + _frame_signature(self.frame) == evidence["frame_sha256"] + and _scope(self.frame) == self.scope, + "SOURCE_CHANGED", + ) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError): + raise MoneyRefusalError("SOURCE_CHANGED") from None + + def views(self): + self.validate() + from .asec_current_money import _restored_source + + return _views(self.frame, self.scope, restored=_restored_source(self.source)) + + def ready(self): + views = self.views() + decoded = classify_asec_money(views, self.spec, self.scope) + return require_complete_current_money( + restate_asec_current_money(decoded, self.spec), self.spec, production=True + ) + + +def load_authenticated_current_money_source(parent_path, attachment_path): + """Issue authority only for the two reviewed full-source files and exact scope.""" + try: + return _load(parent_path, attachment_path) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, OSError): + raise MoneyRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(parent_path, attachment_path): + before = _verification_identity() + with tempfile.TemporaryDirectory(prefix="microcosm-money-source-") as directory: + directory = Path(directory) + parent_copy = directory / "parent.h5" + _stage_verified(parent_path, _SOURCE_PINS[0], parent_copy) + parent, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + parent_copy + ) + parent_copy.unlink() + attachment_copy = directory / "attachment.h5" + _stage_verified(attachment_path, _SOURCE_PINS[1], attachment_copy) + loaded = checkpoint.load_frame_checkpoint(attachment_copy) + canonicalize_frame_string_dtypes( + loaded.frame, boundary="ASEC money attachment", in_place=True + ) + cohorts = _validate_attachment(parent, metadata, loaded, _SOURCE_PINS[0]) + sidecars = [] + actual_pins = metadata["raw_source_mappings"]["ED_VAL"]["source_pins"] + _require(len(actual_pins) == 3, "ED_VAL_SOURCE_COVERAGE") + for year in (2022, 2023, 2024): + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year] + expected = { + "income_year": year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + } + _require(actual_pins.count(expected) == 1, "ED_VAL_REGISTERED_PIN") + sidecars.append( + { + "income_year": year, + "survey_year": registered.survey_year, + "archive_sha256": registered.zip_sha256, + "member": registered.member, + "member_sha256": registered.member_sha256, + } + ) + frame = loaded.frame + scope = _scope(frame) + views = _views(frame, scope) + scope_sha, input_sha = _input_binding(views) + evidence = { + "schema_version": 1, + "source_kind": "asec_v4_with_household_observations_v1", + "source_authentication": "checkpoint_bytes_verified", + "parent_sha256": _SOURCE_PINS[0], + "attachment_sha256": _SOURCE_PINS[1], + "cohorts": cohorts, + "sidecars": sidecars, + "field_roster": FIELDS, + "zero_origin_policy": ZERO_POLICY, + "scope_sha256": scope_sha, + "input_sha256": input_sha, + "frame_sha256": _frame_signature(frame), + "verification_sha256": before, + } + _require(_verification_identity() == before, "SOURCE_IMPLEMENTATION_CHANGED") + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + return AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=_LOAD_TOKEN + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py new file mode 100644 index 000000000..3dd275752 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py @@ -0,0 +1,411 @@ +"""Actual tax-only reconstruction from verified target-current ASEC money. + +The transient view has a closed read set. All other observed source columns and +US entities remain untouched. This is not a policy impact or release evaluator. +""" + +import inspect +import sys +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes +from microcosm.frame import Frame, Weights +from microcosm.frame import units as frame_units + +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + MICROUNIT_VERSION, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_current_money_source import ( + AuthenticatedCurrentMoneySource, + _detach_frame_axes, + _frame_signature, + _same_table, + _source_verification_identity, +) +from .asec_student_controls import ( + ALIASES as STUDENT_ALIASES, +) +from .asec_student_controls import ( + CONTROLS as STUDENT_CONTROLS, +) +from .asec_student_controls import ( + StudentControlsAttachedAsec, +) + +MICROUNIT_CURRENT_MONEY_COLUMNS = tuple( + "WSAL_VAL SEMP_VAL FRSE_VAL INT_VAL DIV_VAL RNT_VAL CAP_VAL UC_VAL OI_VAL ANN_VAL PNSN_VAL SS_VAL PTOTVAL".split() +) +_STRUCTURAL_COLUMNS = ( + "PH_SEQ", + "A_LINENO", + "A_AGE", + "A_MARITL", + "A_SPOUSE", + "PEPAR1", + "PEPAR2", + "A_EXPRRP", +) +_CONTROL_COLUMNS = ( + "A_ENRLW", + "A_FTPT", + "A_HSCOL", + "PEDISDRS", + "PEDISEAR", + "PEDISEYE", + "PEDISOUT", + "PEDISPHY", + "PEDISREM", +) +MICROUNIT_CURRENT_INPUT_COLUMNS = ( + _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS + MICROUNIT_CURRENT_MONEY_COLUMNS +) +TAX_PERSON_OUTPUTS = ( + "person_tax_unit_id", + "tax_unit_role_input", + "is_related_to_head_or_spouse", +) +_RESULT_TOKEN = object() + + +class MicrounitControlRefusalError(MoneyRefusalError): + """Closed control-field diagnostics without changing the money-source schema.""" + + def __init__(self, reason, column): + super().__init__(reason) + self.field = ( + column if column in _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS else "contract" + ) + self.args = (f"{self.field}: {reason}",) + + +def _require_control(condition, reason, column): + if not condition: + raise MicrounitControlRefusalError(reason, column) + + +# Exact 0.1.0 import/runtime closure: __init__ imports core/diagnostics/registry; +# tax construction calls rule_helpers, whose cached threshold reads this YAML. +# No PolicyEngine package is needed for those rules. +_MICROUNIT_EXECUTION_PINS = { + "__init__.py": "6164ebb289659a027d2bdd419781bd1025483bb2baf5576487f8e3f4ac594e01", + "core.py": "95ec837e23e9a61a20bb48bcf20f87ffbfa536aaf0536706d020c6f79dc47701", + "diagnostics.py": "c1a683b98ceac7a44438653eb992240e5d9ed130119f75f69f6bed3a02e6c7c6", + "registry.py": "3ae5442a7bc637f8f786913abae97f63997baba5cedac948a80ce3b00f8cf673", + "rule_helpers.py": "7d32f1b3ab855a21ec0ca2e6d30431c4dcb030ffb250a28ef1978c50ed334243", + "tax_unit_construction.py": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "data/dependent_gross_income_limit.yaml": "3e20dd8f44d5ff919497fe38c2f68aa97a9f333539fbea7d0ca4f2c29e588da0", +} + + +def _runtime_file_bytes(path): + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + value = path.read_bytes() + _require(len(value) <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + return value + + +def _execution(): + try: + distribution = metadata.distribution("microunit") + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + root = Path(distribution.locate_file("microunit")).resolve() + inventory = {str(p) for p in distribution.files or ()} + for name, expected in _MICROUNIT_EXECUTION_PINS.items(): + _require( + "microunit/" + name in inventory + and _sha(_runtime_file_bytes(root / name)) == expected, + "MICROUNIT_EXECUTION_PIN", + ) + import microunit + + _require( + Path(microunit.__file__).resolve() == root / "__init__.py", + "MICROUNIT_IMPORT_ORIGIN", + ) + _require( + inspect.getsourcefile(microunit.construct_tax_units) + == str(root / "tax_unit_construction.py"), + "MICROUNIT_IMPORT_ORIGIN", + ) + for name in _MICROUNIT_EXECUTION_PINS: + if name.endswith(".py"): + module_name = ( + "microunit" if name == "__init__.py" else "microunit." + name[:-3] + ) + module = sys.modules.get(module_name) + _require( + module is not None + and Path(module.__file__).resolve() == root / name, + "MICROUNIT_IMPORT_ORIGIN", + ) + package = resources.files(__package__) + files = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_current_money_units.py", + "asec_current_money_source.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + "asec_student_controls.py", + ) + } + files["frame.units"] = _sha(Path(frame_units.__file__).read_bytes()) + identity = { + "version": distribution.version, + "microunit": _MICROUNIT_EXECUTION_PINS, + "adapter_modules": files, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "pyyaml") + }, + "year": 2024, + "mode": "policyengine", + } + return microunit.construct_tax_units, _json(identity) + except (metadata.PackageNotFoundError, OSError, ImportError): + raise MoneyRefusalError("MICROUNIT_EXECUTION_UNAVAILABLE") from None + + +def current_money_microunit_view(source, ready): + """Project only reviewed controls and corrected amounts, in actual person order.""" + _require( + type(source) in (AuthenticatedCurrentMoneySource, StudentControlsAttachedAsec) + and type(ready) is ReadyCurrentMoney, + "AUTHENTICATED_MONEY_REQUIRED", + ) + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "MONEY_CONTENT_MISMATCH", + ) + _require(ready.bindings.spec == source.spec, "SPEC_MISMATCH") + _validate_money(ready, source.spec, nominal=False) + _require( + all(field.validity.all() for field in ready.fields), "MISSING_REQUIRED_AMOUNT" + ) + person = source.frame.person + _require( + set(_STRUCTURAL_COLUMNS + _CONTROL_COLUMNS) <= set(person), + "MICROUNIT_INPUT_ROSTER", + ) + view = ( + person.loc[:, _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS] + .copy() + .reset_index(drop=True) + ) + if type(source) is StudentControlsAttachedAsec: + # Only authenticated, code-owned aliases replace the transient logical + # controls. Original source observations, including NaNs, stay untouched. + source.validate() + for name, alias in zip(STUDENT_CONTROLS, STUDENT_ALIASES, strict=True): + view[name] = person[alias].to_numpy(copy=True) + for column in view: + values = view[column] + _require_control( + pd.api.types.is_numeric_dtype(values.dtype) + and not pd.api.types.is_bool_dtype(values.dtype), + "MICROUNIT_CONTROL_DTYPE", + column, + ) + _require_control(not values.isna().any(), "MICROUNIT_CONTROL_MISSING", column) + array = values.to_numpy(dtype=np.float64) + _require_control( + np.isfinite(array).all() and (array == np.floor(array)).all(), + "MICROUNIT_CONTROL_VALUES", + column, + ) + _require( + np.array_equal(view.PH_SEQ.to_numpy(), person.person_household_id.to_numpy()), + "MICROUNIT_HOUSEHOLD_BINDING", + ) + _require( + not view[["PH_SEQ", "A_LINENO"]].duplicated().any() + and (view.A_LINENO > 0).all(), + "MICROUNIT_PERSON_LINES", + ) + for field in MICROUNIT_CURRENT_MONEY_COLUMNS: + view[field] = ready.field(field).amounts.copy() + _require( + tuple(view.columns) == MICROUNIT_CURRENT_INPUT_COLUMNS, "MICROUNIT_INPUT_ROSTER" + ) + return view + + +def _partitions(frame): + grouped = {} + for person, group in zip( + frame.person.person_id, frame.person.person_tax_unit_id, strict=True + ): + grouped.setdefault(int(group), []).append(int(person)) + return tuple(sorted(tuple(sorted(members)) for members in grouped.values())) + + +@dataclass(frozen=True) +class CurrentMoneyTaxUnitResult: + frame: Frame + money: ReadyCurrentMoney + _receipt: bytes + _token: InitVar[object] = None + _student_source: StudentControlsAttachedAsec | None = None + + def __post_init__(self, _token): + _require(_token is _RESULT_TOKEN, "TAX_RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self): + """Check the permitted tax-only reattachment before consuming a live result.""" + receipt = self.receipt + if self._student_source is not None: + _require( + type(self._student_source) is StudentControlsAttachedAsec, + "STUDENT_TAX_PARENT", + ) + self._student_source.validate() + _require( + self.money.bindings == self._student_source.money.bindings + and self.money.fields == self._student_source.money.fields + and receipt["student_controls"] == self._student_source.receipt, + "STUDENT_TAX_PARENT", + ) + else: + _require(receipt["student_controls"] is None, "STUDENT_TAX_PARENT") + _validate_money(self.money, self.money.bindings.spec, nominal=False) + source_evidence = _parse(self.money.bindings.spec.source.identity) + _require( + _source_verification_identity(source_evidence) + == source_evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + frame_signature = _frame_signature(self.frame) + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RESULT_CHANGED") from None + _require( + _sha(self.money.header) == receipt["money_header_sha256"] + and current_money_content_sha256(self.money) + == receipt["money_content_sha256"] + and frame_signature == receipt["output_frame_sha256"], + "TAX_RESULT_CHANGED", + ) + _require( + _execution()[1] == _json(receipt["implementation"]), + "MICROUNIT_EXECUTION_CHANGED", + ) + + +def reconstruct_current_money_tax_units(source, ready): + """Run the pinned real engine, graft tax outputs, and verify preservation.""" + try: + return _reconstruct(source, ready) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RECONSTRUCTION_REFUSAL") from None + + +def _reconstruct(source, ready): + view = current_money_microunit_view(source, ready) + construct, implementation = _execution() + original = source.frame + assignments, tax_unit = frame_units._construct_tax_units( + construct, view, year=2024, mode="policyengine" + ) + tax_unit = canonicalize_table_string_dtypes( + tax_unit, + boundary="ASEC current-money tax reconstruction", + table_name="tax_unit", + ) + _require( + assignments.index.equals(view.index) and len(assignments) == len(view), + "MICROUNIT_OUTPUT_ROWS", + ) + person = original.person.copy(deep=True) + for target, output in zip( + TAX_PERSON_OUTPUTS, + ("TAX_ID", "tax_unit_role_input", "is_related_to_head_or_spouse"), + strict=True, + ): + dtype = person[target].dtype if target in person else assignments[output].dtype + person[target] = pd.array(assignments[output].to_numpy(copy=True), dtype=dtype) + tables = {entity: original.table(entity) for entity in original.entities} + tables["person"] = person + tables["tax_unit"] = tax_unit + result = Frame( + tables, + original.schema, + { + entity: Weights( + original.weights_for(entity).values, original.weights_for(entity).kind + ) + for entity in original.weighted_entities + }, + original.strata, + mass_log=original.mass_log, + metadata=original.metadata, + ) + _detach_frame_axes(result) + source.validate() + _require(_execution()[1] == implementation, "MICROUNIT_EXECUTION_CHANGED") + for entity in original.entities: + if entity not in ("person", "tax_unit"): + _same_table(result.table(entity), original.table(entity)) + untouched = [name for name in original.person if name not in TAX_PERSON_OUTPUTS] + _same_table(result.person[untouched], original.person[untouched]) + pd.testing.assert_series_equal(result.strata, original.strata, check_exact=True) + _require( + result.metadata == original.metadata and result.mass_log == original.mass_log, + "TAX_CONTEXT_PRESERVATION", + ) + for entity in original.weighted_entities: + _require( + result.weights_for(entity).kind == original.weights_for(entity).kind + and result.weights_for(entity).values.tobytes() + == original.weights_for(entity).values.tobytes(), + "TAX_WEIGHT_PRESERVATION", + ) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_current_money_tax_units", + "release_eligible": False, + "all_current_money_consumers_wired": False, + "year": 2024, + "mode": "policyengine", + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "source_frame_sha256": _parse(source.source.identity)["frame_sha256"], + "student_controls": source.receipt + if type(source) is StudentControlsAttachedAsec + else None, + "output_frame_sha256": _frame_signature(result), + "implementation": _parse(implementation), + "person_outputs": TAX_PERSON_OUTPUTS, + "tax_table_outputs": tuple(tax_unit.columns), + "old_tax_units": original.n("tax_unit"), + "new_tax_units": result.n("tax_unit"), + "old_partition_sha256": _sha(_json(_partitions(original))), + "new_partition_sha256": _sha(_json(_partitions(result))), + "scope_sha256": _parse(ready.header)["scope_sha256"], + } + return CurrentMoneyTaxUnitResult( + result, + ready, + _json(receipt), + _token=_RESULT_TOKEN, + _student_source=source if type(source) is StudentControlsAttachedAsec else None, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py new file mode 100644 index 000000000..5648d95a4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py @@ -0,0 +1,473 @@ +"""Closure-admitted PolicyEngine-US evaluation of the corrected money leaves. + +An output is admitted only when its complete static input closure is produced by +this graph. The defaults allowlist is empty by decision, so a closure leaf this +slice does not map blocks its root outright rather than being zero-filled or +split into an unreviewed surface. Each admitted root carries a pinned closure +identity; engine or closure drift refuses before any simulation runs. + +Outputs are formula-owned. They are retained in an evaluation artifact and are +never written back as population cells. +""" + +from __future__ import annotations + +import struct +import sys +from dataclasses import dataclass +from hashlib import sha256 +from importlib import resources +from importlib.metadata import distribution +from importlib.util import find_spec +from pathlib import Path + +import numpy as np + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ArtifactType +from microcosm.graph.canonical import canonical_json + +from .operator_boundary import PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + +US_ASEC_ENGINE_EVALUATION_TYPE = ArtifactType("microcosm.us.asec_engine_evaluation", 1) +ENGINE_PACKAGE = "policyengine-us" +ENGINE_DEFAULTS_RESOURCE = "asec_current_money_engine_defaults_v1.json" +ENGINE_DEFAULTS_SHA256 = ( + "090dd200ecdb837127dba1d1f22903d152cef8a434085e4ead4d379350ee5b82" +) +ENGINE_DEFAULTS_ARTIFACT_KIND = "microcosm.us.asec_current_money_engine_defaults.v1" +EVALUATION_MAGIC = b"MCASECEV\x01" +EVALUATION_ARTIFACT_KIND = "microcosm.us.asec_engine_evaluation.v1" +EVALUATION_PERIOD = 2024 +EVALUATION_HEADER_MAX_BYTES = 64 * 1024 +EVALUATION_MAX_ROWS = 1_000_000 +_PRIOR_YEAR_MARKERS = ("last_year", "previous_year", "prior_year") + + +class EngineAdmissionError(ValueError): + """A candidate output is not admitted, or a pinned identity drifted.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise EngineAdmissionError(reason) + + +@dataclass(frozen=True) +class EngineOutputContract: + """Pinned identity of one admitted root's static PolicyEngine-US closure.""" + + root: str + entity: str + engine_version: str + input_leaves: tuple[str, ...] + formula_node_count: int + edge_count: int + sha256: str + + +ADMITTED_ENGINE_OUTPUT_CONTRACTS: tuple[EngineOutputContract, ...] = ( + EngineOutputContract( + root="capital_gains", + entity="person", + engine_version="1.819.0", + input_leaves=( + "long_term_capital_gains_before_response", + "short_term_capital_gains", + ), + formula_node_count=4, + edge_count=6, + sha256="1768cb833b183b3898567910b463e645a2744c2138688c50b5fc1cc17c3319df", + ), + EngineOutputContract( + root="dividend_income", + entity="person", + engine_version="1.819.0", + input_leaves=("non_qualified_dividend_income", "qualified_dividend_income"), + formula_node_count=2, + edge_count=3, + sha256="a27f07536f4909361de7444831f7852a3f976e12d40d6afe4059c6a5fbdfbbe3", + ), + EngineOutputContract( + root="employment_income", + entity="person", + engine_version="1.819.0", + input_leaves=("employment_income_before_lsr",), + formula_node_count=3, + edge_count=4, + sha256="bdcd8da23a79ff228a610a4d0b483f5fd2068d968f31e96c07cea8a86cc3366b", + ), + EngineOutputContract( + root="social_security", + entity="person", + engine_version="1.819.0", + input_leaves=( + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + ), + formula_node_count=1, + edge_count=4, + sha256="3f5a5304ac1e8c8d785614069867590176325b212a0a080457f80594a6fd53c0", + ), +) +BLOCKED_ENGINE_OUTPUTS: tuple[str, ...] = ( + "interest_income", + "pension_income", + "self_employment_income", +) +ADMITTED_ROOTS: tuple[str, ...] = tuple( + contract.root for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS +) + + +def load_engine_defaults() -> dict: + """Read and pin-verify the defaults resource; the allowlist must stay empty.""" + payload = ( + resources.files(__package__).joinpath(ENGINE_DEFAULTS_RESOURCE).read_bytes() + ) + _require( + sha256(payload).hexdigest() == ENGINE_DEFAULTS_SHA256, + "ENGINE_DEFAULTS_FINGERPRINT", + ) + import json + + document = json.loads(payload) + _require( + document["schema_version"] == 1 + and document["artifact_kind"] == ENGINE_DEFAULTS_ARTIFACT_KIND, + "ENGINE_DEFAULTS_SCHEMA", + ) + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + _require( + tuple(sorted(document["admitted_roots"])) == ADMITTED_ROOTS, + "ENGINE_DEFAULTS_ROOTS", + ) + _require( + tuple(sorted(document["blocked_roots"])) == BLOCKED_ENGINE_OUTPUTS, + "ENGINE_DEFAULTS_BLOCKED", + ) + _require(document["engine_package"] == ENGINE_PACKAGE, "ENGINE_DEFAULTS_PACKAGE") + _require(document["release_eligible"] is False, "ENGINE_DEFAULTS_RELEASE_CLAIM") + return document + + +def _runtime_package_identity(package: str, module: str) -> dict: + """Hash installed model code and parameter files, never population data. + + Include relative names so additions, removals and relocation have explicit + semantics. Recompute on every invocation: cache admission must see an edit + made after a previous evaluation in the same process. + """ + installed = distribution(package) + root = Path(installed.locate_file(module)) + files = sorted( + path + for path in root.rglob("*") + if path.is_file() + and ( + path.suffix == ".py" + or ( + "parameters" in path.relative_to(root).parts + and path.suffix in (".yaml", ".yml", ".json") + ) + ) + ) + records = [ + [path.relative_to(root).as_posix(), sha256(path.read_bytes()).hexdigest()] + for path in files + ] + return { + "version": installed.version, + "files": len(records), + "source_and_parameters_sha256": sha256(canonical_json(records)).hexdigest(), + } + + +def _verify_runtime_origin(package: str, module: str) -> None: + """The adapter must import the same distribution whose bytes are pinned.""" + root = Path(distribution(package).locate_file(module)).resolve() + spec = find_spec(module) + _require( + spec is not None + and spec.origin is not None + and Path(spec.origin).resolve() == root / "__init__.py" + and spec.submodule_search_locations is not None + and [Path(path).resolve() for path in spec.submodule_search_locations] + == [root], + "ENGINE_RUNTIME_ORIGIN", + ) + for name, loaded in tuple(sys.modules.items()): + if name != module and not name.startswith(module + "."): + continue + filename = getattr(loaded, "__file__", None) + if filename is not None: + _require( + Path(filename).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN" + ) + for path in getattr(loaded, "__path__", ()): + _require(Path(path).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN") + + +def engine_runtime_identity() -> dict: + """Verify the pinned baseline runtime before execution or a cache lookup.""" + document = load_engine_defaults() + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ): + _verify_runtime_origin(package, module) + observed = { + package: _runtime_package_identity(package, module) + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ) + } + _require(observed == document["baseline_runtime"], "ENGINE_RUNTIME_DRIFT") + return {"policy": "installed_baseline_no_reforms", "packages": observed} + + +def _prior_year_leaves() -> frozenset[str]: + family = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + return frozenset(name for columns in family.values() for name in columns) + + +def _closure(index, root: str): + try: + return index.variable_dependency_closure(root) + except ValueError as error: + raise EngineAdmissionError(f"UNKNOWN_ENGINE_ROOT:{root}") from error + + +def _check_contract(closure, contract: EngineOutputContract) -> None: + observed = { + "engine_version": closure.engine_version, + "root": closure.root, + "input_leaves": tuple(closure.input_leaves), + "formula_node_count": len(closure.formula_nodes), + "edge_count": len(closure.edges), + "sha256": closure.sha256, + } + expected = { + "engine_version": contract.engine_version, + "root": contract.root, + "input_leaves": contract.input_leaves, + "formula_node_count": contract.formula_node_count, + "edge_count": contract.edge_count, + "sha256": contract.sha256, + } + if observed != expected: + raise EngineAdmissionError( + "PolicyEngine-US closure drifted for " + f"{contract.root!r}; expected={expected}, observed={observed}." + ) + + +def admit_engine_outputs( + index, + *, + produced_leaves, + defaults: dict | None = None, +) -> tuple[EngineOutputContract, ...]: + """Admit only roots whose complete closure this graph produces.""" + document = load_engine_defaults() if defaults is None else defaults + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + produced = frozenset(produced_leaves) + prior_year = _prior_year_leaves() + for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS: + closure = _closure(index, contract.root) + _check_contract(closure, contract) + metadata = index.variable_metadata(contract.root) + _require(metadata.entity == contract.entity, "ADMITTED_ROOT_ENTITY") + missing = sorted(set(closure.input_leaves) - produced) + if missing: + raise EngineAdmissionError( + f"Root {contract.root!r} has unproduced closure leaves {missing}; " + "the engine-default allowlist is empty by decision." + ) + reaching = sorted(set(closure.input_leaves) & prior_year) + _require(not reaching, f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}") + _require( + not any( + marker in leaf + for leaf in closure.input_leaves + for marker in _PRIOR_YEAR_MARKERS + ), + f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}", + ) + for root in BLOCKED_ENGINE_OUTPUTS: + closure = _closure(index, root) + blocked = sorted(set(closure.input_leaves) - produced) + _require(bool(blocked), f"BLOCKED_ROOT_NOW_COMPLETE:{root}") + return ADMITTED_ENGINE_OUTPUT_CONTRACTS + + +@dataclass(frozen=True) +class EngineEvaluation: + """Engine outputs retained as evidence; never population cells.""" + + header: dict + values: dict[str, np.ndarray] + + @property + def roots(self) -> tuple[str, ...]: + return tuple(sorted(self.values)) + + +def _aggregates(values: np.ndarray) -> dict[str, object]: + return { + "rows": int(values.size), + "nonzero_rows": int(np.count_nonzero(values)), + "unweighted_total": float(values.sum()), + "minimum": float(values.min()), + "maximum": float(values.max()), + } + + +def materialize_engine_outputs( + frame: Frame, + engine, + contracts: tuple[EngineOutputContract, ...], + *, + period: int = EVALUATION_PERIOD, + context_bindings: dict | None = None, +) -> EngineEvaluation: + """Run the real adapter for the admitted roots and retain the arrays only.""" + _require(type(frame) is Frame and frame.schema == US_SCHEMA, "ENGINE_FRAME_SCHEMA") + _require(getattr(engine, "country", None) == "us", "ENGINE_COUNTRY") + _require(period == EVALUATION_PERIOD, "ENGINE_PERIOD") + _require(bool(contracts), "ENGINE_NO_ADMITTED_OUTPUTS") + _require(contracts == ADMITTED_ENGINE_OUTPUT_CONTRACTS, "ENGINE_CONTRACT_ROSTER") + runtime = engine_runtime_identity() + rows = frame.n("person") + _require(0 < rows <= EVALUATION_MAX_ROWS, "ENGINE_ROW_BOUND") + roots = tuple(sorted(contract.root for contract in contracts)) + _require(len(set(roots)) == len(roots), "ENGINE_DUPLICATE_ROOT") + results = engine.materialize(frame, list(roots), period) + _require(set(results) == set(roots), "ENGINE_RESULT_ROSTER") + values: dict[str, np.ndarray] = {} + for root in roots: + array = np.asarray(results[root], dtype="float64") + _require(array.shape == (rows,), "ENGINE_RESULT_SHAPE") + _require(bool(np.isfinite(array).all()), "ENGINE_RESULT_NONFINITE") + values[root] = np.ascontiguousarray(array, dtype=" tuple[bytes, ...]: + _require(type(evaluation) is EngineEvaluation, "TYPED_EVALUATION_REQUIRED") + header = canonical_json(evaluation.header) + _require(0 < len(header) <= EVALUATION_HEADER_MAX_BYTES, "HEADER_SIZE") + roots = tuple(evaluation.header["roots"]) + _require(roots == tuple(sorted(evaluation.values)), "ENGINE_RESULT_ROSTER") + rows = evaluation.header["person_rows"] + parts = [EVALUATION_MAGIC, struct.pack(" bytes: + """Encode the evaluation header and arrays with a transport checksum.""" + parts = _evaluation_parts(evaluation) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_engine_evaluation(payload: bytes) -> EngineEvaluation: + """Decode a stored evaluation, verifying framing, checksum and shapes.""" + _require( + type(payload) is bytes and len(payload) > len(EVALUATION_MAGIC) + 4 + 32, + "PAYLOAD_SIZE", + ) + _require(payload.startswith(EVALUATION_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + if not condition: + raise HouseholdCoverageSourceError(reason) + + +def _producer() -> dict: + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + return { + "protocol": _SCHEMA, + "owner_sha256": shared._sha( + resources.files(__package__) + .joinpath("asec_household_coverage_fields.py") + .read_bytes() + ), + "bounded_reader_dependency": shared._implementation(), + "csv_builtin_sha256": shared._sha( + resources.files(__package__).joinpath("source_csv_builtin.py").read_bytes() + ), + "columns": COLUMNS, + "fields": _FIELDS, + "interpretation_notes": _INTERPRETATION_NOTES, + } + + +def _code_state(name: str, token: str, interview_code: int | None) -> dict: + contract = _FIELDS[name] + code, label = None, None + if token == "": + status = "missing" + elif re.fullmatch(rf"[0-9]{{1,{contract['printed_length']}}}", token) is None: + status = "malformed" + else: + code = int(token) + label = contract["labels"].get(str(code)) + status = "valid" if label is not None else "unlabelled" + universe = ( + "in" + if contract["universe_as_printed"] == "All Households" or interview_code == 1 + else "outside" + if interview_code in (2, 3) + else "unresolved" + ) + return { + "code": code, + "label": label, + "code_status": status, + "universe_status": universe, + } + + +def _project(row: Sequence[str]) -> dict: + _require( + all(len(token) <= shared._MAX_FIELD_CHARS for token in row), "MEMBER_FIELD_SIZE" + ) + native = row[0] + _require( + re.fullmatch(r"[0-9]+", native) is not None and 1 <= int(native) <= 99999, + "MEMBER_NATIVE_KEY", + ) + literals = dict(zip(COLUMNS, row, strict=True)) + interview = _code_state("H_HHTYPE", literals["H_HHTYPE"], None) + interview_code = interview["code"] if interview["code_status"] == "valid" else None + states = { + name: _code_state(name, literals[name], interview_code) for name in COLUMNS[1:] + } + hr = ( + states["HRHTYPE"]["code"] + if states["HRHTYPE"]["code_status"] == "valid" + else None + ) + count = ( + states["H_NUMPER"]["code"] + if states["H_NUMPER"]["code_status"] == "valid" + else None + ) + diagnostics = [] + if interview_code == 1: + if hr == 0: + diagnostics.append("interview_with_noninterview_household_type") + if count == 0: + diagnostics.append("interview_with_noninterview_person_count") + elif interview_code in (2, 3): + if hr is not None and hr > 0: + diagnostics.append("noninterview_with_interview_household_type") + if count is not None and count > 0: + diagnostics.append("noninterview_with_positive_person_count") + if states["H_LIVQRT"]["code"] == 11: + diagnostics.append("student_quarters_dictionary_methodology_unresolved") + return { + **literals, + "native_household_id": int(native), + "field_states": states, + "reported_person_count": count + if interview_code == 1 and count is not None and count > 0 + else None, + "diagnostics": diagnostics, + "coverage_interpretation": "unresolved", + } + + +def _read_capture(capture: Path, pin) -> list[dict]: + """Compose existing bounded byte/record primitives with this field projection.""" + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + result, seen = [], set() + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + with open(capture, "rb", buffering=0, opener=shared._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = shared._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = shared._RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _project([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, key) == getattr(after, key) + for key in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdCoverageFields: + """Reader-issued immutable original observations, without Frame authority.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + try: + digest = shared._issue_capsule(self, self.payload) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + object.__setattr__(self, "_issued_sha256", digest) + + @property + def document(self) -> dict: + """Return a defensive view after checking issuance and current producer.""" + return verify_asec_household_coverage_fields(self) + + def households_for( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[dict, ...]: + """Look up exact cohort/native keys; preserve unresolved observations.""" + document = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= shared._MAX_ROWS, + "LOOKUP_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "LOOKUP_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "LOOKUP_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in document["records"] + } + _require(all(key in records for key in keys), "LOOKUP_UNKNOWN") + return tuple(records[key] for key in keys) + + +def verify_asec_household_coverage_fields( + source: AuthenticatedAsecHouseholdCoverageFields, +) -> dict: + """Validate a reader-issued capsule; JSON is not an admission credential.""" + _require(type(source) is AuthenticatedAsecHouseholdCoverageFields, "SOURCE_TYPE") + try: + payload = shared._checked_capsule_payload(source) + document = json.loads(payload) + _require( + shared._encode(document["producer"]) == shared._encode(_producer()), + "PRODUCER_CHANGED", + ) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + return document + + +def load_authenticated_asec_household_coverage_fields( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdCoverageFields: + """Reconstruct whole requested original cohorts, never caller-provided rows.""" + try: + pins = shared._registry() + paths = shared._member_path_snapshot(member_paths, pins) + _require( + candidate is None + or ( + type(candidate) is bytes and len(candidate) <= shared._MAX_PAYLOAD_BYTES + ), + "CANDIDATE_SIZE", + ) + producer_document = _producer() + _require( + shared._encode(producer_document["bounded_reader_dependency"]["registry"]) + == shared._encode([asdict(pin) for pin in pins]), + "REGISTRY_TRANSITION", + ) + producer = shared._encode(producer_document) + selected = [pin for pin in pins if pin.income_year in paths] + _require(bool(selected), "MEMBER_PATHS") + _require(sum(pin.rows for pin in selected) <= shared._MAX_ROWS, "REQUEST_ROWS") + records = [] + with tempfile.TemporaryDirectory( + prefix="asec-household-coverage-fields-" + ) as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = shared._capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdCoverageSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except ( + HouseholdCoverageSourceError, + shared.HouseholdWeightSourceError, + ): + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdCoverageSourceError("MEMBER_ENCODING") from None + _require(shared._encode(_producer()) == producer, "PRODUCER_CHANGED") + payload = shared._encode( + { + "schema": _SCHEMA, + "producer": json.loads(producer), + "members": [asdict(pin) for pin in selected], + "fields": _FIELDS, + "dictionary_authorities": [ + { + "survey_year": year, + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, _page, digest in shared._DICTIONARIES + ], + "interpretation_notes": _INTERPRETATION_NOTES, + "native_roster_sha256": shared._sha( + shared._encode( + [ + [row["income_year"], row["native_household_id"]] + for row in records + ] + ) + ), + "projection_sha256": shared._sha(shared._encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "coverage_classification_performed": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdCoverageFields(payload, _token=_TOKEN) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py new file mode 100644 index 000000000..c0ba8eb56 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py @@ -0,0 +1,392 @@ +"""Attach authenticated raw ASEC household observations by exact original keys. + +The caller authenticates the v4 checkpoint bytes and loads its actual Frame and +metadata with ``load_asec_raw_stage_checkpoint_v4`` before calling this helper. +Structural checks here do not authenticate arbitrary caller-supplied Frame +values. The augmented source has a new attachment receipt; it is not the input +checkpoint's v4 content identity. No graph registration or price conversion runs. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +import h5py +import numpy as np +import pandas as pd + +from microcosm.build.outer_stage_runtime import FrameIdentity, frame_identity +from microcosm.build.us_runtime import asec_checkpoint +from microcosm.build.us_runtime.operator_boundary import ( + assert_operator_free_source_frame, +) +from microcosm.frame import Frame + +ASEC_HOUSEHOLD_OBSERVATION_COLUMNS = ( + "HTOTVAL", + "H_LIVQRT", + "HRHTYPE", + "H_HHTYPE", + "H_SEQ", + "H_TENURE", + "GESTFIPS", +) +_OUTPUT_COLUMNS = tuple( + f"asec_{column}" for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) +_SHA256 = re.compile(r"[0-9a-f]{64}") +_CONTRACT_LABEL = Path("authenticated-v4-frame") + + +def _require_sha(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest.") + return value + + +@dataclass(frozen=True) +class AsecHouseholdObservationSource: + """Explicit raw household file binding to the checkpoint's income year.""" + + year: int + path: Path + sha256: str + + def __post_init__(self) -> None: + if type(self.year) is not int: + raise ValueError("ASEC household source income year must be an integer.") + _require_sha(self.sha256, "ASEC household source pin") + object.__setattr__(self, "path", Path(self.path)) + + +@dataclass(frozen=True) +class AsecHouseholdObservationResult: + """The augmented full source Frame and a reproducible, value-free receipt.""" + + frame: Frame + _receipt_json: str + + @property + def receipt(self) -> dict[str, object]: + """Return an independent JSON-safe copy of the immutable receipt.""" + return json.loads(self._receipt_json) + + +def _stat_identity(stat: os.stat_result) -> tuple[int, ...]: + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + + +def _hash_file(handle: BinaryIO) -> str: + handle.seek(0) + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def _hard_child(group: h5py.Group, name: str) -> h5py.Group | h5py.Dataset: + if not isinstance(group.get(name, getlink=True), h5py.HardLink): + raise ValueError(f"ASEC household HDF requires local hard-linked {name!r}.") + child = group[name] + if isinstance(child, h5py.Dataset) and (child.is_virtual or child.external): + raise ValueError("ASEC household HDF cannot read virtual or external storage.") + return child + + +def _dataset(group: h5py.Group, name: str) -> h5py.Dataset: + child = _hard_child(group, name) + if not isinstance(child, h5py.Dataset): + raise ValueError(f"ASEC household HDF {name!r} must be a dataset.") + return child + + +def _column_names(group: h5py.Group, name: str) -> tuple[str, ...]: + dataset = _dataset(group, name) + if dataset.ndim != 1 or dataset.dtype.kind != "S": + raise ValueError("ASEC household column metadata must be fixed-width strings.") + try: + values = tuple(value.decode("utf-8") for value in dataset[()]) + except UnicodeDecodeError as error: + raise ValueError("ASEC household column metadata must be UTF-8.") from error + if len(values) != len(set(values)) or any(not name for name in values): + raise ValueError("ASEC household column metadata is empty or duplicated.") + return values + + +def _read_numeric_household( + handle: BinaryIO, + *, + columns: Sequence[str] = ASEC_HOUSEHOLD_OBSERVATION_COLUMNS, + require_same_block: bool = False, +) -> pd.DataFrame: + """Read selected primitive observations without decoding any object block.""" + if ( + isinstance(columns, str) + or not columns + or any(not isinstance(name, str) or not name for name in columns) + or len(set(columns)) != len(columns) + ): + raise ValueError("ASEC household numeric projection requires unique names.") + requested = tuple(columns) + with h5py.File(handle, "r") as h5: + group = _hard_child(h5, "household") + if not isinstance(group, h5py.Group): + raise ValueError("ASEC household HDF lacks its household group.") + if group.attrs.get("pandas_type") != b"frame" or group.attrs.get("ndim") != 2: + raise ValueError( + "ASEC household HDF requires the pandas fixed frame format." + ) + nblocks = group.attrs.get("nblocks") + if ( + isinstance(nblocks, (bool, np.bool_)) + or not isinstance(nblocks, (int, np.integer)) + or nblocks < 1 + ): + raise ValueError("ASEC household HDF has an invalid block count.") + columns = _column_names(group, "axis0") + if set(requested) - set(columns): + raise ValueError("ASEC household HDF lacks required observed columns.") + axis1 = _dataset(group, "axis1") + if axis1.ndim != 1: + raise ValueError("ASEC household HDF row axis must be one-dimensional.") + rows = axis1.shape[0] + if not rows: + raise ValueError("ASEC household HDF has no household rows.") + all_items = [] + output = {} + for index in range(int(nblocks)): + names = _column_names(group, f"block{index}_items") + all_items.extend(names) + wanted = [i for i, name in enumerate(names) if name in requested] + if not wanted: + # Do not even open an unrelated object/VLARRAY value dataset. + continue + if require_same_block and output: + raise ValueError( + "ASEC household selected observations require one shared block." + ) + values = _dataset(group, f"block{index}_values") + if values.dtype != np.dtype(" pd.DataFrame: + with source.path.open("rb", buffering=0) as handle: + before = _stat_identity(os.fstat(handle.fileno())) + if ( + _stat_identity(source.path.stat()) != before + or _hash_file(handle) != source.sha256 + ): + raise ValueError(f"ASEC household source {source.year} SHA-256 mismatch.") + handle.seek(0) + table = _read_numeric_household(handle) + if ( + _hash_file(handle) != source.sha256 + or _stat_identity(os.fstat(handle.fileno())) != before + or _stat_identity(source.path.stat()) != before + ): + raise ValueError( + f"ASEC household source {source.year} changed during reading." + ) + if table.H_SEQ.duplicated().any(): + raise ValueError(f"ASEC household source {source.year} repeats H_SEQ.") + return table + + +def _int64_column(table: pd.DataFrame, name: str) -> np.ndarray: + if name not in table or table[name].dtype != np.dtype("int64"): + raise ValueError( + f"ASEC household join requires complete ordinary int64 {name}." + ) + return table[name].to_numpy(copy=False) + + +def _validate_input(frame: Frame, metadata: Mapping[str, object]) -> str: + if not isinstance(frame, Frame) or not isinstance(metadata, Mapping): + raise TypeError( + "ASEC household attachment requires an actual v4 Frame and metadata." + ) + asec_checkpoint._validate_raw_stage_binding( + metadata, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_asec_frame( + frame, path=_CONTRACT_LABEL, artifact_label="ASEC v4 source" + ) + assert_operator_free_source_frame(frame, label="ASEC household observation input") + asec_checkpoint._validate_raw_stage_source_columns( + frame, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_coverage_v4_binding(frame, metadata, path=_CONTRACT_LABEL) + actual = frame_identity(frame) + for key in ("identity", "source_construction_identity"): + if FrameIdentity.from_payload(metadata[key], label=key) != actual: + raise ValueError( + "ASEC household input structural identity differs from v4 metadata." + ) + if set(_OUTPUT_COLUMNS) & set(frame.table("household").columns): + raise ValueError( + "ASEC household observation attachment refuses existing output columns." + ) + return actual.sha256 + + +def _household_keys(frame: Frame) -> pd.DataFrame: + person, household = frame.person, frame.table("household") + person_keys = pd.DataFrame( + { + "household_id": _int64_column(person, "person_household_id"), + "source_year": _int64_column(person, "source_year"), + "H_SEQ": _int64_column(person, "source_household_id"), + } + ) + keys = person_keys.drop_duplicates() + if keys.household_id.duplicated().any(): + raise ValueError("ASEC persons disagree on the source key within a household.") + if keys.duplicated(["source_year", "H_SEQ"]).any(): + raise ValueError( + "ASEC raw source key is claimed by multiple normalized households." + ) + ids = _int64_column(household, "household_id") + if set(keys.household_id) != set(ids): + raise ValueError( + "ASEC household source keys do not cover actual household IDs." + ) + return keys.set_index("household_id").loc[ids].reset_index() + + +def with_asec_household_observations( + frame: Frame, + *, + checkpoint_metadata: Mapping[str, object], + checkpoint_sha256: str, + sources: Sequence[AsecHouseholdObservationSource], +) -> AsecHouseholdObservationResult: + """Augment the actual authenticated v4 source without changing its input. + + ``checkpoint_sha256`` records the caller's authenticated input-byte pin; this + in-memory helper cannot verify that file. Explicit household sources must + match all per-year pins in the loaded checkpoint receipt. Every raw source + is authenticated locally before and after numeric-only reading. Source codes + remain unclassified; nominal signed amounts remain exact int64 observations. + """ + checkpoint_sha256 = _require_sha(checkpoint_sha256, "Input checkpoint pin") + input_identity = _validate_input(frame, checkpoint_metadata) + declared = { + item["year"]: item["sha256"] + for item in checkpoint_metadata["source_receipt"]["sources"] + } + if any( + not isinstance(source, AsecHouseholdObservationSource) for source in sources + ): + raise TypeError("ASEC household sources require explicit typed file bindings.") + actual = {source.year: source.sha256 for source in sources} + if len(actual) != len(sources) or actual != declared: + raise ValueError("ASEC household source years/pins differ from the v4 receipt.") + keys = _household_keys(frame) + household = frame.table("household").copy() + _int64_column(household, "state_fips") + _int64_column(household, "H_TENURE") + outputs = { + name: np.empty(len(household), dtype=np.int64) for name in _OUTPUT_COLUMNS + } + source_receipts = [] + for source in sorted(sources, key=lambda value: value.year): + raw = _load_source(source) + positions = np.flatnonzero(keys.source_year.to_numpy() == source.year) + source_positions = pd.Index(raw.H_SEQ).get_indexer(keys.H_SEQ.iloc[positions]) + if (source_positions < 0).any(): + raise ValueError( + f"ASEC household source {source.year} lacks a referenced H_SEQ." + ) + joined = raw.iloc[source_positions] + for raw_name, existing_name in ( + ("H_TENURE", "H_TENURE"), + ("GESTFIPS", "state_fips"), + ): + if not np.array_equal( + joined[raw_name].to_numpy(), + household[existing_name].iloc[positions].to_numpy(), + ): + raise ValueError( + f"ASEC household joined {raw_name} disagrees with carried {existing_name}." + ) + for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS: + outputs[f"asec_{column}"][positions] = joined[column].to_numpy(copy=False) + source_receipts.append( + { + "income_year": source.year, + "sha256": source.sha256, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + for name, values in outputs.items(): + household[name] = values + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + augmented = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + assert_operator_free_source_frame( + augmented, label="ASEC household observation output" + ) + if frame_identity(augmented).sha256 != input_identity: + raise AssertionError("ASEC household attachment changed structural identity.") + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observation_attachment", + "input_checkpoint_sha256": checkpoint_sha256, + "input_structural_identity_sha256": input_identity, + "reader": "pandas_fixed_hdf_numeric_int64_v1", + "join_keys": ["income_year", "H_SEQ"], + "sources": source_receipts, + "household_identity": { + "rows": len(household), + "id_dtype": "int64", + "ordered_ids_sha256": hashlib.sha256( + household.household_id.to_numpy(dtype=" None: + if not condition: + raise HousingStatusRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + parsed = json.loads(value) + _require(type(parsed) is dict and _json(parsed) == value, "HEADER_CANONICAL") + return parsed + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingStatusRefusalError("HEADER_FORMAT") from None + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _codebook() -> dict: + return { + "status": { + "unknown": 0, + "receipt": 1, + "nonreceipt": 2, + "not_in_universe": 3, + "conflict": 4, + }, + "route": { + "unresolved": 0, + "public_housing": 1, + "lower_rent": 2, + "both_negative": 3, + "not_in_universe": 4, + }, + "group_quarters": {"not_gq": 0, "gq": 1, "unavailable": 2}, + "conflict_bits": { + "excluded_scope": 1, + "parent_route": 2, + "public_allocation": 4, + "lower_allocation": 8, + "context": 16, + }, + "unknown_reason_bits": {"parent_answer": 1, "lower_answer": 2, "scope": 4}, + "quality": { + "not_applicable": 0, + "no_allocation_code_origin_unresolved": 1, + "allocated": 2, + "conflict": 3, + }, + "validity": "1 iff substantive response; receipt_valid additionally requires consistent known receipt/nonreceipt", + "zero_origin": {"not_zero": 0, "frozen_zero_origin_unresolved": 1}, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_status.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + """Vectorized dictionary-domain checks and explicit interpretation axes.""" + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("HPUBLIC", 2), + ("HLORENT", 2), + ("I_HPUBLI", 1), + ("I_HLOREN", 1), + ): + _require(bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CODE_DOMAIN") + for name, maximum in (("H_TENURE", 3), ("HRHTYPE", 10), ("H_HHTYPE", 3)): + # Zero context remains unresolved under the frozen fill provenance. + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], [2022, 2023, 2024]).all()), "INCOME_YEAR") + _require(bool((raw["survey_year"] == raw["income_year"] + 1).all()), "SURVEY_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + keys = pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]) + _require(keys.is_unique, "DUPLICATE_NATIVE_KEY") + public, lower = raw["HPUBLIC"], raw["HLORENT"] + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + known = interview & np.isin(raw["H_TENURE"], [2, 3]) + excluded = noninterview | (interview & (raw["H_TENURE"] == 1)) + scope_unknown = ~(known | excluded) + context_conflict = (interview & (raw["HRHTYPE"] == 0)) | ( + noninterview & ((raw["HRHTYPE"] > 0) | (raw["H_TENURE"] > 0)) + ) + parent_conflict = (lower > 0) & (public != 2) + public_alloc_conflict = (raw["I_HPUBLI"] == 1) & ((public == 0) | excluded) + lower_alloc_conflict = (raw["I_HLOREN"] == 1) & ( + (lower == 0) | (public != 2) | excluded + ) + conflicts = ( + ((excluded & ((public > 0) | (lower > 0))).astype("uint8")) + | (parent_conflict.astype("uint8") * 2) + | (public_alloc_conflict.astype("uint8") * 4) + | (lower_alloc_conflict.astype("uint8") * 8) + | (context_conflict.astype("uint8") * 16) + ) + status = np.zeros(n, dtype="uint8") + route = np.zeros(n, dtype="uint8") + for mask, state, path in ( + (excluded, 3, 4), + (known & (public == 1) & (lower == 0), 1, 1), + (known & (public == 2) & (lower == 1), 1, 2), + (known & (public == 2) & (lower == 2), 2, 3), + ): + status[mask], route[mask] = state, path + status[conflicts > 0] = 4 + route[conflicts > 0] = 0 + unknown = (known & (public == 0)).astype("uint8") + unknown |= (known & (public == 2) & (lower == 0)).astype("uint8") * 2 + unknown |= scope_unknown.astype("uint8") * 4 + quality = [] + for answer, flag, invalid in ( + (public, raw["I_HPUBLI"], public_alloc_conflict), + (lower, raw["I_HLOREN"], lower_alloc_conflict), + ): + q = np.where(answer > 0, np.where(flag == 1, 2, 1), 0).astype("uint8") + q[invalid] = 3 + quality.append(q) + return { + "status": status, + "route": route, + "receipt_valid": np.isin(status, [1, 2]).astype("uint8"), + "group_quarters": np.where( + raw["HRHTYPE"] == 0, 2, np.isin(raw["HRHTYPE"], [9, 10]) + ).astype("uint8"), + "conflicts": conflicts, + "unknown_reasons": unknown, + "public_quality": quality[0], + "lower_quality": quality[1], + "HPUBLIC_valid": (public > 0).astype("uint8"), + "HLORENT_valid": (lower > 0).astype("uint8"), + **{ + name + "_zero_origin": (raw[name] == 0).astype("uint8") + for name in OBSERVED_COLUMNS + }, + } + + +@dataclass(frozen=True) +class _HousingStatus: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATUS_TOKEN, "STATUS_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + """Independent provenance copy; modifying it cannot change the artifact.""" + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + """Return an immutable byte-backed view in declared household order.""" + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recheck shape, actual classifier closure, and all derived evidence.""" + _require( + type(self) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + data = self.header_data + _require( + data["definition_sha256"] == _definition_identity(), "DEFINITION_CHANGED" + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if type(self) is AuthenticatedHousingStatus + else "synthetic_unverified" + ), + "STATUS_AUTHENTICATION", + ) + raw = {name: self.array(name) for name in RAW_COLUMNS} + derived = _derive(raw) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if type(self) is AuthenticatedHousingStatus: + from .asec_housing_status_source import _implementation + + _require( + data["implementation"] == _implementation(), "IMPLEMENTATION_CHANGED" + ) + + @property + def content_sha256(self) -> str: + """Complete header and raw/derived buffer identity, without a large join.""" + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-status-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingStatus(_HousingStatus): + """Invented observations; never accepted by the authenticated attachment.""" + + +class AuthenticatedHousingStatus(_HousingStatus): + """Only the fixed-cohort source loader may issue this status evidence.""" + + +def _issue( + raw: pd.DataFrame, *, source: dict | None = None, implementation: dict | None = None +) -> _HousingStatus: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + # Copy into immutable bytes before either validating values or deriving output. + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingStatus: + """Classify explicitly invented rows; this API never mints source authority.""" + return _issue(observations) + + +def _parts(status): + yield MAGIC + yield struct.pack(" bytes: + """Serialize primitive buffers with a bounded header and transport checksum.""" + _require( + type(status) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + status.validate() + parts = tuple(_parts(status)) + digest = hashlib.sha256() + for part in parts: + digest.update(part) + return b"".join((*parts, digest.digest())) + + +def decode_housing_status( + payload: bytes, *, expected: _HousingStatus +) -> _HousingStatus: + """Require independently issued COMPLETE expected content before replay authority.""" + _require( + type(expected) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "EXPECTED_STATUS", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" str: + """Write an artifact and return its complete-content identity.""" + Path(path).write_bytes(encode_housing_status(status)) + return status.content_sha256 + + +def read_housing_status( + path: str | Path, *, expected: _HousingStatus +) -> _HousingStatus: + """Bound disk reads and require independently source-issued expected content.""" + with Path(path).open("rb") as handle: + payload = handle.read(PAYLOAD_MAX_BYTES + 1) + return decode_housing_status(payload, expected=expected) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py new file mode 100644 index 000000000..eb0152e31 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py @@ -0,0 +1,338 @@ +"""Fixed-cohort numeric source authentication and tax-result composition. + +This module does not calculate housing benefits. Public source authority requires +an existing verified ASEC source and privately staged exact cohort bytes. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, Weights + +from . import asec_current_money_source as money_source +from . import asec_household_observations as household_reader +from . import asec_housing_status as status_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + OBSERVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingStatus, + HousingStatusRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +_READ_COLUMNS = ("H_SEQ", "H_YEAR") + CONTEXT_COLUMNS + OBSERVED_COLUMNS +ATTACHED_COLUMNS = tuple("asec_housing_observed_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Actual classifier, reader, source, Frame, codec and composition closure.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": status_module._definition_identity(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_housing_status_source.py", + "asec_housing_status.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_units.py", + ) + }, + } + + +def _read_cohort(path: Path) -> pd.DataFrame: + with path.open("rb") as handle: + return household_reader._read_numeric_household( + handle, columns=_READ_COLUMNS, require_same_block=True + ) + + +def load_authenticated_housing_status( + source: money_source.AuthenticatedCurrentMoneySource, + *, + cohort_paths: Mapping[int, str | Path], +) -> AuthenticatedHousingStatus: + """Join declared full cohort snapshots to the verified source's native keys. + + No caller pin override. This does not certify release or annual participation. + The source's sealed scope supplies ordered IDs/year/native keys; the freshly + authenticated cohort context must also equal the carried attachment context. + """ + try: + return _load(source, cohort_paths) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (OSError, ValueError, KeyError, TypeError, OverflowError): + raise HousingStatusRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(source, cohort_paths: Mapping[int, str | Path]): + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + before = _implementation() + evidence = _parse(source.source.identity) + pins = tuple((year, pin) for year, pin in evidence["cohorts"]) + _require( + isinstance(cohort_paths, Mapping) + and set(cohort_paths) == {year for year, _ in pins} + and all(type(year) is int for year in cohort_paths), + "COHORT_PATHS", + ) + # Scope tuples were sealed from the actual Frame by the source loader. + # They cannot borrow mutable pandas index/column backing arrays. + scope = source.scope + output = { + "household_id": np.asarray(scope.household_ids, dtype=np.int64), + "income_year": np.asarray(scope.household_years, dtype=np.int64), + "survey_year": np.empty(len(scope.household_ids), dtype=np.int64), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype=np.int64 + ), + **{ + name: np.empty(len(scope.household_ids), dtype=np.int64) + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS + }, + } + carried = ( + source.frame.table("household") + .loc[:, ["household_id"] + ["asec_" + n for n in CONTEXT_COLUMNS]] + .copy(deep=True) + ) + carried.index = carried.index.copy(deep=True) + carried.columns = carried.columns.copy(deep=True) + _require( + np.array_equal(carried.household_id.to_numpy(), output["household_id"]), + "HOUSEHOLD_ORDER", + ) + _require( + all(carried[name].dtype == np.dtype("int64") for name in carried), + "CARRIED_INT64", + ) + joins = [] + with tempfile.TemporaryDirectory(prefix="microcosm-housing-status-") as directory: + for year, pin in pins: + staged = Path(directory) / f"cohort-{year}.h5" + money_source._stage_verified(cohort_paths[year], pin, staged) + raw = _read_cohort(staged) + staged.unlink() + _require( + not raw.H_SEQ.duplicated().any() and bool((raw.H_SEQ > 0).all()), + "NATIVE_KEY", + ) + _require(bool((raw.H_YEAR == year + 1).all()), "SURVEY_YEAR") + positions = np.flatnonzero(output["income_year"] == year) + source_positions = pd.Index(raw.H_SEQ).get_indexer( + output["H_SEQ"][positions] + ) + _require(bool((source_positions >= 0).all()), "SOURCE_KEY_COVERAGE") + joined = raw.iloc[source_positions] + for name in CONTEXT_COLUMNS: + _require( + np.array_equal( + joined[name].to_numpy(), + carried["asec_" + name].iloc[positions].to_numpy(), + ), + "CARRIED_CONTEXT", + ) + output["survey_year"][positions] = joined.H_YEAR.to_numpy() + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS: + output[name][positions] = joined[name].to_numpy() + joins.append( + { + "income_year": year, + "sha256": pin, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + source.validate() + _require(_implementation() == before, "IMPLEMENTATION_CHANGED") + return status_module._issue( + pd.DataFrame(output, columns=RAW_COLUMNS), + source={"identity": source.source.identity.decode(), "joins": joins}, + implementation=before, + ) + + +def _owned_frame(frame: Frame) -> Frame: + # Frame copies native buffers; explicitly detach axes and weight vectors too. + # This is one complete result copy, not a partial/fabricated US population. + strata = frame.strata.copy(deep=True) + strata.index = money_source._owned_index(strata.index) + result = Frame( + { + **{entity: frame.table(entity) for entity in frame.entities}, + **{name: frame.link(name) for name in frame.links}, + }, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values, frame.weights_for(entity).kind + ) + for entity in frame.weighted_entities + }, + strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + money_source._detach_frame_axes(result) + return result + + +def _check_parent_status(parent, status): + _require(type(parent) is CurrentMoneyTaxUnitResult, "TAX_RESULT_REQUIRED") + _require(type(status) is AuthenticatedHousingStatus, "AUTHENTICATED_STATUS") + parent.validate() + status.validate() + _require( + status.header_data["source"]["identity"].encode() + == parent.money.bindings.spec.source.identity, + "STATUS_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + np.array_equal(household.household_id.to_numpy(), status.array("household_id")), + "STATUS_HOUSEHOLD_BINDING", + ) + for name in CONTEXT_COLUMNS: + _require( + np.array_equal(household["asec_" + name].to_numpy(), status.array(name)), + "STATUS_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult, + status: AuthenticatedHousingStatus, +) -> None: + """Verify a live or checkpoint-reloaded composition against both trusted parents.""" + try: + _check_parent_status(parent, status) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame(frame, label="ASEC housing status attachment") + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + _require( + household[attached].dtype == np.dtype("uint8") + and household[attached].to_numpy().tobytes() + == status.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + # Drop ONLY our columns, then compare the existing full Frame signature. + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingStatusAttachedAsec: + """Separate composed result retaining the identical sealed tax/money parents.""" + + frame: Frame + tax_result: CurrentMoneyTaxUnitResult + status: AuthenticatedHousingStatus + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.tax_result, status=self.status) + _require( + self.receipt["housing_content_sha256"] == self.status.content_sha256 + and self.receipt["tax_receipt_sha256"] + == _sha(_json(self.tax_result.receipt)) + and self.receipt["output_frame_sha256"] + == money_source._frame_signature(self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def attach_housing_status( + tax_result: CurrentMoneyTaxUnitResult, status: AuthenticatedHousingStatus +) -> HousingStatusAttachedAsec: + """Compose source-status evidence after actual tax reconstruction, without mutation.""" + try: + _check_parent_status(tax_result, status) + _require( + not set(ATTACHED_COLUMNS) & set(tax_result.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + result = _owned_frame(tax_result.frame) + _require( + money_source._frame_signature(result) + == tax_result.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, output in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + result.table("household")[output] = status.array(name).copy() + validate_attached_frame(result, parent=tax_result, status=status) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_status_attachment.v1", + "release_eligible": False, + "tax_receipt_sha256": _sha(_json(tax_result.receipt)), + "housing_content_sha256": status.content_sha256, + "parent_frame_sha256": tax_result.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(result), + "outputs": ATTACHED_COLUMNS, + } + return HousingStatusAttachedAsec( + result, tax_result, status, _json(receipt), _token=_COMPOSE_TOKEN + ) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py new file mode 100644 index 000000000..e67327f8d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py @@ -0,0 +1,423 @@ +"""Source-specific occupied-HU evidence; never a population or release verdict.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import struct +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from .asec_housing_status import DICTIONARIES, ZERO_ORIGIN_EVIDENCE, ZERO_POLICY + +RAW_COLUMNS = ( + "household_id", + "income_year", + "H_SEQ", + "H_HHTYPE", + "H_LIVQRT", + "HRHTYPE", + "H_TENURE", +) +CONTEXT_COLUMNS = ("H_HHTYPE", "H_LIVQRT", "HRHTYPE", "H_TENURE") +DERIVED_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", +) +COLUMNS = RAW_COLUMNS + DERIVED_COLUMNS +MAGIC = b"MCAHUNIV\x01" +HEADER_MAX_BYTES = 65536 +MAX_HOUSEHOLDS = 400_000 +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + + 4 + + HEADER_MAX_BYTES + + MAX_HOUSEHOLDS * (8 * len(RAW_COLUMNS) + len(DERIVED_COLUMNS)) + + 32 +) +_UNIVERSE_TOKEN = object() + + +class HousingUniverseRefusalError(ValueError): + """Sanitized reason for unavailable or inconsistent source evidence.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise HousingUniverseRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + result = json.loads(value) + _require(type(result) is dict and _json(result) == value, "HEADER_CANONICAL") + return result + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingUniverseRefusalError("HEADER_FORMAT") from None + + +def _codebook() -> dict: + return { + "interview_scope": { + "unavailable": 0, + "interview": 1, + "type_a_noninterview": 2, + "type_bc_noninterview": 3, + }, + "physical_unit": {"unavailable": 0, "housing_unit": 1, "other_unit": 2}, + "household_kind": { + "source_zero_or_not_established": 0, + "primary_household": 1, + "group_quarters": 2, + }, + "tenure_subtype": { + "niu_or_zero_unresolved": 0, + "owned_or_being_bought": 1, + "cash_renter": 2, + "no_cash_rent": 3, + }, + "occupied_hu": { + "not_established": 0, + "occupied_housing_unit": 1, + "outside_housing_unit": 2, + }, + "hu_tenure_class": { + "outside_or_unestablished_hu": 0, + "owner": 1, + "renter": 2, + "occupied_hu_unresolved_tenure": 3, + }, + "unresolved_reason_bits": { + "outside_interview_scope": 1, + "interview_zero": 2, + "quarters_zero": 4, + "household_kind_zero": 8, + "hu_quarters_gq_kind": 16, + "context_conflict": 32, + "hu_tenure_zero": 64, + "other_quarters_primary_kind": 128, + }, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_universe.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + set(raw) == set(RAW_COLUMNS) + and all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("H_HHTYPE", 3), + ("H_LIVQRT", 12), + ("HRHTYPE", 10), + ("H_TENURE", 3), + ): + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], (2022, 2023, 2024)).all()), "INCOME_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + _require( + pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]).is_unique, + "DUPLICATE_NATIVE_KEY", + ) + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + quarters, kind, tenure = raw["H_LIVQRT"], raw["HRHTYPE"], raw["H_TENURE"] + hu_quarters = (quarters >= 1) & (quarters <= 7) + other_quarters = quarters >= 8 + primary = (kind >= 1) & (kind <= 8) + gq = kind >= 9 + context_conflict = (interview & (kind == 0)) | ( + noninterview & ((kind > 0) | (tenure > 0)) + ) + hu = interview & hu_quarters & primary & ~context_conflict + outside = interview & other_quarters & gq & ~context_conflict + reasons = noninterview.astype("uint8") + for mask, bit in ( + (raw["H_HHTYPE"] == 0, 2), + (interview & (quarters == 0), 4), + (interview & (kind == 0), 8), + (interview & hu_quarters & gq, 16), + (context_conflict, 32), + (hu & (tenure == 0), 64), + (interview & other_quarters & primary, 128), + ): + reasons |= mask.astype("uint8") * bit + return { + "interview_scope": raw["H_HHTYPE"].astype("uint8"), + "physical_unit": np.where( + hu_quarters, 1, np.where(other_quarters, 2, 0) + ).astype("uint8"), + "household_kind": np.where(primary, 1, np.where(gq, 2, 0)).astype("uint8"), + "tenure_subtype": tenure.astype("uint8"), + "occupied_hu": np.where(hu, 1, np.where(outside, 2, 0)).astype("uint8"), + "hu_tenure_class": np.where( + hu, np.where(tenure == 1, 1, np.where(tenure >= 2, 2, 3)), 0 + ).astype("uint8"), + "unresolved_reasons": reasons, + } + + +@dataclass(frozen=True) +class _HousingUniverse: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _UNIVERSE_TOKEN, "UNIVERSE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recompute every derived code from immutable raw evidence.""" + _require( + type(self) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + data = self.header_data + try: + _require( + data["definition_sha256"] == _definition_identity(), + "DEFINITION_CHANGED", + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + authenticated = type(self) is AuthenticatedHousingUniverse + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if authenticated + else "synthetic_unverified" + ), + "UNIVERSE_AUTHENTICATION", + ) + derived = _derive({name: self.array(name) for name in RAW_COLUMNS}) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if authenticated: + from .asec_housing_universe_source import _implementation + + _require( + data["implementation"] == _implementation(), + "IMPLEMENTATION_CHANGED", + ) + except (KeyError, TypeError, OverflowError): + raise HousingUniverseRefusalError("UNIVERSE_CONTRACT") from None + + @property + def content_sha256(self) -> str: + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-universe-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingUniverse(_HousingUniverse): + """Invented observations, with no source authority.""" + + +class AuthenticatedHousingUniverse(_HousingUniverse): + """Issued only from an authenticated current-money source.""" + + +def _issue(raw: pd.DataFrame, *, source=None, implementation=None) -> _HousingUniverse: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + _require(0 < len(raw) <= MAX_HOUSEHOLDS, "ROW_COUNT") + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingUniverse: + """Classify invented rows without granting source authority.""" + return _issue(observations) + + +def _parts(value): + yield MAGIC + yield struct.pack(" bytes: + """Canonical primitive bytes with a bounded header and transport checksum.""" + _require( + type(value) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + value.validate() + payload = b"".join(_parts(value)) + return payload + hashlib.sha256(payload).digest() + + +def decode_housing_universe( + payload: bytes, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Only independently issued complete expected evidence confers replay authority.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + # Compare before parsing any candidate header or buffer; no alternate encoding. + _require(payload == encode_housing_universe(expected), "EXPECTED_CONTENT") + return type(expected)(expected.header, expected.buffers, _token=_UNIVERSE_TOKEN) + + +def write_housing_universe(value: _HousingUniverse, path: str | Path) -> str: + """Create an immutable artifact; never overwrite an existing path.""" + payload = encode_housing_universe(value) + try: + with Path(path).open("xb") as handle: + handle.write(payload) + except OSError: + raise HousingUniverseRefusalError("ARTIFACT_WRITE") from None + return value.content_sha256 + + +def read_housing_universe( + path: str | Path, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Bound regular-file reads; no FIFO, device or candidate header parsing.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + size = len(encode_housing_universe(expected)) + descriptor = None + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode) and info.st_size == size, "ARTIFACT_FILE") + with os.fdopen(descriptor, "rb") as handle: + descriptor = None + payload = handle.read(size + 1) + return decode_housing_universe(payload, expected=expected) + except (OSError, TypeError): + raise HousingUniverseRefusalError("ARTIFACT_READ") from None + finally: + if descriptor is not None: + os.close(descriptor) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py new file mode 100644 index 000000000..23deb4385 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py @@ -0,0 +1,336 @@ +"""Authenticate carried ASEC HU evidence and preserve the chosen parent chain.""" + +from __future__ import annotations + +from dataclasses import InitVar, dataclass +from importlib import resources + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as universe_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import HousingStatusRefusalError +from .asec_housing_status_source import HousingStatusAttachedAsec, _owned_frame +from .asec_housing_universe import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingUniverse, + HousingUniverseRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +ATTACHED_COLUMNS = tuple("asec_housing_universe_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Bind new interpretation/composition without editing any old verifier roster.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": universe_module._definition_identity(), + "housing_implementation": housing_source._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_housing_universe_source.py", "asec_housing_universe.py") + }, + } + + +def load_authenticated_housing_universe( + source: money_source.AuthenticatedCurrentMoneySource, +) -> AuthenticatedHousingUniverse: + """Read only the already-authenticated household attachment, with no new joins.""" + try: + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + implementation = _implementation() + scope = source.scope + table = source.frame.table("household") + columns = ("household_id",) + tuple( + "asec_" + name for name in ("H_SEQ",) + CONTEXT_COLUMNS + ) + _require( + table.columns.is_unique and set(columns) <= set(table), "CARRIED_COLUMNS" + ) + _require( + all(table[name].dtype == np.dtype("int64") for name in columns), + "CARRIED_INT64", + ) + raw = pd.DataFrame( + { + "household_id": np.asarray(scope.household_ids, dtype="int64"), + "income_year": np.asarray(scope.household_years, dtype="int64"), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype="int64" + ), + **{ + name: table["asec_" + name].to_numpy(copy=True) + for name in CONTEXT_COLUMNS + }, + }, + columns=RAW_COLUMNS, + ) + _require( + np.array_equal(table.household_id.to_numpy(), raw.household_id.to_numpy()), + "HOUSEHOLD_ORDER", + ) + _require( + np.array_equal(table.asec_H_SEQ.to_numpy(), raw.H_SEQ.to_numpy()), + "NATIVE_KEY_BINDING", + ) + result = universe_module._issue( + raw, + source={ + "identity": source.source.identity.decode(), + "carried_columns": columns, + "income_year_origin": "authenticated_money_scope", + }, + implementation=implementation, + ) + source.validate() + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, OverflowError, AssertionError, OSError): + raise HousingUniverseRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _tax_parent(parent) -> CurrentMoneyTaxUnitResult: + _require( + type(parent) in (CurrentMoneyTaxUnitResult, HousingStatusAttachedAsec), + "TAX_OR_HOUSING_PARENT_REQUIRED", + ) + return parent if type(parent) is CurrentMoneyTaxUnitResult else parent.tax_result + + +def _check_parent(parent, universe): + tax = _tax_parent(parent) + _require(type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE") + parent.validate() + universe.validate() + _require( + universe.header_data["source"]["identity"].encode() + == tax.money.bindings.spec.source.identity, + "UNIVERSE_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + household.household_id.dtype == np.dtype("int64") + and np.array_equal( + household.household_id.to_numpy(), universe.array("household_id") + ), + "UNIVERSE_HOUSEHOLD_BINDING", + ) + _require( + _parse(tax.money.header)["household_year_sha256"] + == _sha(universe.array("income_year").tobytes()), + "UNIVERSE_YEAR_BINDING", + ) + for name in ("H_SEQ",) + CONTEXT_COLUMNS: + column = household["asec_" + name] + _require( + column.dtype == np.dtype("int64") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "UNIVERSE_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> None: + """Check a whole owned parent extension; a selected child cannot inherit this authority.""" + try: + _check_parent(parent, universe) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame( + frame, label="ASEC housing universe attachment" + ) + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + column = household[attached] + _require( + column.dtype == np.dtype("uint8") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + universe.validate() + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingUniverseAttachedAsec: + """Owned HU extension retaining its exact tax or housing-status parent.""" + + frame: Frame + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec + universe: AuthenticatedHousingUniverse + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def tax_result(self): + return _tax_parent(self.parent) + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.parent, universe=self.universe) + _require( + self.receipt == _receipt(self.parent, self.universe, self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def _receipt(parent, universe, frame) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_universe_attachment.v1", + "release_eligible": False, + "parent_kind": type(parent).__name__, + "parent_receipt_sha256": _sha(_json(parent.receipt)), + "tax_receipt_sha256": _sha(_json(_tax_parent(parent).receipt)), + "money_header_sha256": _sha(parent.money.header), + "universe_content_sha256": universe.content_sha256, + "parent_frame_sha256": parent.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(frame), + "outputs": list(ATTACHED_COLUMNS), + } + + +def attach_housing_universe( + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> HousingUniverseAttachedAsec: + """Preserve all chosen parent columns, including student and subsidy evidence.""" + try: + _check_parent(parent, universe) + _require( + not set(ATTACHED_COLUMNS) & set(parent.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + frame = _owned_frame(parent.frame) + _require( + money_source._frame_signature(frame) + == parent.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, alias in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + frame.table("household")[alias] = universe.array(name).copy() + validate_attached_frame(frame, parent=parent, universe=universe) + receipt = _receipt(parent, universe, frame) + result = HousingUniverseAttachedAsec( + frame, parent, universe, _json(receipt), _token=_COMPOSE_TOKEN + ) + result.validate() + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +def verify_housing_universe_rows( + household_table: pd.DataFrame, + universe: AuthenticatedHousingUniverse, + *, + lineage_ids: np.ndarray, +) -> None: + """Verify source-derived cells of subsets/clones, without issuing Frame authority. + + ``lineage_ids`` are the caller's original source household IDs, repeated for + clones. Their custody/assembly provenance is a separate obligation. This + checks carried H_SEQ/context and derived values, not a selected Frame receipt. + """ + try: + _require( + type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE" + ) + universe.validate() + _require( + type(household_table) is pd.DataFrame and household_table.columns.is_unique, + "HOUSEHOLD_TABLE", + ) + _require( + type(lineage_ids) is np.ndarray + and lineage_ids.dtype == np.dtype("int64") + and lineage_ids.shape == (len(household_table),), + "LINEAGE_INT64", + ) + positions = pd.Index(universe.array("household_id")).get_indexer(lineage_ids) + _require(bool((positions >= 0).all()), "LINEAGE_COVERAGE") + if "household_source_id" in household_table: + carried_ids = household_table.household_source_id + _require( + carried_ids.dtype == np.dtype("int64") + and np.array_equal(carried_ids.to_numpy(), lineage_ids), + "LINEAGE_BINDING", + ) + for name, column, dtype in ( + *((n, "asec_" + n, "int64") for n in ("H_SEQ",) + CONTEXT_COLUMNS), + *( + (n, c, "uint8") + for n, c in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True) + ), + ): + values = household_table[column] + _require( + values.dtype == np.dtype(dtype) + and np.array_equal(values.to_numpy(), universe.array(name)[positions]), + "ROW_EVIDENCE", + ) + universe.validate() + except HousingUniverseRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ROW_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py new file mode 100644 index 000000000..0c0283d17 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py @@ -0,0 +1,574 @@ +"""Closed PAW source observations, independent of the sealed 33-field money body. + +Only exact source reconstruction issues this authority. Candidate replay compares +canonical expected bytes before parsing any candidate. No Frame columns are added. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESOURCE_PINS, + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCAINCOB\x01" +FILENAME = "asec_income_observations.bin" +ARTIFACT_KIND = "microcosm.us.asec_income_observations.v1" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +INCOME_PAYLOAD_MAX_BYTES = len(MAGIC) + 4 + _HEADER_MAX + 8 * 8 * _MAX_PERSONS + 32 +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + ("PAW_YN", "PAW_VAL", "PAW_VAL_2024_price") +COLUMN_DTYPES = { + name: "little_endian_float64" if name == COLUMNS[-1] else "little_endian_int64" + for name in COLUMNS +} +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PAW_VAL", "PAW_YN") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +RESOURCE = "asec_income_observations_v1.json" +RESOURCE_SHA256 = "6e37855ddd6642d02231efc664644029ed6131885eb743a00197261bc5a6a51f" +ZERO_CLASSES = ( + "observed_amount", + "niu_below_receipt_age", + "reported_receipt_zero_amount", + "reported_no_receipt", + "niu_in_age_universe", +) +_TOKEN = object() +_CSV_DIALECT = "excel" +_CSV_ENCODING = "utf-8" +_PRICE_TARGET = "174.4" +_PRICE_YEARS = (2022, 2023, 2024) + + +def parser_profile() -> dict: + """Read the effective CSV parser limits and dialect, without changing them. + + ``csv.field_size_limit`` is process state that silently changes which member + files parse. Binding it into the source identity makes any change to it a + different artifact identity instead of an invisible reparse. + """ + # _read_member uses csv.reader's built-in default, which is independent of + # the mutable named-dialect registry. Inspect that same reader's dialect. + dialect = csv.reader(()).dialect + return { + "header_engine": "stdlib_csv_reader", + "table_engine": "pandas_read_csv", + "dialect": _CSV_DIALECT, + "encoding": _CSV_ENCODING, + "newline": "", + "delimiter": dialect.delimiter, + "quotechar": dialect.quotechar, + "doublequote": bool(dialect.doublequote), + "skipinitialspace": bool(dialect.skipinitialspace), + "strict": bool(dialect.strict), + "quoting": int(dialect.quoting), + "csv.field_size_limit": csv.field_size_limit(), + } + + +def price_factors(contract) -> tuple[tuple[int, str, str], ...]: + """The single restatement-factor derivation, shared with graph transport.""" + return tuple( + ( + year, + f"{_PRICE_TARGET}/{contract['price']['cells'][str(year)]}", + struct.pack( + " 0).all()) + and bool((table.A_LINENO > 0).all()), + "INCOME_MEMBER_COORDINATES", + ) + _validate_observations( + table.A_AGE.to_numpy(), table.PAW_YN.to_numpy(), table.PAW_VAL.to_numpy() + ) + return table + + +def _validate_observations( + age: np.ndarray, receipt_code: np.ndarray, nominal: np.ndarray +): + _require( + all( + type(a) is np.ndarray + and a.dtype == np.dtype("int64") + and a.ndim == 1 + and a.shape == nominal.shape + for a in (age, receipt_code, nominal) + ), + "INCOME_OBSERVATION_ARRAYS", + ) + _require( + bool(((nominal >= 0) & (nominal <= 99999)).all()), + "INCOME_CODE_DOMAIN", + "PAW_VAL", + ) + _require( + bool(np.isin(receipt_code, [0, 1, 2]).all()), "INCOME_CODE_DOMAIN", "PAW_YN" + ) + _require( + bool((receipt_code[nominal != 0] == 1).all()) + and bool((nominal[receipt_code != 1] == 0).all()) + and bool((receipt_code[age < 15] == 0).all()) + and bool((nominal[age < 15] == 0).all()), + "INCOME_SOURCE_UNIVERSE", + ) + + +def _restate_amounts(nominal, years, factors): + """The single PAW restatement path, also used to verify graph transport.""" + amounts = nominal.astype("= 15) & (receipt_code == 1)] = 2 + result[zero & (age >= 15) & (receipt_code == 2)] = 3 + result[zero & (age >= 15) & (receipt_code == 0)] = 4 + return result + + +@dataclass(frozen=True) +class AuthenticatedIncomeObservations: + """Immutable source authority; only the complete reconstructed join issues it.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "INCOME_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "INCOME_HEADER_SIZE", + ) + header = self.receipt + _require( + set(header) == _HEADER_KEYS and _json(header) == self._header, + "INCOME_HEADER_SCHEMA", + ) + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "INCOME_ROWS") + _require( + header["schema_version"] == 1 + and header["artifact_kind"] == ARTIFACT_KIND + and type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["column_dtypes"] == COLUMN_DTYPES + and _json(header["implementation"]) == _json(_implementation()) + and header["column_sha256"] + == { + name: _sha(self._body[i * rows * 8 : (i + 1) * rows * 8]) + for i, name in enumerate(COLUMNS) + }, + "INCOME_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "INCOME_COLUMN") + n = self.receipt["rows"] + return np.frombuffer( + self._body, + dtype="= 0).all()) and len(np.unique(indices)) == rows, + "INCOME_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "INCOME_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PAW_VAL.iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined.PAW_VAL.to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "INCOME_INCUMBENT_CONFLICT", + "PAW_VAL", + ) + for name in ("PAW_YN", "PAW_VAL"): + output[name][positions] = joined[name].to_numpy(dtype=np.int64) + classes = zero_origin_classes( + output["A_AGE"][positions], output["PAW_YN"][positions], observed + ) + counts = { + name: int((classes == i).sum()) for i, name in enumerate(ZERO_CLASSES) + } + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": {"PAW_VAL": int(known.sum())}, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + "zero_origin_counts": counts, + "paw_yn_zero_at_age_15_plus": counts["niu_in_age_universe"], + } + ) + output["PAW_VAL_2024_price"] = _restate_amounts( + output["PAW_VAL"], years, ready.bindings.spec.factors + ) + _parent(source, ready) + _require(_json(_implementation()) == _json(before), "INCOME_IMPLEMENTATION_CHANGED") + buffers = [ + output[name] + .astype(" None: + if not condition: + raise HouseholdWeightSourceError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _issue_capsule(source: object, payload: bytes) -> str: + """Seal this identity outside writable capsule state without retaining it.""" + _require( + type(payload) is bytes and len(payload) <= _MAX_PAYLOAD_BYTES, "SOURCE_BYTES" + ) + digest = _sha(payload) + identity = id(source) + + def discard(reference): + entry = _CAPSULE_ISSUANCE.get(identity) + if entry is not None and entry[0] is reference: + del _CAPSULE_ISSUANCE[identity] + + # WeakKeyDictionary would merge distinct, value-equal frozen dataclasses. + _CAPSULE_ISSUANCE[identity] = (weakref.ref(source, discard), digest) + return digest + + +def _checked_capsule_payload(source: object) -> bytes: + """Return exactly the checked bytes; copies and deserialization are unissued.""" + entry = _CAPSULE_ISSUANCE.get(id(source)) + _require(entry is not None and entry[0]() is source, "SOURCE_MUTATION") + payload = getattr(source, "payload", None) + _require( + type(payload) is bytes + and len(payload) <= _MAX_PAYLOAD_BYTES + and _sha(payload) == entry[1], + "SOURCE_MUTATION", + ) + return payload + + +def _member_path_snapshot( + member_paths: Mapping[int, str | Path], + pins: tuple[_origin.AsecNativeMemberPin, ...], +) -> dict[int, Path]: + """Bound the closed cohort snapshot before allocation and each value lookup.""" + _require(isinstance(member_paths, Mapping), "MEMBER_PATHS") + try: + count = len(member_paths) + _require(0 < count <= len(pins), "MEMBER_PATHS") + allowed = {pin.income_year for pin in pins} + paths = {} + for year in member_paths: + _require( + len(paths) < count + and type(year) is int + and year in allowed + and year not in paths, + "MEMBER_PATHS", + ) + path = member_paths[year] + _require(isinstance(path, (str, Path)), "MEMBER_PATHS") + paths[year] = Path(path) + _require(len(paths) == count == len(member_paths), "MEMBER_PATHS") + except (TypeError, ValueError, RuntimeError, KeyError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_PATHS") from None + return paths + + +def _encode(value: object) -> bytes: + """Canonical JSON with a bound during encoding, not after a huge allocation.""" + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + result = bytearray() + for fragment in encoder.iterencode(value): + encoded = fragment.encode("ascii") + _require(len(result) + len(encoded) <= _MAX_PAYLOAD_BYTES, "ENCODING_SIZE") + result.extend(encoded) + return bytes(result) + + +def _registry() -> tuple[_origin.AsecNativeMemberPin, ...]: + pins = tuple(_origin._ASEC_MEMBER_PINS) + _require(0 < len(pins) <= 3, "MEMBER_REGISTRY") + try: + for pin in pins: + _origin.AsecNativeMemberPin(**asdict(pin)) + _require( + pin.income_year in (2022, 2023, 2024) + and pin.size_bytes <= _MAX_MEMBER_BYTES + and pin.rows <= _MAX_ROWS, + "MEMBER_REGISTRY", + ) + _require( + len({p.income_year for p in pins}) == len(pins) + and len({p.canonical_member_id for p in pins}) == len(pins), + "MEMBER_REGISTRY", + ) + except (TypeError, ValueError, AttributeError): + raise HouseholdWeightSourceError("MEMBER_REGISTRY") from None + return tuple(sorted(pins, key=lambda pin: pin.income_year)) + + +def _implementation() -> dict: + """Bind the live owner, registry, private capture, codec and field policy.""" + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "schema": _SCHEMA, + "modules": { + name: _sha(resources.files(__package__).joinpath(name).read_bytes()) + for name in ( + "asec_original_household_weights.py", + "source_csv_builtin.py", + "native_household_origin.py", + "asec_student_controls.py", + "education_assistance_source.py", + ) + }, + "origin_codec_sha256": _sha( + resources.files("microcosm.build.cd_benchmark") + .joinpath("origin.py") + .read_bytes() + ), + "python": sys.version, + "csv_python_sha256": _sha(Path(csv.__file__).read_bytes()), + "csv_native_origin": _csv.__spec__.origin, + "csv_native_provider": Path(parser_binary).name, + "csv_native_sha256": _sha(Path(parser_binary).read_bytes()), + "registry": [asdict(pin) for pin in _registry()], + "fields": _FIELD_CONTRACT, + "dictionaries": _DICTIONARIES, + "limits": [ + _MAX_MEMBER_BYTES, + _MAX_ROWS, + _MAX_RECORD_CHARS, + _MAX_FIELD_CHARS, + _MAX_PAYLOAD_BYTES, + ], + } + + +class _DigestReader(io.RawIOBase): + """Bound and hash the actual captured bytes consumed by UTF-8 decoding.""" + + def __init__(self, raw: io.FileIO, size: int): + self.raw = raw + self.size = size + self.count = 0 + self.digest = hashlib.sha256() + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: + limit = min(len(buffer), self.size - self.count + 1) + count = self.raw.readinto(memoryview(buffer)[:limit]) + self.count += count + _require(self.count <= self.size, "CAPTURE_SIZE") + self.digest.update(memoryview(buffer)[:count]) + return count + + +class _RecordLines: + """Limit the complete logical CSV record, including embedded newlines.""" + + def __init__(self, text: io.TextIOWrapper): + self.text = text + self.characters = 0 + + def __iter__(self): + return self + + def __next__(self) -> str: + line = self.text.readline(_MAX_RECORD_CHARS - self.characters + 1) + if not line: + raise StopIteration + self.characters += len(line) + _require(self.characters <= _MAX_RECORD_CHARS, "MEMBER_RECORD_SIZE") + return line + + +def _record_fields(tokens: Sequence[str]) -> dict: + native_token, household_type, weight_token = tokens + _require(all(len(t) <= _MAX_FIELD_CHARS for t in tokens), "MEMBER_FIELD_SIZE") + _require( + re.fullmatch(r"[0-9]+", native_token) is not None + and 1 <= int(native_token) <= 99999, + "MEMBER_NATIVE_KEY", + ) + if household_type == "1": + universe = "in" + elif re.fullmatch(r"(?:0|[2-9])", household_type): + # Establish only that the printed weight predicate is false, not the + # meaning/validity of any other household classification. + universe = "outside" + else: + universe = "unresolved" + units = None + if weight_token == "": + state = "missing" + elif re.fullmatch(r"[0-9]+", weight_token) is None: + state = "malformed" + else: + units = int(weight_token) + state = "integer" if units <= 999999999 else "out_of_range" + return { + "H_SEQ": native_token, + "native_household_id": int(native_token), + "H_HHTYPE": household_type, + "HSUP_WGT": weight_token, + "universe_status": universe, + "weight_token_status": state, + "weight_integer_units": units, + "weight_denominator": 100 if units is not None else None, + "weight_status": ( + "valid_in_universe" + if universe == "in" and state == "integer" + else "unresolved_weight" + if universe == "in" + else "outside_universe" + if universe == "outside" + else "unresolved_universe" + ), + } + + +def _regular_opener(path, flags): + fd = os.open(path, flags | os.O_NONBLOCK) + try: + _require(stat.S_ISREG(os.fstat(fd).st_mode), "CAPTURE_KIND") + except BaseException: + os.close(fd) + raise + return fd + + +def _read_capture(capture: Path, pin: _origin.AsecNativeMemberPin) -> list[dict]: + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + result, seen = [], set() + with open(capture, "rb", buffering=0, opener=_regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest_reader = _DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest_reader), encoding="utf-8-sig", newline="" + ) as text: + lines = _RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in _COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _record_fields([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest_reader.count == pin.size_bytes + and digest_reader.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, attribute) == getattr(after, attribute) + for attribute in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdWeights: + """Reader-issued immutable source evidence, not a population-weight vector.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _issue_capsule(self, self.payload)) + + @property + def document(self) -> dict: + """Fresh defensive decoding, checked against the live source producer.""" + return verify_asec_household_weights_source(self) + + def original_weight_fractions( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[Fraction, ...]: + """Exact weights for known in-universe keys; no Frame binding or stamping.""" + doc = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= _MAX_ROWS, + "ANCHOR_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "ANCHOR_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "ANCHOR_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in doc["records"] + } + values = [] + for key in keys: + _require(key in records, "ANCHOR_UNKNOWN") + row = records[key] + _require(row["weight_status"] == "valid_in_universe", "ANCHOR_UNRESOLVED") + values.append(Fraction(row["weight_integer_units"], 100)) + return tuple(values) + + +def verify_asec_household_weights_source( + source: AuthenticatedAsecHouseholdWeights, +) -> dict: + """Validate reader-issued evidence; arbitrary JSON/digests cannot issue it.""" + _require(type(source) is AuthenticatedAsecHouseholdWeights, "SOURCE_TYPE") + payload = _checked_capsule_payload(source) + doc = json.loads(payload) + _require(_encode(doc["producer"]) == _encode(_implementation()), "PRODUCER_CHANGED") + return doc + + +def load_authenticated_asec_household_weights( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdWeights: + """Reconstruct complete requested cohorts from closed original member pins. + + Mapping keys are income/source years, not survey years. Any nonempty subset + of the closed cohorts is explicit; every requested original member is read + completely. Optional candidate bytes must equal the independent canonical + reconstruction. There is no public pin override or candidate-only admission. + """ + pins = _registry() + paths = _member_path_snapshot(member_paths, pins) + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_PAYLOAD_BYTES), + "CANDIDATE_SIZE", + ) + producer = _implementation() + _require(producer["registry"] == [asdict(pin) for pin in pins], "PRODUCER_CHANGED") + implementation = _encode(producer) + selected = [pin for pin in pins if pin.income_year in paths] + records = [] + with tempfile.TemporaryDirectory(prefix="asec-original-household-weights-") as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = _capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdWeightSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except HouseholdWeightSourceError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_ENCODING") from None + _require(_encode(_implementation()) == implementation, "PRODUCER_CHANGED") + payload = _encode( + { + "schema": _SCHEMA, + "producer": json.loads(implementation), + "members": [asdict(pin) for pin in selected], + "fields": _FIELD_CONTRACT, + "dictionary_authorities": [ + { + "survey_year": year, + "pdf_page_1based": page, + "printed_page": "6A-1", + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, page, digest in _DICTIONARIES + ], + "native_roster_sha256": _sha( + _encode([[r["income_year"], r["native_household_id"]] for r in records]) + ), + "projection_sha256": _sha(_encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "period_and_coverage_policy": "not_selected_by_this_source_owner", + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdWeights(payload, _token=_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py new file mode 100644 index 000000000..eef9b20ae --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py @@ -0,0 +1,299 @@ +"""Bounded, non-authoritative original-CSV ASEC PRPERTYP projection. + +Only invented cohorts have been exercised. This reader neither authenticates +source bytes nor attaches observations to a Frame. Genuine use requires a +separately reviewed authentication/binding wrapper and source-read approval. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import stat +from collections.abc import Mapping +from pathlib import Path + +import pandas as pd + +from . import source_csv_builtin as _csv_builtin + +PROTOCOL = "microcosm.asec_person_coverage_source.v1" +# Existing ASEC adapters use income/source year as the cohort key. This is an +# identity convention only, not a decision about the observation's period. +SOURCE_YEARS = (2022, 2023, 2024) +KEYS = ("source_year", "PERIDNUM") +CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +ROSTER_COLUMNS = KEYS + CROSSCHECKS +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PRPERTYP") +MAX_PERSONS = 600_000 +MAX_MEMBER_BYTES = 1_000_000_000 +_INT64_MAX = 2**63 - 1 + + +class CoverageSourceRefusalError(ValueError): + """Static refusal codes only; no path, key, row or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CoverageSourceRefusalError(reason) + + +def coverage_field_contract() -> dict: + """Fresh metadata from the public dictionaries reviewed on 2026-09-07. + + The review's receipt.json and field-pages.json establish these definitions; + their document hashes are metadata, not original CSV authentication pins. + """ + documents = ( + (2023, 26, "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117"), + (2024, 27, "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840"), + (2025, 28, "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f"), + ) + return { + "protocol": PROTOCOL, + "survey": "CPS ASEC", + "grain": "person", + "field": "PRPERTYP", + "concept_as_printed": "Type of person record recode", + "universe_as_printed": "All Persons", + "printed_length": 1, + "printed_position": 155, + "printed_range": [-4, 3], + "codes": { + "1": "Child household member", + "2": "Adult civilian household member", + "3": "Adult Armed Forces household member", + }, + "unlabelled_in_range_codes": [-4, -3, -2, -1, 0], + "blank_meaning": "unresolved", + "authority": [ + { + "survey_year": year, + "source_year": year - 1, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + "sha256": digest, + "pdf_page_1based": page, + "printed_page": "6C-7", + } + for year, page, digest in documents + ], + "join_keys": list(KEYS), + "crosschecks": dict(zip(CROSSCHECKS, READ_COLUMNS[1:4], strict=True)), + "roster_scope": "all_person_rows_in_each_of_the_three_source_cohorts", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + } + + +def _state(token: str) -> str: + if token in ("1", "2", "3"): + return "observed_code" + if token == "": + return "blank_unresolved" + if token in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", token) and token != "-0": + return "out_of_range" + return "malformed_token" + + +def _roster(person_roster: pd.DataFrame) -> pd.DataFrame: + _require( + isinstance(person_roster, pd.DataFrame) + and person_roster.columns.is_unique + and set(ROSTER_COLUMNS) <= set(person_roster.columns), + "ROSTER_COLUMNS", + ) + _require(0 < len(person_roster) <= MAX_PERSONS, "ROSTER_ROWS") + result = person_roster.loc[:, list(ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require(not result.isna().any().any(), "ROSTER_MISSING") + _require( + all(str(result[name].dtype) == "int64" for name in (KEYS[0], *CROSSCHECKS)), + "ROSTER_INTEGER_DTYPE", + ) + _require(set(result.source_year) == set(SOURCE_YEARS), "ROSTER_COHORTS") + _require( + result.PERIDNUM.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(), + "ROSTER_PERSON_KEY", + ) + _require( + (result.source_household_id > 0).all() + and (result.A_LINENO > 0).all() + and (result.A_AGE >= 0).all(), + "ROSTER_COORDINATES", + ) + _require(not result.duplicated(list(KEYS)).any(), "ROSTER_DUPLICATE_KEY") + _require( + not result.duplicated([KEYS[0], *CROSSCHECKS[:2]]).any(), + "ROSTER_DUPLICATE_COORDINATES", + ) + return result + + +def _coordinate(token: str) -> int: + _require(re.fullmatch(r"[0-9]{1,19}", token) is not None, "MEMBER_INTEGER") + number = int(token) + _require(number <= _INT64_MAX, "MEMBER_INTEGER") + return number + + +def _digest(table: pd.DataFrame) -> str: + # Stream canonical records into the hash; never log native values or retain + # a second full JSON copy. Row order is part of the projection identity. + digest = hashlib.sha256() + digest.update(json.dumps(list(table.columns), separators=(",", ":")).encode()) + for row in table.itertuples(index=False, name=None): + digest.update(b"\n") + digest.update(json.dumps(row, separators=(",", ":"), allow_nan=False).encode()) + return digest.hexdigest() + + +def _regular_member_opener(path, flags): + # Check the opened descriptor, not a race-prone path snapshot. Nonblocking + # open lets us refuse a FIFO even when no writer has opened its other end. + descriptor = os.open(path, flags | os.O_NONBLOCK) + try: + snapshot = os.fstat(descriptor) + _require(stat.S_ISREG(snapshot.st_mode), "MEMBER_FILE_TYPE") + _require(snapshot.st_size <= MAX_MEMBER_BYTES, "MEMBER_BYTES") + except BaseException: + os.close(descriptor) + raise + return descriptor + + +class _BoundedMember(io.RawIOBase): + """Bound raw reads beneath buffering/decoding; the caller owns the raw file.""" + + def __init__(self, raw): + super().__init__() + self._raw = raw + self._remaining = MAX_MEMBER_BYTES + + def readable(self): + return True + + def readinto(self, buffer): + # One excess byte distinguishes exact-limit EOF from a growing member. + # Never pass that probe byte to the decoder or issue an unbounded read. + count = self._raw.readinto(memoryview(buffer)[: self._remaining + 1]) + self._remaining -= count + _require(self._remaining >= 0, "MEMBER_BYTES") + return count + + +def read_asec_person_coverage_source( + member_paths: Mapping[int, str | Path], *, person_roster: pd.DataFrame +) -> tuple[pd.DataFrame, dict]: + """Read exact whole cohorts, returning literals in requested native order. + + Paths are keyed by source/income years 2022, 2023, 2024 (survey years + 2023, 2024, 2025), following the existing ASEC source adapters. The caller + supplies PERIDNUM and int64 source_year/source_household_id/A_LINENO/A_AGE. + Every CSV row must match exactly one requested person in its own cohort; + PH_SEQ, A_LINENO and A_AGE crosscheck the requested native coordinates. + Age is only a crosscheck: it never supplies or overrides PRPERTYP meaning. + + All PRPERTYP cells remain strings, including blanks and malformed tokens. + The state column describes lexical/printed-code evidence, not coverage or + eligibility. Receipt digests bind this local projection, not source custody. + """ + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == set(SOURCE_YEARS) + and all(type(year) is int for year in member_paths) + and all(isinstance(path, (str, Path)) for path in member_paths.values()), + "MEMBER_PATHS", + ) + expected = _roster(person_roster) + lookup = { + row[:2]: (position, row[2:]) + for position, row in enumerate(expected.itertuples(index=False, name=None)) + } + tokens = [""] * len(expected) + seen = set() + cohort_counts = [] + for year in SOURCE_YEARS: + coordinates = set() + rows = 0 + try: + with ( + open( + member_paths[year], + "rb", + buffering=0, + opener=_regular_member_opener, + ) as raw, + io.BufferedReader(_BoundedMember(raw)) as bounded, + io.TextIOWrapper(bounded, encoding="utf-8", newline="") as handle, + ): + reader = csv_reader(handle, strict=True) + header = next(reader, []) + _require( + len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + indices = [header.index(name) for name in READ_COLUMNS] + for cells in reader: + rows += 1 + _require(len(seen) < MAX_PERSONS, "MEMBER_ROWS") + _require(len(cells) == len(header), "MEMBER_ROW_WIDTH") + key, household, line, age, token = (cells[i] for i in indices) + _require( + re.fullmatch(r"[0-9]{22}", key) is not None, "MEMBER_PERSON_KEY" + ) + native_key = (year, key) + _require(native_key not in seen, "MEMBER_DUPLICATE_KEY") + _require(native_key in lookup, "MEMBER_EXTRA_KEY") + actual = tuple( + _coordinate(value) for value in (household, line, age) + ) + _require(actual[0] > 0 and actual[1] > 0, "MEMBER_COORDINATES") + _require( + actual[:2] not in coordinates, "MEMBER_DUPLICATE_COORDINATES" + ) + position, wanted = lookup[native_key] + _require(actual == wanted, "MEMBER_CROSSCHECK") + coordinates.add(actual[:2]) + seen.add(native_key) + tokens[position] = token + except CoverageSourceRefusalError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise CoverageSourceRefusalError("MEMBER_READ") from None + cohort_counts.append( + {"source_year": year, "survey_year": year + 1, "rows": rows} + ) + _require(len(seen) == len(expected), "MEMBER_MISSING_KEY") + result = expected.copy() + result["PRPERTYP"] = pd.array(tokens, dtype="string") + result["PRPERTYP_state"] = pd.array( + [_state(token) for token in tokens], dtype="string" + ) + receipt = { + "protocol": PROTOCOL, + "source_authenticated": False, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "contract": coverage_field_contract(), + "read_columns": list(READ_COLUMNS), + "cohorts": cohort_counts, + "person_rows": len(result), + "native_roster_sha256": _digest(expected), + "projection_sha256": _digest(result), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py new file mode 100644 index 000000000..245a8caef --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py @@ -0,0 +1,488 @@ +"""Restore an observed person total from closed, authenticated Census members. + +P (v4) and H (household attachment) remain immutable. T appends one nominal +observation. A candidate T digest is a locator identity, never source authority: +verification reconstructs all of T from authenticated parents and CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.frame import Frame, Weights + +from . import asec_current_money as money_schema +from . import asec_current_money_source as legacy +from .asec_current_money import MoneyRefusalError, _json, _require, _sha +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CHECKPOINT_FILENAME = "asec_person_income_source.checkpoint.h5" +ARTIFACT_KIND = "microcosm.asec_person_income_observation_attachment" +OBSERVED_COLUMN = "asec_PTOTVAL" +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PTOTVAL") +# Closed code-owned source pins. No public caller pin authority exists. +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_ATTACHMENT_MAX_BYTES = 2_000_000_000 +_ENCODING_CONTRACT = "independently_reconstructed_canonical_checkpoint_bytes_v1" + + +def _implementation() -> str: + return _sha( + _json( + { + "schema": 1, + "legacy_verification": legacy._verification_identity(), + "schema_module": _sha( + resources.files(__package__) + .joinpath("asec_current_money.py") + .read_bytes() + ), + "source_kind": money_schema.RESTORED_SOURCE_KIND, + "field_columns": dict(money_schema.RESTORED_FIELD_COLUMNS), + "field_zero_policy": dict(money_schema.RESTORED_FIELD_ZERO_POLICY), + "reader_producer": _sha( + resources.files(__package__) + .joinpath("asec_person_income_source.py") + .read_bytes() + ), + "member_pins": _MEMBER_PINS, + "columns": _READ_COLUMNS, + "output": OBSERVED_COLUMN, + "origin": "authenticated_census_csv_encoded_zero_v1", + "encoding_contract": _ENCODING_CONTRACT, + } + ) + ) + + +def _paths(member_paths): + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(y) is int for y in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "MEMBER_PATHS", + ) + + +def _read_member(path: Path, *, rows: int, size: int) -> pd.DataFrame: + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + # Preserve source tokens until validation. In particular, never fillna and + # never let permissive numeric coercion turn absent observations into zeros. + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for column in _READ_COLUMNS[1:]: + _require( + bool(table[column].str.fullmatch(r"-?[0-9]+").all()), + "MEMBER_INTEGER" if column == "PTOTVAL" else f"MEMBER_INTEGER_{column}", + column, + ) + table[column] = table[column].astype("int64") + _require( + bool(table.PTOTVAL.between(-99999, 99999999).all()), + "MEMBER_AMOUNT_DOMAIN", + "PTOTVAL", + ) + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any(), + "MEMBER_DUPLICATE_KEY", + ) + return table + + +def _owned_frame(parent: Frame) -> Frame: + strata = parent.strata.copy(deep=True) + strata.index = legacy._owned_index(strata.index) + result = Frame( + { + **{e: parent.table(e) for e in parent.entities}, + **{n: parent.link(n) for n in parent.links}, + }, + parent.schema, + { + e: Weights(parent.weights_for(e).values, parent.weights_for(e).kind) + for e in parent.weighted_entities + }, + strata, + mass_log=parent.mass_log, + metadata=parent.metadata, + ) + legacy._detach_frame_axes(result) + return result + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + _paths(member_paths) + source.validate() + before = _implementation() + parent = source.frame + person = parent.person + _require(OBSERVED_COLUMN not in person, "RESTORATION_ALREADY_ATTACHED") + _require( + all( + c in person for c in ("source_household_id", "A_LINENO", "A_AGE", "PTOTVAL") + ), + "RESTORATION_SOURCE_COLUMNS", + ) + for c in ("source_household_id", "A_LINENO", "A_AGE"): + _require( + person[c].dtype == np.dtype("int64") and not person[c].isna().any(), + "RESTORATION_COORDINATE_DTYPE", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output = np.empty(len(person), dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-person-income-members-" + ) as directory: + for year, member, archive_pin, pin, rows, size in _MEMBER_PINS: + staged = Path(directory) / member + legacy._stage_verified(member_paths[year], pin, staged) + raw = _read_member(staged, rows=rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == rows, "RESTORATION_COHORT_ROWS") + recipient_keys = pd.Index(keys[positions]) + _require(not recipient_keys.has_duplicates, "RESTORATION_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient_keys) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == rows, + "RESTORATION_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for source_col, raw_col in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + person[source_col].iloc[positions].to_numpy(), + joined[raw_col].to_numpy(), + ), + "RESTORATION_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PTOTVAL.iloc[positions] + observed = joined.PTOTVAL.to_numpy(dtype=np.int64) + known = ~incumbent.isna().to_numpy() + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "RESTORATION_INCUMBENT_CONFLICT", + "PTOTVAL", + ) + # The full current cohort has an incumbent and must be checked, not + # silently treated as another missing-column restoration. + if year == 2024: + _require( + bool(known.all()), "RESTORATION_INCUMBENT_INCOMPLETE", "PTOTVAL" + ) + output[positions] = observed + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": int(known.sum()), + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + result = _owned_frame(parent) + result.person[OBSERVED_COLUMN] = output + source.validate() + _require(_implementation() == before, "RESTORATION_IMPLEMENTATION_CHANGED") + evidence = source.source.evidence + from .asec_current_money import _parse + + parent_evidence = _parse(evidence) + receipt = { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "original_v4_sha256": parent_evidence["parent_sha256"], + "household_attachment_sha256": parent_evidence["attachment_sha256"], + "parent_frame_sha256": legacy._frame_signature(parent), + "output_frame_sha256": legacy._frame_signature(result), + "reader": "authenticated_census_csv_exact_integer_no_fill_v1", + "join_keys": ["source_year", "PERIDNUM"], + "crosscheck_columns": ["source_household_id", "A_LINENO", "A_AGE"], + "sources": joins, + "output_column": OBSERVED_COLUMN, + "logical_field": "PTOTVAL", + "nominal_basis": "income_year_us_dollars", + "dtype": "int64", + "observation_sha256": _sha(output.astype(" _VerifiedRestoration: + """Authenticate and reconstruct T fully; this function does not issue money.""" + try: + return _verify( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths, + ) + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def restore_asec_person_income_source( + parent_path: str | Path, + household_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Write a new local T and receipt from closed sources; never replace parents.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + try: + _paths(member_paths) + source = legacy.load_authenticated_current_money_source( + parent_path, household_attachment_path + ) + frame, receipt = _reconstruct(source, member_paths) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-person-income-", dir=destination.parent + ) as directory: + staging = Path(directory) + output = staging / CHECKPOINT_FILENAME + checkpoint.write_frame_checkpoint( + output, frame, metadata=_document(receipt) + ) + verified = _verify( + parent_path, household_attachment_path, output, member_paths + ) + report = { + **receipt, + "output_sha256": verified.attachment_sha256, + "output_file": CHECKPOINT_FILENAME, + } + (staging / "restoration.receipt.json").write_bytes(_json(report) + b"\n") + # Exclusive final directory creation avoids replacing even a raced + # empty directory. No historical artifact is opened for writing. + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return report + except FileExistsError: + raise + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def load_authenticated_restored_current_money_source( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], +) -> legacy.AuthenticatedCurrentMoneySource: + """Issue restored authority only after closed-source complete reconstruction.""" + from .asec_current_money import ( + _SOURCE_TOKEN, + RESTORED_FIELD_COLUMNS, + RESTORED_FIELD_ZERO_POLICY, + RESTORED_SOURCE_KIND, + AuthenticatedAsecSource, + _parse, + compile_asec_current_money_spec, + ) + from .asec_current_money_resources import load_current_money_resources + + checked = verify_asec_person_income_source( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths=member_paths, + ) + frame = checked.frame + scope = legacy._scope(frame) + views = legacy._views(frame, scope, restored=True) + scope_sha, input_sha = legacy._input_binding(views) + evidence = _parse(checked.parent_source.source.identity) + evidence.update( + schema_version=2, + source_kind=RESTORED_SOURCE_KIND, + person_income_attachment_sha256=checked.attachment_sha256, + field_source_columns=dict(RESTORED_FIELD_COLUMNS), + field_zero_origin_policy=dict(RESTORED_FIELD_ZERO_POLICY), + scope_sha256=scope_sha, + input_sha256=input_sha, + frame_sha256=legacy._frame_signature(frame), + verification_sha256=checked.receipt["implementation_sha256"], + ) + checked.parent_source.validate() + _require( + _implementation() == evidence["verification_sha256"], + "RESTORATION_IMPLEMENTATION_CHANGED", + ) + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + result = legacy.AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=legacy._LOAD_TOKEN + ) + result.validate() + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py new file mode 100644 index 000000000..09d971b68 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -0,0 +1,507 @@ +"""Authenticate the complete 2024 ASEC catalogue before population construction. + +The closed parent still authenticates three cohorts. Only original 2024 records +are exposed here. No descendant Frame, unit assignment, domain classification, +selection, or weight allocation is performed. Household literals are retained; +A_AGE and A_LINENO are canonical strings of authenticated numeric originals. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import weakref +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import NamedTuple + +from . import asec_2024_native_population as native +from . import survey_population_domains as domains + +ARTIFACT_KIND = "microcosm.us.asec-source-catalogue.v1" +_MAX_ENVELOPE_BYTES = 1024**2 +_MAX_RECORD_BYTES = 128 * 1024**2 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecSourceCatalogueError(ValueError): + """Static refusal codes, without source paths or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecSourceCatalogueError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + # Only compact identities and individual records use this encoder. The + # complete catalogue is hashed incrementally, never encoded as another body. + result = bytearray() + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + for part in encoder.iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_ENVELOPE_BYTES, "ENVELOPE_SIZE") + result.extend(part) + return bytes(result) + + +def _implementation(): + modules = (sys.modules[__name__], domains) + runtime = {m.__name__: native.coverage_owner._runtime_code(m) for m in modules} + code = {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in modules} + _require( + runtime == _RUNTIME_AUTHORITY and code == _BYTE_AUTHORITY, + "PRODUCER_CODE_CHANGED", + ) + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": code, + "source_owner": native._implementation(), + "limits": [_MAX_ENVELOPE_BYTES, _MAX_RECORD_BYTES], + "periods": [2024, 2025], + } + + +def _source_authority(): + """Snapshot exact code-owned source pin values without file/resource I/O. + + Canonical bytes detach the seal from mutable registry objects. The original + household registry helper validates its records but reads no source files. + """ + return _encode( + { + "parent": native.parent_owner._SOURCE_PINS, + "restoration": native.restoration._MEMBER_PINS, + "coverage": native.coverage_owner._MEMBER_PINS, + "household": [asdict(pin) for pin in native.anchor_owner._registry()], + } + ) + + +@dataclass(frozen=True, slots=True) +class UnrepresentedAsecHousehold: + """Original noninterview household, absent from the authenticated roster. + + This is a custody ledger, not a scientific coverage classification. No empty + person list is supplied as a claim about the household's actual membership. + """ + + key: domains.HouseholdKey + h_hhtype: str + hrhtype: str + h_livqrt: str + h_numper: str + hsup_wgt: str + reason: str = "original_noninterview_not_in_authenticated_person_roster" + + +def _key_record(key): + _require(type(key) is domains.HouseholdKey, "RECORD_TYPE") + _require(key.source is domains.Source.ASEC, "RECORD_SOURCE") + return [key.source.value, key.source_year, key.survey_year, key.native_id] + + +def _record(row): + _require( + type(row) in (domains.AsecHousehold, UnrepresentedAsecHousehold), + "RECORD_TYPE", + ) + result = { + "key": _key_record(row.key), + "H_HHTYPE": row.h_hhtype, + "HRHTYPE": row.hrhtype, + "H_LIVQRT": row.h_livqrt, + "H_NUMPER": row.h_numper, + "HSUP_WGT": row.hsup_wgt, + } + if type(row) is UnrepresentedAsecHousehold: + result["unrepresented_reason"] = row.reason + else: + _require(type(row.persons) is tuple, "RECORD_TYPE") + persons = [] + for person in row.persons: + _require(type(person) is domains.AsecPerson, "RECORD_TYPE") + persons.append( + [ + person.peridnum, + person.a_lineno, + person.age, + person.prpertyp, + person.prpertyp_state, + _key_record(person.household_key), + ] + ) + result["persons"] = persons + return result + + +def _records_identity(households, ledger): + _require(type(households) is tuple and type(ledger) is tuple, "RECORD_TYPE") + digest, count, size, persons = hashlib.sha256(), 0, 0, 0 + for kind, rows in (("household", households), ("unrepresented", ledger)): + for row in rows: + data = _encode([kind, _record(row)]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + if kind == "household": + persons += len(row.persons) + return (digest.hexdigest(), size, count, persons) + + +def _bound_records_identity(rows, roster): + """Stream complete receiving/native keys, source positions and exact anchors.""" + digest, count, size = hashlib.sha256(), 0, 0 + for kind, values in (("household", rows), ("person", roster)): + for row in values: + data = _encode([kind, row]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + return {"sha256": digest.hexdigest(), "canonical_bytes": size, "records": count} + + +def _rows(rows, roster, anchors, fields): + people = {} + for person in roster: + people.setdefault(int(person["source_household_id"]), []).append(person) + households = [] + for row in sorted(rows, key=lambda r: r["source"]["member_row_1based"]): + original = row["household_fields"] + key = domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, original["H_SEQ"]) + members = tuple( + domains.AsecPerson( + p["PERIDNUM"], + str(int(p["A_LINENO"])), + str(int(p["A_AGE"])), + p["PRPERTYP"], + p["PRPERTYP_state"], + key, + ) + for p in people[row["native_key"][1]] + ) + households.append( + domains.AsecHousehold( + key, + original["H_HHTYPE"], + original["HRHTYPE"], + original["H_LIVQRT"], + original["H_NUMPER"], + row["HSUP_WGT"], + members, + ) + ) + # _roster(None) already authenticates every original member row and refuses + # interview rows absent from the represented roster or noninterviews in it. + weights = {r["H_SEQ"]: r for r in anchors.document["records"]} + ledger = tuple( + UnrepresentedAsecHousehold( + domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, r["H_SEQ"]), + r["H_HHTYPE"], + r["HRHTYPE"], + r["H_LIVQRT"], + r["H_NUMPER"], + weights[r["H_SEQ"]]["HSUP_WGT"], + ) + for r in fields.document["records"] + if r["H_HHTYPE"] != "1" + ) + return tuple(households), ledger + + +class _State(NamedTuple): + parent: object + parent_frame: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + source_authority: bytes + parent_identity: str + attached_evidence: tuple + households: tuple + ledger: tuple + records_identity: tuple + + +def _validate_state(state): + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _records_identity(state.households, state.ledger) == state.records_identity, + "RECORDS_CHANGED", + ) + state.parent.validate() + native.coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + native.anchor_owner.verify_asec_household_weights_source(state.anchors) + native.field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + native._file_identity(path, expected, maximum, size) + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _records_identity(state.households, state.ledger) == state.records_identity, + "RECORDS_CHANGED", + ) + # Pure final seals after source/resource I/O; no recursively repeated reads. + current = native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + # Producer helpers read registry values before later helper source-code I/O. + # A current borrow must refuse a late registry change, without waiting for + # the next producer pass. This final comparison is pure and bounded. + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecSourceCatalogue: + """Process-issued immutable records with checked borrows and compact evidence.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsecSourceCatalogue + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecSourceCatalogueError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecSourceCatalogueError("CATALOGUE_BINDING_REFUSAL") from None + + @property + def households(self) -> tuple[domains.AsecHousehold, ...]: + return self._checked()[2].households + + @property + def exclusion_ledger(self) -> tuple[UnrepresentedAsecHousehold, ...]: + return self._checked()[2].ledger + + @property + def receipt(self) -> dict: + return json.loads(self._checked()[1]) + + def to_bytes(self) -> bytes: + return self._checked()[1] + + def validate(self) -> None: + self._checked() + + +def verify_asec_source_catalogue(value: object) -> AuthenticatedAsecSourceCatalogue: + _require(type(value) is AuthenticatedAsecSourceCatalogue, "UNISSUED_OR_CHANGED") + value.validate() + return value + + +def issue_asec_source_catalogue( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + person_member_paths: Mapping[int, str | Path], + household_member_path: str | Path, + candidate: bytes | None = None, +) -> AuthenticatedAsecSourceCatalogue: + """Reconstruct the complete 2024 source catalogue from authenticated paths. + + Candidate bytes are compared only after actual reconstruction. No caller + Frame, record list, weights or receipt can grant source authority. + """ + try: + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_ENVELOPE_BYTES), + "CANDIDATE_SIZE_OR_TYPE", + ) + parent_path, household_path, restored_path, persons, member, _, candidate = ( + native._inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + None, + candidate, + ) + ) + source_authority = _source_authority() + producer = _encode(_implementation()) + parent = native.restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = native._frame_identity(parent.frame) + coverage = native.coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = native.anchor_owner.load_authenticated_asec_household_weights( + {2024: member} + ) + fields = native.field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + _, _, rows, roster = native._roster(parent, coverage, anchors, fields, None) + households, ledger = _rows(rows, roster, anchors, fields) + records_identity = _records_identity(households, ledger) + identity = json.loads(parent.source.identity) + files = [ + ( + parent_path, + identity["parent_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + household_path, + identity["attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ] + sources = coverage.receipt["sources"] + for source in sources: + files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + attached = native._attached_evidence(parent, coverage, anchors, fields) + state = _State( + parent, + parent.frame, + coverage, + anchors, + fields, + tuple(files), + producer, + source_authority, + parent_identity, + attached, + households, + ledger, + records_identity, + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "survey_year": 2025, + "counts": { + "households": len(households), + "persons": records_identity[3], + "unrepresented_households": len(ledger), + }, + "records": { + "sha256": records_identity[0], + "canonical_bytes": records_identity[1], + "count": records_identity[2], + }, + "native_binding": _bound_records_identity(rows, roster), + "producer_sha256": _sha(producer), + "source_authority_sha256": _sha(source_authority), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "catalogue_cohorts": [2024], + "identity": identity, + "frame_sha256": parent_identity, + }, + "source_members": sources, + "household_member": pin, + "attached_capsules_sha256": [_sha(value) for value in attached[1:]], + "age_representation": "canonical_integer_string_of_authenticated_original_A_AGE_not_retained_CSV_lexeme", + "line_representation": "canonical_integer_string_of_authenticated_original_A_LINENO_not_retained_CSV_lexeme", + "household_key_representation": "original_H_SEQ_CSV_lexeme", + "classification_performed": False, + "weights_applied": False, + "selection_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsecSourceCatalogue(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecSourceCatalogueError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecSourceCatalogueError("CATALOGUE_SOURCE_REFUSAL") from None + + +_RUNTIME_AUTHORITY = { + m.__name__: native.coverage_owner._runtime_code(m) + for m in (sys.modules[__name__], domains) +} + +_BYTE_AUTHORITY = { + m.__name__: _sha(Path(m.__file__).read_bytes()) + for m in (sys.modules[__name__], domains) +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py new file mode 100644 index 000000000..0aa7ae532 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py @@ -0,0 +1,567 @@ +"""Full-source ASEC preparation for the current-money graph slice. + +The graph source is one directory holding exactly the reviewed restoration +inputs: the raw-stage parent P, the household-observation attachment H, the +restored person-income attachment T with its receipt, the three housing cohort +HDFs, and the three official Census PERSON CSV members. No ZIP archive is +accepted and no file is discovered by pattern: the roster below is the contract. + +The executor keys a directory source by relative name plus file bytes, which is +a cache identity, not an authentication. Authentication stays where it already +is — the pinned digests inside the reviewed loaders and the canonical byte +replay of T — and this module simply refuses to run unless those loaders issue +authority. + +The authenticated ``ReadyCurrentMoney`` exists only inside this call. What +leaves is the canonical encoded body, a receipt of identities, and an +operator-free frame whose dtypes have been promoted to graph tokens. +""" + +from __future__ import annotations + +import os +import stat +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.population import dtype_for_token, token_for_dtype + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as housing_universe +from . import asec_housing_universe_source as housing_universe_source +from . import asec_income_observations as income_source +from . import asec_person_income_source as person_income +from . import asec_student_controls as student_source +from ._asec_current_money_codec import ( + current_money_content_sha256, + encode_current_money, +) +from .asec_checkpoint import ASEC_RAW_STAGE_CHECKPOINT_FILENAME +from .asec_current_money import ( + ENCODED_ZERO_POLICY, + FIELDS, + RESTORED_SOURCE_KIND, + ZERO_POLICY, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _sha, +) +from .asec_current_money import ( + _digest as _digest_shape, +) +from .asec_current_money_units import reconstruct_current_money_tax_units +from .asec_housing_status_source import ( + attach_housing_status, + load_authenticated_housing_status, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .operator_boundary import assert_operator_free_source_frame + +PREPARED_SOURCE_KIND = "us_asec_prepared_current_money_v3" +PREPARED_RECEIPT_SCHEMA = "microcosm.us.asec_prepared_receipt.v3" +PREPARATION_BOUNDARY = "US graph ASEC prepared source" +SOURCE_YEARS: tuple[int, ...] = (2022, 2023, 2024) +PARENT_FILENAME = ASEC_RAW_STAGE_CHECKPOINT_FILENAME +HOUSEHOLD_ATTACHMENT_FILENAME = "asec_household_observations.checkpoint.h5" +PERSON_INCOME_FILENAME = person_income.CHECKPOINT_FILENAME +RESTORATION_RECEIPT_FILENAME = "restoration.receipt.json" +COHORT_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, f"asec_household_cohort_{year}.h5") for year in SOURCE_YEARS +) +MEMBER_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].member) for year in SOURCE_YEARS +) +PREPARED_SOURCE_FILES: tuple[str, ...] = ( + PARENT_FILENAME, + HOUSEHOLD_ATTACHMENT_FILENAME, + PERSON_INCOME_FILENAME, + RESTORATION_RECEIPT_FILENAME, + *(name for _year, name in COHORT_FILENAMES), + *(name for _year, name in MEMBER_FILENAMES), +) +RESTORATION_RECEIPT_MAX_BYTES = 256 * 1024 +_INTEGER_WIDTH_TARGETS = frozenset({"int8", "int16", "uint8", "uint16", "uint32"}) +_NULLABLE_INTEGER_TARGETS = frozenset( + {"Int8", "Int16", "Int32", "UInt8", "UInt16", "UInt32", "UInt64"} +) + + +class PreparedSourceRefusalError(ValueError): + """Sanitized preparation refusal; never carries rows, values or paths.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedSourceRefusalError(reason) + + +def _roster(source_dir) -> dict[str, Path]: + directory = Path(source_dir) + _require(directory.is_dir(), "SOURCE_DIRECTORY") + present = sorted(entry.name for entry in directory.iterdir()) + _require(present == sorted(PREPARED_SOURCE_FILES), "SOURCE_FILE_ROSTER") + paths = {} + for name in PREPARED_SOURCE_FILES: + path = directory / name + _require(path.is_file() and not path.is_symlink(), "SOURCE_REGULAR_FILE") + paths[name] = path + return paths + + +RESTORATION_READER = "authenticated_census_csv_exact_integer_no_fill_v1" +RESTORATION_JOIN_KEYS = ["source_year", "PERIDNUM"] +RESTORATION_CROSSCHECK_COLUMNS = ["source_household_id", "A_LINENO", "A_AGE"] +RESTORATION_LOGICAL_FIELD = "PTOTVAL" +RESTORATION_NOMINAL_BASIS = "income_year_us_dollars" +RESTORATION_ZERO_CLAIM = "encoded_source_value_not_respondent_answer" +_RESTORATION_RECEIPT_KEYS = frozenset( + { + "artifact_kind", + "crosscheck_columns", + "dtype", + "encoding_contract", + "household_attachment_sha256", + "implementation_sha256", + "join_keys", + "logical_field", + "nominal_basis", + "observation_sha256", + "original_v4_sha256", + "output_column", + "output_file", + "output_frame_sha256", + "output_sha256", + "parent_frame_sha256", + "reader", + "schema_version", + "sources", + "zero_claim", + "zero_origin_policy", + } +) +_RESTORATION_SOURCE_KEYS = frozenset( + { + "archive_sha256", + "income_year", + "incumbent_compared_rows", + "incumbent_conflicts", + "joined_rows", + "member", + "member_sha256", + "native_key_or_age_conflicts", + "source_rows", + "survey_year", + "unreferenced_source_rows", + } +) + + +def _restoration_sources(rows, source) -> None: + _require( + type(rows) is list and len(rows) == len(SOURCE_YEARS), "RESTORATION_SOURCES" + ) + for row, pins in zip(rows, person_income._MEMBER_PINS, strict=True): + year, member, archive_pin, member_pin, source_rows, _size = pins + positions = np.flatnonzero(np.asarray(source.scope.person_years) == year) + known_incumbents = int( + source.frame.person.PTOTVAL.iloc[positions].notna().sum() + ) + _require( + type(row) is dict and set(row) == _RESTORATION_SOURCE_KEYS, + "RESTORATION_SOURCES", + ) + _require( + row["income_year"] == year + and row["survey_year"] == year + 1 + and row["member"] == member + and row["archive_sha256"] == archive_pin + and row["member_sha256"] == member_pin + and row["source_rows"] == source_rows, + "RESTORATION_SOURCE_PINS", + ) + _require( + all( + type(row[name]) is int and row[name] >= 0 + for name in ( + "joined_rows", + "unreferenced_source_rows", + "incumbent_compared_rows", + ) + ) + and row["joined_rows"] == len(positions) == source_rows + and row["unreferenced_source_rows"] == 0 + and row["incumbent_compared_rows"] == known_incumbents + and row["incumbent_conflicts"] == 0 + and row["native_key_or_age_conflicts"] == 0, + "RESTORATION_SOURCE_COVERAGE", + ) + + +def _restoration_receipt(path: Path, evidence: dict, source) -> dict: + """Bind the shipped restoration receipt to the authority the loader issued.""" + # Refuse special files even if the roster check raced with a replacement, + # and bound allocation before parsing an untrusted sidecar. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode), "RESTORATION_RECEIPT") + _require( + 0 < info.st_size <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT" + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + payload = stream.read(RESTORATION_RECEIPT_MAX_BYTES + 1) + finally: + os.close(descriptor) + _require(0 < len(payload) <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT") + _require(payload.endswith(b"\n"), "RESTORATION_RECEIPT") + try: + receipt = _parse(payload[:-1], RESTORATION_RECEIPT_MAX_BYTES) + except MoneyRefusalError as error: + raise PreparedSourceRefusalError("RESTORATION_RECEIPT") from error + _require(_json(receipt) == payload[:-1], "RESTORATION_RECEIPT_CANONICAL") + _require(set(receipt) == _RESTORATION_RECEIPT_KEYS, "RESTORATION_RECEIPT_SCHEMA") + _require( + receipt["schema_version"] == 1 + and receipt["artifact_kind"] == person_income.ARTIFACT_KIND + and receipt["output_file"] == PERSON_INCOME_FILENAME + and receipt["output_column"] == person_income.OBSERVED_COLUMN + and receipt["encoding_contract"] == person_income._ENCODING_CONTRACT + and receipt["reader"] == RESTORATION_READER + and receipt["join_keys"] == RESTORATION_JOIN_KEYS + and receipt["crosscheck_columns"] == RESTORATION_CROSSCHECK_COLUMNS + and receipt["logical_field"] == RESTORATION_LOGICAL_FIELD + and receipt["nominal_basis"] == RESTORATION_NOMINAL_BASIS + and receipt["dtype"] == "int64" + and receipt["zero_origin_policy"] == ENCODED_ZERO_POLICY + and receipt["zero_claim"] == RESTORATION_ZERO_CLAIM, + "RESTORATION_RECEIPT_SCHEMA", + ) + _require( + all( + _digest_shape(receipt[name]) + for name in ( + "observation_sha256", + "output_frame_sha256", + "parent_frame_sha256", + ) + ), + "RESTORATION_RECEIPT_DIGEST", + ) + _restoration_sources(receipt["sources"], source) + # The reconstruction the loader just performed is the authority; the shipped + # receipt must agree with the evidence it issued, digest for digest. + _require( + receipt["output_sha256"] == evidence["person_income_attachment_sha256"] + and receipt["original_v4_sha256"] == evidence["parent_sha256"] + and receipt["household_attachment_sha256"] == evidence["attachment_sha256"] + and receipt["implementation_sha256"] == evidence["verification_sha256"], + "RESTORATION_RECEIPT_BINDING", + ) + # T1's pinned producer only appends the observed person-income column to an + # owned copy of the authenticated parent. Recover that exact parent view + # without reading another source or changing the original T1 authority. + # The sidecar must describe those reconstructed facts, not merely contain + # syntactically valid digest strings beside an authentic checkpoint. + source.validate() + observed = source.frame.person[person_income.OBSERVED_COLUMN] + _require(observed.dtype == np.dtype("int64"), "RESTORATION_OBSERVATION_DTYPE") + _require( + receipt["observation_sha256"] + == _sha(observed.to_numpy(dtype=" str: + try: + return token_for_dtype(dtype) + except Exception as error: # PopulationError is a ValueError subclass + del error + name = getattr(dtype, "name", None) + if isinstance(dtype, pd.api.extensions.ExtensionDtype): + _require(name in _NULLABLE_INTEGER_TARGETS, "UNPROMOTABLE_DTYPE") + return "Int64" + normalized = np.dtype(dtype) + _require( + normalized.name in _INTEGER_WIDTH_TARGETS or normalized.name == "float16", + "UNPROMOTABLE_DTYPE", + ) + return "float64" if normalized.name == "float16" else "int64" + + +def _promote(frame: Frame) -> tuple[dict[str, str], ...]: + """Promote every column to its graph dtype token, value- and null-preserving.""" + canonicalize_frame_string_dtypes( + frame, boundary=PREPARATION_BOUNDARY, in_place=True + ) + transitions: list[dict[str, str]] = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in list(table.columns): + _require(isinstance(column, str), "COLUMN_NAME") + original = table[column] + token = _target_token(original.dtype) + target = dtype_for_token(token) + if original.dtype == target: + continue + promoted = original.astype(target) + _require(original.isna().equals(promoted.isna()), "DTYPE_PROMOTION_NULLS") + observed = original.notna() + _require( + np.array_equal( + original[observed].to_numpy(dtype=object), + promoted[observed].to_numpy(dtype=object), + ), + "DTYPE_PROMOTION_VALUES", + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": str(original.dtype), + "to": token, + } + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + token_for_dtype(table[column].dtype) + return tuple(transitions) + + +def _entity_rows(frame: Frame) -> dict[str, int]: + return {entity: frame.n(entity) for entity in US_SCHEMA.entities} + + +def prepared_source_implementation_identity() -> str: + """Bind this preparation's own code plus the reviewed loaders it calls.""" + package = resources.files(__package__) + modules = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_prepared_source.py", + "asec_income_observations.py", + "asec_current_money_selection.py", + "cps_carried_current.py", + "graph_housing_universe.py", + ) + } + return _sha( + _json( + { + "schema": 3, + "source_kind": PREPARED_SOURCE_KIND, + "modules": modules, + "money_source_verification": money_source._verification_identity(), + "restoration_verification": person_income._implementation(), + "student_controls_verification": student_source._implementation(), + "income_observations_verification": income_source._implementation(), + "housing_verification": housing_source._implementation(), + "housing_universe_verification": housing_universe_source._implementation(), + "file_roster": list(PREPARED_SOURCE_FILES), + "boundary": PREPARATION_BOUNDARY, + } + ) + ) + + +@dataclass(frozen=True) +class PreparedAsecPopulation: + """An operator-free prepared frame with its money body and identity receipt.""" + + frame: Frame + money_payload: bytes + receipt: dict + field_entities: tuple[tuple[str, str], ...] + housing_universe_payload: bytes + income_observations_payload: bytes + + @property + def receipt_payload(self) -> bytes: + return _json(self.receipt) + + @property + def receipt_sha256(self) -> str: + return _sha(self.receipt_payload) + + +def prepare_asec_current_money_population(source_dir) -> PreparedAsecPopulation: + """Load, reconstruct, attach and promote the full authenticated population.""" + paths = _roster(source_dir) + identity = prepared_source_implementation_identity() + source = person_income.load_authenticated_restored_current_money_source( + paths[PARENT_FILENAME], + paths[HOUSEHOLD_ATTACHMENT_FILENAME], + paths[PERSON_INCOME_FILENAME], + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + evidence = _parse(source.source.identity) + _require(evidence["source_kind"] == RESTORED_SOURCE_KIND, "RESTORED_SOURCE_KIND") + _require(evidence["zero_origin_policy"] == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + _require(evidence["field_roster"] == list(FIELDS), "FIELD_ROSTER") + restoration = _restoration_receipt( + paths[RESTORATION_RECEIPT_FILENAME], evidence, source + ) + ready = source.ready() + _require(type(ready) is ReadyCurrentMoney, "AUTHENTICATED_READINESS_REQUIRED") + income = income_source.load_authenticated_income_observations( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + # S is separate source authority. Its public loader authenticates the same + # official members against the unchanged T1 and money parents; attachment + # adds two aliases while retaining original missing controls untouched. + controls = student_source.load_authenticated_student_controls( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + with_students = student_source.attach_student_controls(source, ready, controls) + tax = reconstruct_current_money_tax_units(with_students, ready) + # Housing still authenticates original T1, never the transient tax/S view. + status = load_authenticated_housing_status( + source, cohort_paths={year: paths[name] for year, name in COHORT_FILENAMES} + ) + attached = attach_housing_status(tax, status) + attached.validate() + # HU authenticates original T1 and retains the complete S/tax/housing parent. + # Its strict uint8 attachment validation runs before graph width promotion. + universe = housing_universe_source.load_authenticated_housing_universe(source) + with_universe = housing_universe_source.attach_housing_universe(attached, universe) + with_universe.validate() + frame = housing_source._owned_frame(with_universe.frame) + pre_promotion = money_source._frame_signature(frame) + _require( + pre_promotion == with_universe.receipt["output_frame_sha256"], "PARENT_CAPTURE" + ) + transitions = _promote(frame) + post_promotion = money_source._frame_signature(frame) + # The graph carries metadata as a typed context artifact, and the store's + # frame codec does not persist Frame.metadata at all. A non-empty prepared + # metadata mapping would be silently dropped, so refuse instead. + _require(dict(frame.metadata) == {}, "PREPARED_FRAME_METADATA") + _require(frame.mass_log == (), "PREPARED_FRAME_MASS_LOG") + _require(frame.weighted_entities == ("household",), "PREPARED_FRAME_WEIGHTS") + frame.revalidate() + assert_operator_free_source_frame(frame, label=PREPARATION_BOUNDARY) + _require( + prepared_source_implementation_identity() == identity, + "PREPARATION_IMPLEMENTATION_CHANGED", + ) + money_payload = encode_current_money(ready) + universe_payload = housing_universe.encode_housing_universe(universe) + income_payload = income_source.encode_income_observations(income) + field_entities = tuple( + (domain.name, domain.entity) for domain in ready.bindings.spec.fields + ) + receipt = { + "schema": PREPARED_RECEIPT_SCHEMA, + "source_kind": PREPARED_SOURCE_KIND, + "restored_source_kind": evidence["source_kind"], + "release_eligible": False, + "all_current_money_consumers_wired": False, + "source_evidence_sha256": _sha(source.source.identity), + "t1_sha256": evidence["person_income_attachment_sha256"], + "income_observations": { + "parser_profile": income_source.parser_profile(), + "header_sha256": _sha(income._header), + "content_sha256": income.content_sha256, + "payload_sha256": _sha(income_payload), + "rows": income.receipt["rows"], + "columns": list(income_source.COLUMNS), + "kind": income_source.ARTIFACT_KIND, + "scope_sha256": income.receipt["scope_sha256"], + }, + "student_controls_receipt_sha256": _sha(_json(controls.receipt)), + "student_controls_content_sha256": controls.content_sha256, + "student_controls_attachment_receipt_sha256": _sha( + _json(with_students.receipt) + ), + "student_controls_reference_period": controls.receipt["reference_period"], + "annual_five_month_student_status_validated": False, + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "money_spec_sha256": ready.bindings.spec.sha256, + "field_entities": [list(item) for item in field_entities], + "scope_sha256": _parse(ready.header)["scope_sha256"], + "tax_receipt_sha256": _sha(_json(tax.receipt)), + "tax_old_partition_sha256": tax.receipt["old_partition_sha256"], + "tax_new_partition_sha256": tax.receipt["new_partition_sha256"], + "housing_content_sha256": status.content_sha256, + "housing_attachment_receipt_sha256": _sha(_json(attached.receipt)), + "pre_housing_universe_frame_sha256": attached.receipt["output_frame_sha256"], + "housing_universe": { + "header_sha256": _sha(universe.header), + "content_sha256": universe.content_sha256, + "payload_sha256": _sha(universe_payload), + "definition_sha256": universe.header_data["definition_sha256"], + "attachment_receipt_sha256": _sha(_json(with_universe.receipt)), + "parent_kind": "HousingStatusAttachedAsec", + "source_period_kind": "interview_household_universe", + "native_dtype": "uint8", + "graph_dtype": "int64", + "aliases": list(housing_universe_source.ATTACHED_COLUMNS), + }, + "restoration_receipt_sha256": _sha(_json(restoration)), + "pre_promotion_frame_sha256": pre_promotion, + "post_promotion_frame_sha256": post_promotion, + "dtype_transitions": [dict(item) for item in transitions], + "entity_rows": _entity_rows(frame), + "implementation_sha256": identity, + "file_roster": list(PREPARED_SOURCE_FILES), + } + return PreparedAsecPopulation( + frame, money_payload, receipt, field_entities, universe_payload, income_payload + ) + + +def load_graph_asec_prepared(path, *, store=None) -> Frame: + """Registered source codec: the prepared operator-free population only. + + The executor verifies that this loader is installed and content-keys the + directory; it never calls it. The declared computation that invokes the + preparation is the CREATE kernel, which needs the money body and receipt + this signature cannot return, so the two paths share one implementation + rather than preparing the population twice. + """ + del store + return prepare_asec_current_money_population(path).frame + + +__all__ = [ + "COHORT_FILENAMES", + "HOUSEHOLD_ATTACHMENT_FILENAME", + "MEMBER_FILENAMES", + "PARENT_FILENAME", + "PERSON_INCOME_FILENAME", + "PREPARATION_BOUNDARY", + "PREPARED_RECEIPT_SCHEMA", + "PREPARED_SOURCE_FILES", + "PREPARED_SOURCE_KIND", + "PreparedAsecPopulation", + "PreparedSourceRefusalError", + "RESTORATION_RECEIPT_FILENAME", + "load_graph_asec_prepared", + "prepare_asec_current_money_population", + "prepared_source_implementation_identity", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py new file mode 100644 index 000000000..ddf127896 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py @@ -0,0 +1,511 @@ +"""Closed Census interview-week controls, separate from the T1 money source. + +These observations do not establish annual or five-month tax-student status. +Candidate bytes never supply parsed content or authority: replay reconstructs S +from the authenticated T1 parent and the code-owned official CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CONTROLS = ("A_ENRLW", "A_FTPT") +ALIASES = ("asec_A_ENRLW", "asec_A_FTPT") +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + CONTROLS +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE") + CONTROLS +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +MAGIC = b"MCASTUD\x01" +FILENAME = "asec_student_controls.bin" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +def _implementation(): + package = resources.files(__package__) + return { + "schema": 1, + "t1_verification_sha256": restoration._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_student_controls.py", "_asec_current_money_codec.py") + }, + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "aliases": dict(zip(CONTROLS, ALIASES, strict=True)), + } + + +def _parent(source, ready, *, reconstruct_money=False): + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource + and type(ready) is ReadyCurrentMoney, + "STUDENT_AUTHENTICATED_PARENT", + ) + source.validate() + _require( + _parse(source.source.identity)["source_kind"] == RESTORED_SOURCE_KIND, + "STUDENT_RESTORED_PARENT_REQUIRED", + ) + _require(ready.bindings.spec == source.spec, "STUDENT_MONEY_PARENT") + _validate_money(ready, source.spec, nominal=False) + if reconstruct_money: + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "STUDENT_MONEY_PARENT", + ) + + +def _snapshot(path, destination, *, size): + """Exact-size private capture; no candidate parser and no unbounded copy.""" + digest = hashlib.sha256() + # Opening a FIFO in ordinary blocking mode would wait before fstat could + # reject it. Use one nonblocking descriptor, then admit only regular files. + descriptor = os.open(Path(path), os.O_RDONLY | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "STUDENT_SOURCE_FILE_KIND") + _require(before.st_size == size, "STUDENT_SOURCE_SIZE") + count = 0 + with ( + os.fdopen(descriptor, "rb", closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(1024 * 1024, size - count + 1)): + count += len(chunk) + _require(count <= size, "STUDENT_SOURCE_SIZE") + digest.update(chunk) + dst.write(chunk) + after = os.fstat(descriptor) + attrs = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == size + and all(getattr(before, a) == getattr(after, a) for a in attrs), + "STUDENT_SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + return digest.hexdigest() + + +def _read_member(path, *, rows): + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "STUDENT_MEMBER_HEADER", + ) + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "STUDENT_MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), + "STUDENT_MEMBER_PERSON_KEY", + ) + for name in _READ_COLUMNS[1:]: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "STUDENT_MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "STUDENT_MEMBER_COORDINATES", + ) + for name in CONTROLS: + _require(bool(table[name].isin([0, 1, 2]).all()), "STUDENT_CODE_DOMAIN", name) + _require( + bool((table.A_ENRLW[~table.A_AGE.between(16, 54)] == 0).all()) + and bool((table.A_FTPT[table.A_ENRLW == 1] != 0).all()) + and bool((table.A_FTPT[table.A_ENRLW != 1] == 0).all()), + "STUDENT_SOURCE_UNIVERSE", + ) + return table + + +@dataclass(frozen=True) +class AuthenticatedStudentControls: + """Immutable owned numeric buffers; only closed source reconstruction issues S.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "STUDENT_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "STUDENT_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "STUDENT_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["aliases"] == dict(zip(CONTROLS, ALIASES, strict=True)) + and _json(header["implementation"]) == _json(_implementation()), + "STUDENT_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "STUDENT_COLUMN") + n = self.receipt["rows"] + offset = COLUMNS.index(name) * n * 8 + # A bytes-backed buffer cannot be made writeable by the caller. + return np.frombuffer(self._body, dtype="= 0).all()) and len(np.unique(indices)) == rows, + "STUDENT_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "STUDENT_NATIVE_KEY_OR_AGE", + ) + compared = {} + for name in CONTROLS: + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined[name].to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "STUDENT_INCUMBENT_CONFLICT", + name, + ) + if year == 2024: + _require(bool(known.all()), "STUDENT_INCUMBENT_INCOMPLETE", name) + output[name][positions] = observed + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": compared, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + _parent(source, ready) + _require( + _json(_implementation()) == _json(before), "STUDENT_IMPLEMENTATION_CHANGED" + ) + buffers = [output[name].astype(" dict: + """Annotation fields take precedence; retain sentinels and conflicts verbatim.""" + sentinel = SENTINELS.get(str(raw)) + annotation_class = None + if annotation not in (None, ""): + if not isinstance(annotation, str): + raise ValueError("ACS annotation must be text or null") + annotation_class = ANNOTATIONS.get(annotation) + if annotation_class is None: + annotation_class = ( + "open_interval" + if re.fullmatch(r"[\d,.]+[+-]", annotation) + else "unknown_annotation" + ) + classification = annotation_class or sentinel + numeric = None + if classification is None: + if raw in (None, ""): + classification = "missing" + else: + try: + numeric = float(raw) + if not math.isfinite(numeric): + raise ValueError + except (ValueError, TypeError): + raise ValueError("unrecognized ACS numeric value") from None + classification = "numeric" + return { + "raw": raw, + "annotation": annotation, + "class": classification, + "numeric_value": numeric, + "annotation_conflict": bool( + sentinel and annotation_class and sentinel != annotation_class + ), + } + + +def _district_id(state: str, district: str, congress: int) -> str: + if not re.fullmatch(r"\d{2}", state) or not re.fullmatch(r"\d{2}", district): + raise ValueError("invalid district identity") + return f"500{congress - 100:02d}00US{state}{'00' if district == '98' else district}" + + +def acs_table_inventory(table: str, metadata: dict, data_rows: list) -> dict: + if table not in TABLES: + raise ValueError("undeclared ACS table") + variables = metadata["variables"] + stems = sorted( + key[:-1] + for key in variables + if key.startswith(table + "_") and key.endswith("E") + ) + if not stems or not data_rows: + raise ValueError("ACS table has no estimate variables or data") + header = data_rows[0] + if len(header) != len(set(header)): + raise ValueError("duplicate ACS header") + required = {"NAME", "state", "congressional district"} + required.update( + stem + suffix for stem in stems for suffix in ("E", "M", "EA", "MA") + ) + missing = sorted(required - set(header)) + if missing: + raise ValueError("missing ACS columns: " + ", ".join(missing)) + for stem in stems: + if any(stem + suffix not in variables for suffix in ("E", "M", "EA", "MA")): + raise ValueError("missing ACS variable metadata") + cells, geographies, excluded = [], {}, [] + seen = set() + counts = {"estimate": Counter(), "moe": Counter()} + for values in data_rows[1:]: + if len(values) != len(header): + raise ValueError("ACS row width mismatch") + row = dict(zip(header, values, strict=True)) + state, district = row["state"], row["congressional district"] + identity = _district_id(state, district, 119) + if identity in seen: + raise ValueError("duplicate geography in ACS table") + seen.add(identity) + if state not in US_STATE_FIPS: + excluded.append( + { + "state_fips": state, + "district": district, + "name": row["NAME"], + "reason": "outside_us_50_states_dc", + } + ) + continue + geographies[identity] = { + "geography_id": identity, + "state_fips": state, + "published_district": district, + "name": row["NAME"], + "published_geo_id": row.get("GEO_ID"), + } + for stem in stems: + estimate = classify_acs_value(row[stem + "E"], row[stem + "EA"]) + moe = classify_acs_value(row[stem + "M"], row[stem + "MA"]) + if moe["numeric_value"] is not None and moe["numeric_value"] < 0: + raise ValueError("negative numeric ACS MOE") + counts["estimate"][estimate["class"]] += 1 + counts["moe"][moe["class"]] += 1 + cells.append( + { + "geography_id": identity, + "variable": stem, + "estimate": estimate, + "moe": moe, + } + ) + return { + "table": table, + "year": 2024, + "congress": 119, + "confidence_level": 0.9, + "uncertainty": { + "kind": "published_margin_of_error", + "variance_conversion": "not_performed", + "controlled_estimates": "raw sentinel retained; no invented variance", + }, + "grouping": "subject_age_and_sex_published_groups" + if table == "S0101" + else "detailed_sex_by_age_published_groups" + if table == "B01001" + else "published_detailed_table_cells", + "reservation": { + "status": "reserved_same_source_holdout" + if table in ("B19001", "B25003") + else "inventory_only", + "covers_derived_versions": True, + "targets_allowed": False, + "tuning_allowed": False, + "source_independent": False, + "reason": "ACS tabulations and ACS PUMS share a source; no candidate scoring or tuning here", + }, + "lineage": "published", + "geography_ids": sorted(geographies), + "geographies": [geographies[key] for key in sorted(geographies)], + "excluded_geographies": excluded, + "variables": { + stem: { + suffix: variables[stem + suffix] for suffix in ("E", "M", "EA", "MA") + } + for stem in stems + }, + "classification_counts": {key: dict(value) for key, value in counts.items()}, + "cells": sorted( + cells, key=lambda cell: (cell["geography_id"], cell["variable"]) + ), + } + + +def irs_csv_inventory(payload: bytes) -> dict: + reader = csv.DictReader(io.StringIO(payload.decode("utf-8-sig"))) + header = reader.fieldnames or [] + identity_fields = {"STATEFIPS", "STATE", "CONG_DISTRICT", "agi_stub"} + if len(header) != len(set(header)) or not identity_fields.issubset(header): + raise ValueError("IRS CSV identity columns missing or duplicated") + metrics = [field for field in header if field not in identity_fields] + counts = {field: Counter() for field in metrics} + rows, seen = [], set() + stubs = Counter() + for row in reader: + if None in row or None in row.values(): + raise ValueError("IRS row width mismatch") + state, district, stub = row["STATEFIPS"], row["CONG_DISTRICT"], row["agi_stub"] + identity = state, district, stub + if identity in seen: + raise ValueError("duplicate IRS source row") + seen.add(identity) + _district_id(state, district, 117) + if state not in US_STATE_FIPS | {"00"} or not re.fullmatch(r"\d+", stub): + raise ValueError("IRS source row outside declared scope") + if state == "00" and district != "00": + raise ValueError("national IRS row has district code") + level = ( + "national" + if state == "00" + else "state" + if district == "00" + else "congressional_district" + ) + rows.append( + { + "state_fips": state, + "state": row["STATE"], + "district": district, + "agi_stub": stub, + "published_geography_level": level, + "published_geography_id": _district_id(state, district, 117) + if level == "congressional_district" + else None, + "at_large_proxy_geography_id": _district_id(state, "00", 117) + if state in AT_LARGE_117 and district == "00" + else None, + } + ) + stubs[stub] += 1 + for field in metrics: + raw = row[field] + if raw in ("", "**"): + classification = ( + "missing" if raw == "" else "suppression_or_combination_marker" + ) + else: + try: + value = float(raw) + if not math.isfinite(value): + raise ValueError + except ValueError: + raise ValueError("unrecognized IRS metric cell") from None + classification = ( + "numeric_zero_disclosure_unknown" + if value == 0 + else "numeric_disclosure_unknown" + ) + counts[field][classification] += 1 + return { + "tax_year": 2022, + "processing_year": 2023, + "congress": 117, + "lineage": "published", + "rows": rows, + "row_count": len(rows), + "columns": header, + "metric_classification_counts": { + field: dict(count) for field, count in counts.items() + }, + "agi_stub_inventory": { + "counts": dict(sorted(stubs.items())), + "guide_documented_codes": list(range(11)), + "labels_status": "needs_method_resolution", + "note": "Retain published codes. Guide section G describes 0–10; fetched CSV contains 0–9. Do not infer income-bin labels.", + }, + "uncertainty": { + "variance_status": "not_published_needs_method_resolution", + "confidence_level": None, + "note": "Guide section C describes a population; footnote 1 identifies an SOI sample input. No variance is inferred.", + }, + "disclosure": { + "numeric_zero_proves_unsuppressed": False, + "csv_cell_suppression_status": "not_identifiable_from_numeric_cells", + "published_rules": [ + "fewer than 20 returns: combine adjacent AGI cells", + "district item totals under 20 excluded", + "dominant single-return items suppressed using unpublished threshold", + ], + "xlsx_markers": "kept separately by sheet and cell; not joined to CSV AGI slices", + }, + "units": { + "monetary_amounts": "thousands_of_dollars", + "rounding_increment": None, + "rounding_status": "not_established_by_unit_label", + }, + "coverage_caveats": [ + "tax filers only", + "tax address may differ from residence", + "ZIP-based district assignment", + "US national total is not a congressional district", + "at-large state totals are explicit proxies, not directly published district observations", + ], + } + + +def crosswalk_alignment(rows: list[dict]) -> dict: + """Summarize incoming target composition, not fiscal allocation error.""" + incoming, outgoing = defaultdict(list), defaultdict(list) + seen = set() + for row in rows: + source, target = row["source_geography_id"], row["target_geography_id"] + if not re.fullmatch(r"5001700US\d{4}", source) or not re.fullmatch( + r"5001900US\d{4}", target + ): + raise ValueError("crosswalk geography vintage mismatch") + if source[9:11] not in US_STATE_FIPS or source[9:11] != target[9:11]: + raise ValueError("crosswalk outside US50+DC or crosses state") + if (source, target) in seen: + raise ValueError("duplicate crosswalk pair") + seen.add((source, target)) + population, weight = float(row["pair_population"]), float(row["weight"]) + if ( + not math.isfinite(population) + or population <= 0 + or not math.isfinite(weight) + or weight <= 0 + ): + raise ValueError( + "crosswalk population and weight must be positive and finite" + ) + incoming[target].append((source, population)) + outgoing[source].append((population, weight)) + if not seen: + raise ValueError("empty crosswalk") + for pairs in outgoing.values(): + total = math.fsum(population for population, _ in pairs) + if not math.isfinite(total) or any( + not math.isclose(weight, population / total, rel_tol=1e-9, abs_tol=1e-12) + for population, weight in pairs + ): + raise ValueError("crosswalk outgoing weights disagree with pair population") + targets = [] + for target, pairs in sorted(incoming.items()): + total = math.fsum(population for _, population in pairs) + contributions = [ + { + "source_geography_id": source, + "pair_population": population, + "incoming_share": population / total, + } + for source, population in sorted(pairs) + ] + maximum = max(item["incoming_share"] for item in contributions) + targets.append( + { + "target_geography_id": target, + "pair_population": total, + "contributions": contributions, + "max_incoming_share": maximum, + "non_dominant_incoming_share": 1 - maximum, + "known_allocation_error": None, + } + ) + return { + "source_congress": 117, + "target_congress": 119, + "population_vintage": "2020_tabulation_blocks", + "basis": "pair_population_normalized_per_target", + "interpretation": "alignment exposure, not known allocation error", + "targets": targets, + } + + +def xlsx_disclosure_inventory(payload: bytes) -> dict: + ns = {"x": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + shared = ET.fromstring(archive.read("xl/sharedStrings.xml")) + strings = ["".join(item.itertext()) for item in shared.findall("x:si", ns)] + cells = [] + for name in sorted(archive.namelist()): + if not re.fullmatch(r"xl/worksheets/sheet\d+\.xml", name): + continue + root = ET.fromstring(archive.read(name)) + for cell in root.findall(".//x:c", ns): + if cell.attrib.get("t") == "s": + value = cell.findtext("x:v", namespaces=ns) + text = strings[int(value)] if value is not None else "" + else: + text = ( + "".join(cell.find("x:is", ns).itertext()) + if cell.find("x:is", ns) is not None + else cell.findtext("x:v", default="", namespaces=ns) + ) + if "**" in text: + cells.append( + { + "sheet_part": name, + "cell": cell.attrib["r"], + "text": text, + "class": "suppression_or_combination_marker" + if text.strip() == "**" + else "disclosure_note", + } + ) + return { + "markers": cells, + "csv_cell_mapping": "not_performed", + "note": "Published workbook cells and disclosure notes remain separate from CSV AGI rows.", + } + + +def _pin_local(output: Path, payload: bytes, suffix: str, **metadata) -> dict: + digest = hashlib.sha256(payload).hexdigest() + relative = f"raw/{digest}{suffix}" + path = output / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_bytes() != payload: + raise ValueError("local source digest collision") + if not path.exists(): + path.write_bytes(payload) + return {**metadata, "sha256": digest, "size_bytes": len(payload), "path": relative} + + +def build_bundle( + output: Path, + irs_authority: Path, + crosswalk: Path, + crosswalk_provenance: Path, + *, + offline: bool = False, + api_key: str | None = None, + fetch=None, +) -> dict: + """Create or replay an immutable inventory. No candidate values are accepted.""" + output = Path(output) + captures, tables = [], {} + for request in census_requests(): + descriptor = capture_request( + output, + request, + offline=offline, + api_key=api_key, + **({"fetch": fetch} if fetch else {}), + ) + captures.append(descriptor) + tables.setdefault(request["table"], {})[request["kind"]] = strict_json( + (output / descriptor["path"]).read_bytes() + ) + acs = { + table: acs_table_inventory( + table, tables[table]["metadata"], tables[table]["data"] + ) + for table in TABLES + } + expected = acs["S0101"]["geography_ids"] + if any(item["geography_ids"] != expected for item in acs.values()): + raise ValueError("ACS table geography universes differ") + index_payload = (irs_authority / "source-index.json").read_bytes() + index = strict_json(index_payload) + by_name = {item["filename"]: item for item in index} + if len(by_name) != len(index): + raise ValueError("duplicate IRS authority source identity") + local, payloads = [], {} + for name in ("22incd.csv", "22incdall.xlsx"): + descriptor = by_name[name] + if descriptor["url"] != "https://www.irs.gov/pub/irs-soi/" + name: + raise ValueError("IRS authority URL mismatch") + payload = (irs_authority / name).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != descriptor["sha256"] + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("IRS authority source digest mismatch") + payloads[name] = payload + local.append( + _pin_local( + output, payload, Path(name).suffix, url=descriptor["url"], filename=name + ) + ) + guide = (irs_authority / "22incddocguide.docx").read_bytes() + if ( + hashlib.sha256(guide).hexdigest() + != (irs_authority / "22incddocguide.sha256").read_text().split()[0] + ): + raise ValueError("IRS guide digest mismatch") + local.append( + _pin_local( + output, + guide, + ".docx", + url=AUTHORITIES["irs_guide"], + filename="22incddocguide.docx", + ) + ) + local.append(_pin_local(output, index_payload, ".json", role="irs_source_index")) + with zipfile.ZipFile(io.BytesIO(guide)) as archive: + guide_xml = archive.read("word/document.xml") + guide_text = "\n".join(ET.fromstring(guide_xml).itertext()).encode() + local.append( + _pin_local( + output, + guide_text, + ".txt", + role="irs_guide_extracted_text", + derived_from_sha256=hashlib.sha256(guide).hexdigest(), + derivation="word/document.xml text nodes in document order", + ) + ) + crosswalk_bytes, provenance_bytes = ( + crosswalk.read_bytes(), + crosswalk_provenance.read_bytes(), + ) + provenance = strict_json(provenance_bytes) + if any( + provenance.get(key) != value + for key, value in { + "source_geography_vintage": "117th_congress", + "target_geography_vintage": "119th_congress", + "block_vintage": "2020_tabulation_blocks", + }.items() + ): + raise ValueError("crosswalk provenance vintage mismatch") + if hashlib.sha256(crosswalk_bytes).hexdigest() != provenance["crosswalk_sha256"]: + raise ValueError("crosswalk provenance digest mismatch") + local.append(_pin_local(output, crosswalk_bytes, ".csv", role="crosswalk")) + local.append( + _pin_local(output, provenance_bytes, ".json", role="crosswalk_provenance") + ) + alignment = crosswalk_alignment( + list(csv.DictReader(io.StringIO(crosswalk_bytes.decode()))) + ) + irs = irs_csv_inventory(payloads["22incd.csv"]) + irs["workbook_disclosure"] = xlsx_disclosure_inventory(payloads["22incdall.xlsx"]) + published = sorted( + { + row["published_geography_id"] + for row in irs["rows"] + if row["published_geography_id"] + } + ) + proxies = sorted( + { + row["at_large_proxy_geography_id"] + for row in irs["rows"] + if row["at_large_proxy_geography_id"] + } + ) + crosswalk_sources = { + item["source_geography_id"] + for target in alignment["targets"] + for item in target["contributions"] + } + targets = {item["target_geography_id"] for item in alignment["targets"]} + bundle = { + "schema_version": 1, + "kind": "us_cd_reference_inventory", + "purpose": "reference_inventory_only", + "candidate_evaluation": "not_performed", + "target_activation": False, + "authorities": AUTHORITIES, + "scope": { + "country": "us", + "states": "50_states_and_dc", + "puerto_rico": "excluded", + "acs_year": 2024, + "target_congress": 119, + "irs_tax_year": 2022, + "irs_congress": 117, + }, + "sources": {"census": captures, "local": local}, + "acs": acs, + "irs": irs, + "alignment": alignment, + "reconciliation": { + "irs_published_district_ids": published, + "irs_at_large_state_proxy_ids": proxies, + "irs_missing_crosswalk_source_ids": sorted( + crosswalk_sources - set(published) - set(proxies) + ), + "irs_unmapped_source_ids": sorted( + (set(published) | set(proxies)) - crosswalk_sources + ), + "acs_missing_crosswalk_target_ids": sorted(targets - set(expected)), + "acs_unmapped_target_ids": sorted(set(expected) - targets), + }, + "lineage_products": { + "published": "included", + "crosswalked_values": "not_generated", + "state_rescaled_values": "not_generated", + }, + } + write_json(output / "reference-inventory.json", bundle) + return bundle + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--irs-authority-dir", type=Path, required=True) + parser.add_argument("--crosswalk", type=Path, required=True) + parser.add_argument("--crosswalk-provenance", type=Path, required=True) + parser.add_argument("--offline", action="store_true") + parser.add_argument("--census-key-service") + parser.add_argument("--census-key-account") + args = parser.parse_args(argv) + if bool(args.census_key_service) != bool(args.census_key_account): + parser.error("credential service and account must be supplied together") + try: + key = ( + read_census_key(args.census_key_service, args.census_key_account) + if args.census_key_service and not args.offline + else None + ) + bundle = build_bundle( + args.output_dir, + args.irs_authority_dir, + args.crosswalk, + args.crosswalk_provenance, + offline=args.offline, + api_key=key, + ) + except (ValueError, OSError, KeyError, zipfile.BadZipFile) as error: + print(f"Reference bundle failed: {error}", file=sys.stderr) + return 1 + print( + f"Reference inventory saved: {args.output_dir / 'reference-inventory.json'}; {len(bundle['acs']['S0101']['geography_ids'])} ACS districts; no candidate evaluation" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py new file mode 100644 index 000000000..9a1e9b137 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py @@ -0,0 +1,188 @@ +"""Pinned public ACS acquisition; credentials never enter stored provenance.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import HTTPRedirectHandler, Request, build_opener + +TABLES = ("S0101", "B01001", "B19001", "B25003") +MAX_RESPONSE_BYTES = 32 * 1024 * 1024 + + +def census_requests() -> list[dict]: + """The complete, fixed public reference request surface.""" + requests = [] + for table in TABLES: + dataset = "acs/acs1/subject" if table == "S0101" else "acs/acs1" + base = f"https://api.census.gov/data/2024/{dataset}" + for kind, url in ( + ("metadata", f"{base}/groups/{table}.json"), + ( + "data", + base + + "?" + + urlencode( + { + "get": f"group({table})", + "for": "congressional district:*", + "in": "state:*", + } + ), + ), + ): + requests.append( + { + "table": table, + "kind": kind, + "year": 2024, + "dataset": dataset, + "url": url, + } + ) + return requests + + +def strict_json(payload: bytes): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + def nonfinite(_): + raise ValueError("nonfinite JSON number") + + try: + return json.loads(payload, object_pairs_hook=pairs, parse_constant=nonfinite) + except (UnicodeError, json.JSONDecodeError): + raise ValueError("source is not valid UTF-8 JSON") from None + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *_args, **_kwargs): + return None + + +def fetch_census(url: str, api_key: str | None) -> bytes: + """Fetch a declared request, suppressing URL-bearing transport errors.""" + if url not in {item["url"] for item in census_requests()}: + raise ValueError("request is not declared") + request_url = url + if api_key: + request_url += ("&" if "?" in url else "?") + urlencode({"key": api_key}) + try: + with build_opener(_NoRedirect()).open( + Request(request_url, headers={"User-Agent": "Microcosm-CD-reference/1"}), + timeout=90, + ) as response: + payload = response.read(MAX_RESPONSE_BYTES + 1) + except HTTPError as error: + raise ValueError(f"Census request failed (HTTP {error.code})") from None + except (URLError, OSError, ValueError): + raise ValueError("Census request failed (transport error)") from None + if len(payload) > MAX_RESPONSE_BYTES: + raise ValueError("Census response exceeds size limit") + return payload + + +def read_census_key(service: str, account: str) -> str: + """Read only the explicitly requested agent-secret into process memory.""" + try: + result = subprocess.run( + ["agent-secret", "get", service, account], + capture_output=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + raise ValueError("Census credential helper failed") from None + if result.returncode or not result.stdout.strip(): + raise ValueError("Census credential unavailable") + try: + return result.stdout.decode("utf-8").strip() + except UnicodeError: + raise ValueError("Census credential is not text") from None + + +def write_json(path: Path, value) -> None: + payload = ( + json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + b"\n" + ) + if path.exists(): + if path.read_bytes() != payload: + raise ValueError(f"refusing to replace immutable artifact: {path.name}") + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_request( + root: Path, + request: dict, + *, + api_key: str | None = None, + fetch=fetch_census, + offline: bool = False, +) -> dict: + """Fetch once, pin exact response bytes, then verify on every replay.""" + if request not in census_requests(): + raise ValueError("request metadata is not declared") + root = Path(root) + request_key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{request_key}.json" + if descriptor_path.exists(): + descriptor = strict_json(descriptor_path.read_bytes()) + digest = descriptor.get("sha256", "") + expected_path = f"raw/{digest}.json" + if ( + len(digest) != 64 + or any(char not in "0123456789abcdef" for char in digest) + or descriptor.get("path") != expected_path + or any(descriptor.get(key) != value for key, value in request.items()) + ): + raise ValueError("cached request descriptor mismatch") + payload = (root / expected_path).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != digest + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("cached source digest mismatch") + strict_json(payload) + return descriptor + if offline: + raise ValueError("offline source is missing") + payload = fetch(request["url"], api_key) + if api_key and api_key.encode() in payload: + raise ValueError("response contains credential; refusing to persist") + strict_json(payload) + digest = hashlib.sha256(payload).hexdigest() + relative_path = f"raw/{digest}.json" + raw_path = root / relative_path + raw_path.parent.mkdir(parents=True, exist_ok=True) + if raw_path.exists(): + if raw_path.read_bytes() != payload: + raise ValueError("raw source digest collision") + else: + with raw_path.open("xb") as stream: + stream.write(payload) + descriptor = { + **request, + "sha256": digest, + "path": relative_path, + "size_bytes": len(payload), + "retrieved_at": datetime.now(UTC).isoformat(), + } + write_json(descriptor_path, descriptor) + return descriptor diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py new file mode 100644 index 000000000..fac70b2bc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py @@ -0,0 +1,329 @@ +"""Corrected CPS monetary leaves derived from a typed selected-money subset. + +Amounts come only from the selected current-money artifact; routing codes come +only from declared raw code columns. A raw dollar column is never an alternate +input here: the derivation refuses one outright, so a corrected leaf cannot +silently fall back to the nominal source value. + +The split fractions, the RESNSS reason routing with its age-62 fallback, the +OI_OFF 20/12 partition, the DST code-4 IRA rule and the SPM childcare grain are +the legacy modelling choices, reused unchanged and imported from +``cps_carried``/``alimony`` rather than redeclared. Only the amounts they +consume change: nominal source dollars become the 2024-basis restated amounts +the money recipe already produced. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from .alimony import US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, derive_us_alimony_from_asec +from .asec_current_money import FIELDS +from .asec_current_money_selection import SelectedCurrentMoney +from .cps_carried import ( + LONG_TERM_CAPITAL_GAIN_FRACTION, + QUALIFIED_DIVIDEND_FRACTION, + TAXABLE_INTEREST_FRACTION, + TAXABLE_PENSION_FRACTION, +) + +CPS_CARRIED_CURRENT_ROUTING_COLUMNS: tuple[str, ...] = ( + "A_AGE", + "DST_SC1", + "DST_SC1_YNG", + "DST_SC2", + "DST_SC2_YNG", + "OI_OFF", + "RESNSS1", + "RESNSS2", +) +CPS_CARRIED_CURRENT_MONEY_FIELDS: tuple[str, ...] = ( + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL", +) +CPS_CARRIED_CURRENT_PERSON_LEAVES: tuple[str, ...] = ( + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation", +) +CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: tuple[str, ...] = ( + "spm_unit_pre_subsidy_childcare_expenses", +) +CPS_CARRIED_CURRENT_CONTRACT_SCHEMA = "microcosm.us.cps-carried-current-leaves.v1" +_IRA_DISTRIBUTION_CODE = 4 +_SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK = 62 + + +class CpsCarriedCurrentRefusalError(ValueError): + """Sanitized refusal; never carries row identifiers or amounts.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CpsCarriedCurrentRefusalError(reason) + + +def _routing(routing, rows: int) -> dict[str, np.ndarray]: + _require(type(routing) is pd.DataFrame, "ROUTING_TABLE") + _require(len(routing) == rows, "ROUTING_ROW_ALIGNMENT") + columns = tuple(routing.columns) + _require(len(set(columns)) == len(columns), "ROUTING_ROSTER") + # A source dollar column can never reach this derivation, even by accident. + _require( + not any(name in FIELDS or name.endswith("_VAL") for name in columns), + "RAW_MONEY_INPUT_REFUSED", + ) + _require( + tuple(sorted(columns)) == CPS_CARRIED_CURRENT_ROUTING_COLUMNS, "ROUTING_ROSTER" + ) + values = {} + for name in CPS_CARRIED_CURRENT_ROUTING_COLUMNS: + series = routing[name] + _require(pd.api.types.is_integer_dtype(series.dtype), "ROUTING_DTYPE") + _require(not bool(series.isna().any()), "ROUTING_MISSING") + values[name] = series.to_numpy(dtype="int64", copy=True) + return values + + +def _amounts(selected: SelectedCurrentMoney, rows: int) -> dict[str, np.ndarray]: + values = {} + for name in CPS_CARRIED_CURRENT_MONEY_FIELDS: + _require(selected.entity_of(name) == "person", "MONEY_FIELD_ENTITY") + amounts = selected.amounts(name) + _require(len(amounts) == rows, "MONEY_ROW_ALIGNMENT") + _require(bool(np.isfinite(amounts).all()), "MONEY_NONFINITE") + values[name] = amounts + return values + + +def _social_security(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Reuse the legacy RESNSS reason routing and its age-62 fallback exactly.""" + reason_1 = routing["RESNSS1"] + reason_2 = routing["RESNSS2"] + age = routing["A_AGE"] + is_retirement = (reason_1 == 1) | (reason_2 == 1) + is_disability = (reason_1 == 2) | (reason_2 == 2) + is_survivor = np.isin(reason_1, [3, 5]) | np.isin(reason_2, [3, 5]) + is_dependent = np.isin(reason_1, [4, 6, 7]) | np.isin(reason_2, [4, 6, 7]) + unclassified = ( + (amount > 0) & ~is_retirement & ~is_disability & ~is_survivor & ~is_dependent + ) + threshold = _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK + return { + "social_security_retirement": np.where( + is_retirement | (unclassified & (age >= threshold)), amount, 0.0 + ), + "social_security_disability": np.where( + (is_disability & ~is_retirement) | (unclassified & (age < threshold)), + amount, + 0.0, + ), + "social_security_survivors": np.where( + is_survivor & ~is_retirement & ~is_disability, amount, 0.0 + ), + "social_security_dependents": np.where( + is_dependent & ~is_retirement & ~is_disability & ~is_survivor, amount, 0.0 + ), + } + + +def _ira_distributions(amounts, routing, rows: int) -> np.ndarray: + values = np.zeros(rows, dtype=np.float64) + for suffix in ("1", "2", "1_YNG", "2_YNG"): + code = routing[f"DST_SC{suffix}"] + values += np.where( + code == _IRA_DISTRIBUTION_CODE, amounts[f"DST_VAL{suffix}"], 0.0 + ) + return values + + +def _other_income(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Delegate the OI_OFF 20/12 partition to the registered legacy helper.""" + table = pd.DataFrame({"OI_VAL": amount, "OI_OFF": routing["OI_OFF"]}) + split = derive_us_alimony_from_asec(table) + return { + name: split[name].to_numpy(dtype="float64", copy=True) + for name in US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS + } + + +def _childcare(amount: np.ndarray, membership: np.ndarray, ids: np.ndarray): + """Carry the SPM-unit childcare value; members must already agree exactly.""" + _require( + type(membership) is np.ndarray + and membership.dtype == np.dtype("int64") + and membership.shape == amount.shape, + "SPM_MEMBERSHIP", + ) + _require( + type(ids) is np.ndarray and ids.dtype == np.dtype("int64") and ids.ndim == 1, + "SPM_IDS", + ) + _require(len(ids) == len(set(ids.tolist())), "SPM_IDS") + _require(np.array_equal(np.sort(ids), ids), "SPM_IDS") + _require(np.array_equal(np.unique(membership), ids), "SPM_MEMBERSHIP_COVERAGE") + positions = np.searchsorted(ids, membership) + values = np.zeros(len(ids), dtype=np.float64) + seen = np.zeros(len(ids), dtype=bool) + for position, value in zip(positions.tolist(), amount.tolist(), strict=True): + if seen[position]: + # The source repeats one SPM value on every member; a disagreement + # is a source contract failure, never something to reduce away. + _require(values[position] == value, "INCONSISTENT_SPM_AMOUNT") + continue + values[position] = value + seen[position] = True + return values + + +@dataclass(frozen=True) +class CpsCarriedCurrentLeaves: + """Corrected person and SPM-unit leaves, aligned to the selected rows.""" + + person: Mapping[str, np.ndarray] + spm_unit: Mapping[str, np.ndarray] + + +def derive_cps_carried_current_leaves( + selected: SelectedCurrentMoney, + *, + routing: pd.DataFrame, + spm_membership: np.ndarray, + spm_ids: np.ndarray, +) -> CpsCarriedCurrentLeaves: + """Derive the corrected monetary CPS leaves plus ``age`` from selected money.""" + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + rows = selected.person_rows + codes = _routing(routing, rows) + amounts = _amounts(selected, rows) + dividends = amounts["DIV_VAL"] + gains = amounts["CAP_VAL"] + # The money recipe already contributes zero for a declared-NIU annuity, so + # the pension base is the plain sum of the two restated amounts. + pensions = amounts["PNSN_VAL"] + amounts["ANN_VAL"] + person: dict[str, np.ndarray] = { + "age": codes["A_AGE"].astype("float64"), + "employment_income_before_lsr": amounts["WSAL_VAL"], + "self_employment_income_before_lsr": amounts["SEMP_VAL"], + "taxable_interest_income": amounts["INT_VAL"] * TAXABLE_INTEREST_FRACTION, + "qualified_dividend_income": dividends * QUALIFIED_DIVIDEND_FRACTION, + "non_qualified_dividend_income": dividends * (1 - QUALIFIED_DIVIDEND_FRACTION), + "long_term_capital_gains_before_response": gains + * LONG_TERM_CAPITAL_GAIN_FRACTION, + "short_term_capital_gains": gains * (1 - LONG_TERM_CAPITAL_GAIN_FRACTION), + "taxable_private_pension_income": pensions * TAXABLE_PENSION_FRACTION, + "tax_exempt_private_pension_income": pensions * (1 - TAXABLE_PENSION_FRACTION), + "taxable_ira_distributions": _ira_distributions(amounts, codes, rows), + "rental_income": amounts["RNT_VAL"], + "farm_operations_income": amounts["FRSE_VAL"], + "unemployment_compensation": amounts["UC_VAL"], + "health_insurance_premiums_without_medicare_part_b": amounts["PHIP_VAL"], + "other_medical_expenses": amounts["PMED_VAL"], + "over_the_counter_health_expenses": amounts["POTC_VAL"], + } + person.update(_social_security(amounts["SS_VAL"], codes)) + person.update(_other_income(amounts["OI_VAL"], codes)) + spm_unit = { + "spm_unit_pre_subsidy_childcare_expenses": _childcare( + amounts["SPM_CHILDCAREXPNS"], spm_membership, spm_ids + ) + } + _require( + tuple(sorted(person)) == CPS_CARRIED_CURRENT_PERSON_LEAVES, "PERSON_LEAF_ROSTER" + ) + _require( + tuple(sorted(spm_unit)) == CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + "SPM_UNIT_LEAF_ROSTER", + ) + for values in (*person.values(), *spm_unit.values()): + _require(values.dtype == np.dtype("float64"), "LEAF_DTYPE") + _require(bool(np.isfinite(values).all()), "LEAF_NONFINITE") + for name, values in person.items(): + _require(len(values) == rows, "LEAF_ROW_ALIGNMENT") + del name + return CpsCarriedCurrentLeaves(person, spm_unit) + + +def cps_carried_current_leaf_contract() -> dict: + """Return the recorded derivation contract for these corrected leaves.""" + return { + "schema": CPS_CARRIED_CURRENT_CONTRACT_SCHEMA, + "amount_source": "us_asec_selected_current_money", + "amount_basis": "target_current_2024", + "raw_dollar_alternates": [], + "money_fields": list(CPS_CARRIED_CURRENT_MONEY_FIELDS), + "routing_columns": list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + "person_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + "dtype": "float64", + "model_assumptions": { + "taxable_interest_fraction": TAXABLE_INTEREST_FRACTION, + "qualified_dividend_fraction": QUALIFIED_DIVIDEND_FRACTION, + "taxable_pension_fraction": TAXABLE_PENSION_FRACTION, + "long_term_capital_gain_fraction": LONG_TERM_CAPITAL_GAIN_FRACTION, + "annuity_niu_contributes_zero": True, + "signed_losses_retained": ["SEMP_VAL", "RNT_VAL", "FRSE_VAL"], + "social_security_reason_routing": "RESNSS1/RESNSS2 with age-62 fallback", + "social_security_age_fallback": _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK, + "other_income_split": "derive_us_alimony_from_asec on OI_VAL/OI_OFF", + "ira_distribution_code": _IRA_DISTRIBUTION_CODE, + "spm_childcare_grain": "spm_unit_repeated_on_person, member-consistent", + }, + "release_eligible": False, + } + + +__all__ = [ + "CPS_CARRIED_CURRENT_CONTRACT_SCHEMA", + "CPS_CARRIED_CURRENT_MONEY_FIELDS", + "CPS_CARRIED_CURRENT_PERSON_LEAVES", + "CPS_CARRIED_CURRENT_ROUTING_COLUMNS", + "CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES", + "CpsCarriedCurrentLeaves", + "CpsCarriedCurrentRefusalError", + "cps_carried_current_leaf_contract", + "derive_cps_carried_current_leaves", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py new file mode 100644 index 000000000..85d39944f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py @@ -0,0 +1,330 @@ +"""National demographic calibration with registry-bound graph diagnostics. + +This development seam consumes explicitly prepared household count columns. +It does not acquire references, authorize their use, define their universe, or +turn a calibration into national/CD certification. In particular, a captured +reference bundle with targets_allowed=False is not a TargetRegistry. + +The complete declared registry travels in the node key and diagnostics retain +its source, period and family. Standard errors are reported, not used by the +current Adam loss. Reserved B19001 and B25003 observations never enter this +seam. The old calibrate.adam@1 contract and its receipts are unchanged. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from dataclasses import asdict + +import numpy as np +from pandas.api.types import is_complex_dtype, is_numeric_dtype + +import microcosm.build.cd_benchmark.protocol as protocol_module +import microcosm.calibrate._target_loss_attribution as attribution_module +import microcosm.calibrate.diagnostics as diagnostics_module +import microcosm.calibrate.kernels as kernels_module +import microcosm.calibrate.monetary_binding as monetary_binding_module +import microcosm.calibrate.registry as registry_module +import microcosm.calibrate.solve as solve_module +from microcosm.build.cd_benchmark.protocol import RESERVED_FAMILIES +from microcosm.calibrate import calibrate, diagnostics_payload +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + SeedSource, + Slice, + StructuralDelta, + WeightTransition, + source_hash, +) + +DIAGNOSTICS_TYPE = ArtifactType( + "microcosm.calibration_diagnostics", + diagnostics_module.CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, +) +_OUTPUTS = (ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE),) +_PARAMS = frozenset( + { + "registry", + "epochs", + "learning_rate", + "max_weight_ratio", + "max_initial_weight_ratio", + "mass", + "weight_anchor", + } +) +_METADATA = frozenset( + {"table", "reference_sha256", "geography", "universe", "role", "evidence_scope"} +) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _registry_json(registry: TargetRegistry) -> str: + return _json( + {"country": registry.country, "specs": [asdict(s) for s in registry]} + ).decode() + + +def _validate_registry(registry: TargetRegistry) -> None: + if registry.country != "us" or not len(registry): + raise ValueError("A nonempty US demographic registry is required.") + scopes = set() + for spec in registry: + metadata = spec.metadata + table = metadata.get("table") + if table in RESERVED_FAMILIES or spec.family in RESERVED_FAMILIES: + raise ValueError("Income and tenure tables are reserved holdouts.") + if table not in {"S0101", "B01001"} or spec.family != f"acs.{table}": + raise ValueError( + "Only explicit ACS S0101/B01001 demographic families are supported." + ) + cell = re.fullmatch( + r"S0101_C01_(\d{3})" if table == "S0101" else r"B01001_(\d{3})", spec.name + ) + if cell is None or not 1 <= int(cell[1]) <= (19 if table == "S0101" else 49): + raise ValueError( + "A supported count cell must identify each demographic target." + ) + if ( + set(metadata) != _METADATA + or metadata["role"] != "calibration" + or metadata["geography"] != "0100000US" + or metadata["universe"] != "population" + or metadata["evidence_scope"] not in {"invented", "source_documented"} + or re.fullmatch(r"[a-f0-9]{64}", metadata["reference_sha256"]) is None + or spec.entity != "household" + or spec.period != "2024" + or spec.signed + or spec.value < 0 + or spec.filter is not None + or spec.tolerance is not None + or (spec.se is not None and not np.isfinite(spec.se)) + ): + raise ValueError( + "Demographic target scope must be explicit national population counts for 2024." + ) + scopes.add(metadata["evidence_scope"]) + if len(scopes) != 1: + raise ValueError( + "Invented and source-documented evidence scopes cannot be mixed." + ) + + +def _registry_from_json(text: str) -> TargetRegistry: + if not isinstance(text, str) or len(text.encode()) > 1_048_576: + raise ValueError("A bounded canonical registry declaration is required.") + document = json.loads(text) + if not isinstance(document, dict) or set(document) != {"country", "specs"}: + raise ValueError("Unsupported registry declaration shape.") + registry = TargetRegistry( + [TargetSpec(**raw) for raw in document["specs"]], country=document["country"] + ) + if text != _registry_json(registry): + raise ValueError( + "Registry declaration must have its canonical typed representation." + ) + _validate_registry(registry) + return registry + + +def _solver_options(params) -> dict: + if ( + set(params) != _PARAMS + or params["mass"] != "free" + or params["weight_anchor"] != "design" + ): + raise ValueError( + "The development calibration requires free mass and an original design cap." + ) + if type(params["epochs"]) is not int or params["epochs"] < 1: + raise ValueError("epochs must be a positive integer.") + for key, minimum in ( + ("learning_rate", 0), + ("max_weight_ratio", 1), + ("max_initial_weight_ratio", 1), + ): + value = params[key] + if ( + type(value) not in {int, float} + or not np.isfinite(value) + or value < minimum + or (key == "learning_rate" and value == 0) + ): + raise ValueError(f"Unsupported {key}.") + return { + **{key: params[key] for key in ("epochs", "learning_rate", "mass")}, + "max_weight_ratio": params["max_initial_weight_ratio"], + } + + +def demographic_calibration_node( + registry: TargetRegistry, + *, + base: str, + node_id: str = "national.demographic_calibration", + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, +) -> Node: + """Freeze a caller's explicit registry and declare its actual count inputs. + + Source citations/digests are declarations here, not independently verified + acquisitions or permission to repurpose held-out evidence. Activating a + genuine registry remains a separate, predeclared build decision. + + max_initial_weight_ratio bounds the solver relative to incoming importance + weights. max_weight_ratio separately guards the result relative to original + design weights in the executor. These anchors need not be equal. The kernel + cannot inspect undeclared design weights; a violated design cap refuses the + graph result rather than silently changing the solver's answer. + """ + frozen = _registry_from_json(_registry_json(registry)) + params = dict( + registry=_registry_json(frozen), + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + mass="free", + weight_anchor="design", + ) + _solver_options(params) + return Node( + id=node_id, + kernel=DemographicCalibrationKernel.ref, + inputs=(Slice("household", tuple(dict.fromkeys(s.measure for s in frozen))),), + params=params, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + artifact_outputs=_OUTPUTS, + ) + + +class DemographicCalibrationKernel(KernelBase): + """Call the real solver and persist its complete v6 diagnostic artifact.""" + + ref = "us.demographic_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self) -> str: + return hashlib.sha256( + _json( + { + "adapter": source_hash( + sys.modules[__name__], + registry_module, + diagnostics_module, + attribution_module, + solve_module, + kernels_module, + monetary_binding_module, + protocol_module, + dependencies=self.capabilities.dependencies, + ), + "solver": kernels_module.CALIBRATE_ADAM.implementation_hash(), + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + registry = _registry_from_json(context.params["registry"]) + options = _solver_options(context.params) + expected = demographic_calibration_node( + registry, + base=context.node.base, + node_id=context.node.id, + **{ + k: context.params[k] + for k in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "max_initial_weight_ratio", + ) + }, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Calibration node differs from its complete declaration.") + if context.weights["household"].kind is not WeightKind.IMPORTANCE: + raise ValueError("Calibration requires the declared importance weights.") + if not np.all(context.weights["household"].values > 0): + raise ValueError("Calibration requires strictly positive incoming weights.") + table = context.tables["household"] + for spec in registry: + dtype = table[spec.measure].dtype + if not is_numeric_dtype(dtype) or is_complex_dtype(dtype): + raise ValueError("Demographic measures require numeric count columns.") + values = table[spec.measure].to_numpy(dtype=float, na_value=np.nan) + if not np.all( + np.isfinite(values) & (values >= 0) & (values == np.floor(values)) + ): + raise ValueError( + "Demographic measures must be nonnegative finite integer counts." + ) + frame = kernels_module._frame_from_context(context, "household") + result = calibrate( + frame, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + **options, + ) + if result.skipped: + raise ValueError("Demographic calibration cannot skip declared targets.") + anchors = { + "solver_weight_anchor": "incoming_importance", + "solver_max_weight_ratio": context.params["max_initial_weight_ratio"], + "executor_weight_anchor": "original_design", + "executor_max_weight_ratio": context.params["max_weight_ratio"], + } + payload = _json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + return KernelResult( + weights=result.frame.weights_for("household"), + artifacts={"diagnostics": payload}, + receipt={ + "scope": "national_demographic_calibration_development", + "evidence_scope": registry.specs[0].metadata["evidence_scope"], + "registry_sha256": hashlib.sha256( + context.params["registry"].encode() + ).hexdigest(), + "registry_version": registry.version, + "diagnostics_sha256": hashlib.sha256(payload).hexdigest(), + "reserved_families": sorted(RESERVED_FAMILIES), + "consumes_standard_errors": False, + **anchors, + "release_eligible": False, + "national_validation": "not_evaluated", + "congressional_district_validation": "not_evaluated", + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py new file mode 100644 index 000000000..fd0db53e8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py @@ -0,0 +1,528 @@ +"""Complete canonical PUF enrichment boundaries; no source admission is issued. + +An upstream source owner supplies interpreted, period-aligned canonical donor +columns and a complete survey recipient. This module does not read raw PUF, +reinterpret E19200, choose missing-value aliases, or manufacture source evidence. +It reuses the maintained ordered QRF protocol and person/tax-unit finalizer. +Graph hosts must authenticate actual typed edges and retain their independent +full-population replay checks. These value checks do not replace either duty. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import decode_matrix_apply_state +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import Frame + +from . import puf_support as support + +PERSON_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS) +TAX_UNIT_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS) +TARGETS = (*PERSON_OUTPUTS, *TAX_UNIT_OUTPUTS) +PREDICTORS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS) +PHASE = "us_full_canonical_puf_enrichment" + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _numeric(values, *, label, boolean=False, nullable=False): + """Validate physical values before conversion; never parse strings as money.""" + series = pd.Series(values, copy=False) + if not nullable: + _require(not series.isna().any(), f"PUF_UNKNOWN:{label}") + observed = series.dropna() + if boolean: + valid = observed.map(lambda v: isinstance(v, (bool, np.bool_))) + else: + valid = observed.map( + lambda v: ( + not isinstance(v, (bool, np.bool_)) + and isinstance(v, (int, float, np.integer, np.floating)) + ) + ) + _require(bool(valid.all()), f"PUF_PHYSICAL_TYPE:{label}") + if not boolean: + # Do not let integer source identities/amounts silently lose bits at + # the explicit float64 model boundary. + _require( + all( + not isinstance(v, (int, np.integer)) or abs(int(v)) <= 2**53 + for v in observed + ), + f"PUF_FLOAT64_INTEGER_RANGE:{label}", + ) + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + bool(np.isfinite(numeric[~series.isna().to_numpy()]).all()), + f"PUF_NONFINITE:{label}", + ) + return numeric + + +def _ids(values, label): + series = pd.Series(values, copy=False) + _require(series.dtype == np.dtype("int64"), f"PUF_ID_DTYPE:{label}") + result = series.to_numpy(copy=True) + _require(bool((result > 0).all()), f"PUF_ID_DOMAIN:{label}") + return result + + +def _known(values, known, columns, label): + _require( + isinstance(known, pd.DataFrame) + and known.index.equals(values.index) + and tuple(known.columns) == tuple(columns), + f"PUF_KNOWNNESS_AXIS:{label}", + ) + for column in columns: + mask = known[column] + _numeric(mask, label=f"{label}.{column}.known", boolean=True) + _require(bool(mask.all()), f"PUF_UNKNOWN:{label}.{column}") + + +def canonical_full_puf_donor( + person, + tax_unit, + *, + person_known, + tax_unit_known, + person_targets_at_tax_unit=(), +): + """Reduce canonical columns after upstream interpretation, with no imputation. + + Some person destinations are observed only as return totals, for example + self-employed pension contributions. Such targets must be explicitly listed + in ``person_targets_at_tax_unit`` and supplied on the tax-unit table. A + destination's eventual grain never changes the declared donor grain. + Knownness covers every consumed nonstructural field, including true zeros. + Raw-to-canonical judgments (including mortgage decomposition) belong upstream. + When all person destinations are return totals, ``person=None`` avoids + inventing donor persons. Supply the separately interpreted canonical + ``tax_unit_person_count`` on the return table in that case. + """ + _require( + type(person_targets_at_tax_unit) is tuple + and len(set(person_targets_at_tax_unit)) == len(person_targets_at_tax_unit) + and set(person_targets_at_tax_unit) <= set(PERSON_OUTPUTS), + "PUF_DONOR_GRAIN_DECLARATION", + ) + pcols = tuple(c for c in PERSON_OUTPUTS if c not in person_targets_at_tax_unit) + return_only = person is None + _require( + not return_only + or (person_targets_at_tax_unit == PERSON_OUTPUTS and person_known is None), + "PUF_RETURN_ONLY_GRAIN", + ) + tcols = ( + "weight", + "filing_status_code", + *(("tax_unit_person_count",) if return_only else ()), + *person_targets_at_tax_unit, + *TAX_UNIT_OUTPUTS, + ) + tables = [(tax_unit, ("tax_unit_id", *tcols), "tax_unit")] + if not return_only: + tables.append((person, ("person_id", "person_tax_unit_id", *pcols), "person")) + for table, required, label in tables: + _require( + isinstance(table, pd.DataFrame) + and table.index.is_unique + and table.columns.is_unique + and set(required) <= set(table.columns), + f"PUF_DONOR_COLUMNS:{label}", + ) + _require( + (return_only or not set(person_targets_at_tax_unit) & set(person.columns)) + and not set(pcols) & set(tax_unit.columns), + "PUF_DONOR_GRAIN_COLLISION", + ) + tids = _ids(tax_unit.tax_unit_id, "tax_unit") + _require( + len(tids) > 0 and len(set(tids)) == len(tids), + "PUF_DONOR_MEMBERSHIP", + ) + if not return_only: + pids = _ids(person.person_id, "person") + links = _ids(person.person_tax_unit_id, "person_tax_unit_id") + _require( + len(set(pids)) == len(pids) and set(links) == set(tids), + "PUF_DONOR_MEMBERSHIP", + ) + _known(person, person_known, pcols, "person") + _known(tax_unit, tax_unit_known, tcols, "tax_unit") + # Validate EVERY source cell before groupby can skip a missing member. + if not return_only: + normalized = pd.DataFrame(index=person.index) + for column in pcols: + normalized[column] = _numeric( + person[column], + label=f"person.{column}", + boolean=column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS, + ) + grouped = normalized.groupby(links, sort=False).sum(min_count=1).reindex(tids) + count = pd.Series(links).value_counts(sort=False).reindex(tids).to_numpy() + else: + count = _numeric(tax_unit.tax_unit_person_count, label="tax_unit_person_count") + _require( + bool(((count >= 1) & (count == np.floor(count))).all()), + "PUF_PERSON_COUNT_DOMAIN", + ) + donor = pd.DataFrame(index=tax_unit.index) + for column in (*person_targets_at_tax_unit, *TAX_UNIT_OUTPUTS): + donor[column] = _numeric(tax_unit[column], label=f"tax_unit.{column}") + for column in pcols: + donor[column] = _numeric(grouped[column], label=f"reduced.{column}") + status = _numeric(tax_unit.filing_status_code, label="filing_status_code") + _require(bool(np.isin(status, [1, 2, 3, 4, 5]).all()), "PUF_FILING_STATUS_DOMAIN") + for column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + values = donor[column].to_numpy() + _require( + bool( + ((values >= 0) & (values <= count) & (values == np.floor(values))).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS: + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + # These are canonical arithmetic aliases, after complete cell validation. + feature_values = ( + status, + count.astype(np.float64), + donor.employment_income_before_lsr.to_numpy(), + donor.self_employment_income_before_lsr.to_numpy(), + donor.taxable_interest_income.to_numpy(), + donor.qualified_dividend_income.to_numpy() + + donor.non_qualified_dividend_income.to_numpy(), + donor.short_term_capital_gains.to_numpy(), + donor.long_term_capital_gains_before_response.to_numpy(), + ) + for name, values in zip(PREDICTORS, feature_values, strict=True): + donor[name] = _numeric(values, label=name) + donor["weight"] = _numeric(tax_unit.weight, label="weight") + weights = donor.weight.to_numpy() + _require( + bool((weights >= 0).all()) and np.isfinite(weights.sum()) and weights.sum() > 0, + "PUF_DONOR_WEIGHT_DOMAIN", + ) + return donor.loc[:, [*PREDICTORS, *TARGETS, "weight"]].copy() + + +def _recipient_matrix(frame, *, predictor_known): + tax_unit, person = frame.table("tax_unit"), frame.table("person") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + ids = _ids(tax_unit.loc[mask, "tax_unit_id"], "recipient") + selected_index = pd.Index(ids, name="tax_unit_id") + _require(len(ids) > 0, "PUF_NO_RECIPIENTS") + _require( + isinstance(predictor_known, pd.DataFrame) + and predictor_known.index.equals(selected_index), + "PUF_RECIPIENT_KNOWNNESS_AXIS", + ) + _known(predictor_known, predictor_known, PREDICTORS, "recipient_predictor") + person_mask = person.person_tax_unit_id.isin(ids) + # The maintained resolver below owns ACS source applicability and its exact + # age-15 universe zeros. Screen physical numeric types first, since pandas + # numeric conversion otherwise accepts strings, booleans or datetime values. + for name in PREDICTORS: + plan = support._strict_predictor_source_plan( + name, tax_unit=tax_unit, person=person + ) + if plan.source_column == "filing_status_code" or plan.entity == "derived": + continue + table = person if plan.entity == "person" else tax_unit + rows = person_mask if plan.entity == "person" else mask + for column in plan.columns: + if column in table: + _numeric( + table.loc[rows, column], + label=f"recipient.{plan.entity}.{column}", + nullable=True, + ) + features, universe = support._strict_recipient_predictor_surface( + frame, mask, PREDICTORS, person_outputs=PERSON_OUTPUTS + ) + selected = features.loc[mask, list(PREDICTORS)].copy() + selected.index = selected_index + selected = selected.astype("float64") + return ( + model_input.encode_recipient_matrix( + selected, entity="tax_unit", entity_ids=ids + ), + universe, + ) + + +@dataclass(frozen=True) +class FullPufInputs: + donor: pd.DataFrame + donor_frame: Frame + matrix: bytes + recipient_universe: Mapping[str, object] + + +def prepare_full_puf_inputs(frame, donor, *, predictor_known): + """Prepare the full input surfaces; absence never falls back to legacy zeros.""" + _require( + isinstance(donor, pd.DataFrame) + and tuple(donor.columns) == (*PREDICTORS, *TARGETS, "weight"), + "PUF_FULL_DONOR_ROSTER", + ) + for column in donor: + _numeric(donor[column], label=f"donor.{column}") + counts = donor[PREDICTORS[1]].to_numpy() + weights = donor["weight"].to_numpy() + _require( + len(donor) > 0 + and donor.index.is_unique + and bool(((counts >= 1) & (counts == np.floor(counts))).all()) + and bool(np.isin(donor[PREDICTORS[0]], [1, 2, 3, 4, 5]).all()) + and bool((weights >= 0).all()) + and np.isfinite(weights.sum()) + and weights.sum() > 0, + "PUF_DONOR_MODEL_DOMAIN", + ) + for column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + values = donor[column].to_numpy() + _require( + bool( + ( + (values >= 0) & (values <= counts) & (values == np.floor(values)) + ).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS: + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + matrix, universe = _recipient_matrix(frame, predictor_known=predictor_known) + # Every input is now physically numeric, finite, and known. The maintained + # helper constructs the actual weighted donor Frame and repeats its strict + # source checks; its historical fillna operation has no missing values to fill. + inputs = support.prepare_us_puf_tax_detail_chain_inputs( + frame, + donor, + predictors=PREDICTORS, + person_outputs=PERSON_OUTPUTS, + tax_unit_outputs=TAX_UNIT_OUTPUTS, + require_complete_recipient_predictors=True, + ) + _require( + inputs.donor.equals(donor) and inputs.recipient_predictor_universe == universe, + "PUF_PREPARATION_CHANGED_CANONICAL_INPUT", + ) + return FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe) + + +def full_puf_train_apply_nodes( + *, + donor_population, + recipient_population, + matrix_producer, + seed, + n_estimators, + zero_atol, + prefix="puf_full", +): + """Declare all 65 real fits/draws; each draw reads its raw predecessors.""" + fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor_population, + entity="tax_unit", + predictors=PREDICTORS, + targets=TARGETS, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + phase=PHASE, + ) + applies = legacy_qrf_apply_matrix_nodes( + prefix + ".apply", + population=recipient_population, + fit_nodes=fits, + matrix_producer=matrix_producer, + seed=seed, + phase=PHASE, + ) + return fits, applies + + +def decode_full_puf_draws( + *, matrix, matrix_producer_key, raw_draws, apply_state, training_state, seed +): + """Check actual typed-edge payloads against the complete model/raw history.""" + _require(codec._hash(matrix_producer_key), "PUF_MATRIX_PRODUCER_KEY") + prepared = model_input.decode_recipient_matrix(matrix) + packet = decode_matrix_apply_state(apply_state) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + training, fitted = codec.read_training(training_state) + fitted_values = fitted.to_dict() + applied_values = chain.to_dict() + _require( + prepared.entity == chain.entity == "tax_unit" + and tuple(prepared.features.columns) == tuple(chain.predictors) == PREDICTORS + and tuple(chain.targets) == tuple(chain.completed_targets) == TARGETS + and tuple(fitted_values["completed_targets"]) == TARGETS + and all( + fitted_values[key] == applied_values[key] + for key in ( + "predictors", + "targets", + "entity", + "weight_kind", + "weight_sha256", + "model_config", + "donor_index", + ) + ) + and application["models"] == training["models"] + and application["seed"] == seed + and chain.recipient_index == qrf._index_identity(prepared.features.index), + "PUF_FULL_CHAIN_IDENTITY", + ) + _require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "PUF_FULL_MATRIX_BINDING", + ) + _require( + isinstance(raw_draws, Mapping) and tuple(raw_draws) == TARGETS, "PUF_RAW_ROSTER" + ) + _require( + application["raw_targets"] + == [ + {"target": target, "sha256": codec.sha(raw_draws[target])} + for target in TARGETS + ], + "PUF_RAW_HISTORY", + ) + return pd.DataFrame( + { + target: codec.read_raw_target( + raw_draws[target], target=target, index=prepared.features.index + ) + for target in TARGETS + }, + index=prepared.features.index, + ) + + +def finalize_full_puf( + frame, + donor, + *, + predictor_known, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + last_model, + seed, +): + """Use actual finalizer judgments only after the complete raw chain validates. + + Returns a candidate Frame plus tail-cap/universe evidence. A graph placement + owner must expose every changed person/tax-unit column, attach only on the + PUF masks, and run the existing full-population replay verifier afterwards. + This function itself never creates a live Population/source qualification. + """ + inputs = prepare_full_puf_inputs(frame, donor, predictor_known=predictor_known) + _require(matrix == inputs.matrix, "PUF_RECIPIENT_MATRIX_CHANGED") + raw = decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=matrix_producer_key, + raw_draws=raw_draws, + apply_state=apply_state, + training_state=training_state, + seed=seed, + ) + # The final fitted target consumed every preceding donor outcome. Binding + # its actual trusted producer bytes therefore checks the entire donor value + # surface, including fields used only by sparsity/tail finalization. A donor + # with the same index/weights but altered values cannot adjust these draws. + training, fitted_state = codec.read_training(training_state) + last = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + last_model, expected_sha256=training["models"][-1]["sha256"] + ) + _require( + last.target == TARGETS[-1] + and last.next_training_state == fitted_state + and last.training_id == training["models"][-1]["training_id"], + "PUF_FINAL_MODEL_BINDING", + ) + resolved = qrf._resolve_qrf_fit_input( + inputs.donor_frame, list(PREDICTORS), list(TARGETS), "design" + ) + qrf.RegimeGatedQRF._validate_chain_donor(last.training_state._chain(), resolved) + _require( + last.donor_sha256 + == qrf_target._consumed_values_sha256(resolved.table, (*PREDICTORS, *TARGETS)), + "PUF_DONOR_CONSUMED_BYTES", + ) + mask = support.puf_tax_detail_clone_mask(frame.table("tax_unit"), entity="tax_unit") + # Explicit adapter between separate graph entity-ID and pandas row indexes; + # matrix recomputation above proves the ordered IDs before this relabeling. + raw.index = frame.table("tax_unit").index[mask] + caps = [] + result = support.finalize_us_puf_tax_detail_predictions( + frame, + inputs.donor, + raw.copy(deep=True), + person_outputs=PERSON_OUTPUTS, + tax_unit_outputs=TAX_UNIT_OUTPUTS, + tail_bound_diagnostics=caps, + absent_cells=support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + return result, { + "scope": PHASE, + "target_order": list(TARGETS), + "recipient_universe": dict(inputs.recipient_universe), + "tail_bounds": caps, + "matrix_sha256": codec.sha(matrix), + "raw_target_sha256": { + target: codec.sha(raw_draws[target]) for target in TARGETS + }, + "source_admission_issued": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py new file mode 100644 index 000000000..addadd581 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py @@ -0,0 +1,665 @@ +"""Typed PAW transport and reported-income accounting, never source authority. + +The six results are source observations and an accounting residual. They are +not simulated SSI/TANF, ACS other income, or a completed eight-category bridge. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_income_observations as source +from . import graph_context +from .asec_current_money import MoneyRefusalError, _json, _parse, _require, _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + SelectedCurrentMoney, + decode_selected_current_money, + encode_selected_current_money, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +US_ASEC_INCOME_OBSERVATIONS_TYPE = ArtifactType( + "microcosm.us.asec_income_observations", 1 +) +US_ASEC_REPORTED_INCOME_TYPE = ArtifactType("microcosm.us.asec_reported_income", 3) +STAGE = "asec_prepared_v3" +REPORTED_INCOME_NODE = "asec_prepared.reported_income" +RESULT_COLUMNS = tuple( + "asec_reported_" + name + "_2024_price" + for name in ( + "wage_income", + "net_self_employment_income", + "ssi", + "cash_public_assistance", + "total_income", + "income_unallocated", + ) +) +CONTRIBUTING_FIELDS = ("WSAL_VAL", "SEMP_VAL", "FRSE_VAL", "SSI_VAL", "PTOTVAL") +COORDINATES = ("person_id", "income_year", "A_AGE") +EVIDENCE_COLUMNS = tuple( + f"{name}.{axis}" + for name in CONTRIBUTING_FIELDS + for axis in ("status", "validity", "zero_origin") +) + ( + "PAW_VAL.status", + "PAW_VAL.validity", + "PAW_VAL.zero_origin", + "PAW_VAL.source_encoding_class", +) +ACCOUNTING_COLUMNS = COORDINATES + RESULT_COLUMNS + EVIDENCE_COLUMNS +ACCOUNTING_DTYPES = tuple([" BoundIncomeObservations: + """Admit graph transport only against the independently issued CREATE receipt.""" + try: + _require( + type(payload) is bytes + and len(source.MAGIC) + 4 + 32 + < len(payload) + <= source.INCOME_PAYLOAD_MAX_BYTES, + "BOUND_INCOME_SIZE", + ) + binding = prepared_receipt["income_observations"] + _require( + set(binding) == _BINDING_KEYS + and prepared_receipt["schema"] == "microcosm.us.asec_prepared_receipt.v3" + and prepared_receipt["source_kind"] == "us_asec_prepared_current_money_v3" + and prepared_receipt["release_eligible"] is False + and binding["kind"] == source.ARTIFACT_KIND + and binding["columns"] == list(source.COLUMNS), + "BOUND_INCOME_PREPARATION", + ) + _require(_sha(payload) == binding["payload_sha256"], "BOUND_INCOME_PAYLOAD") + _require( + payload.startswith(source.MAGIC) + and _sha(payload[:-32]) == payload[-32:].hex(), + "BOUND_INCOME_FRAMING", + ) + length = struct.unpack_from("= 0).all()) + and len(np.unique(positions)) == len(positions) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())) + and _sha(positions.tobytes()) == person_positions_sha256, + "ACCOUNTING_SOURCE_POSITIONS", + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions]), + "ACCOUNTING_SOURCE_COHORTS", + ) + _require( + all(selected.entity_of(name) == "person" for name in CONTRIBUTING_FIELDS), + "ACCOUNTING_FIELD_ENTITY", + ) + w = selected.field("WSAL_VAL").amounts + e = selected.field("SEMP_VAL").amounts + selected.field("FRSE_VAL").amounts + s = selected.field("SSI_VAL").amounts + p = income.array("PAW_VAL_2024_price")[positions] + t = selected.field("PTOTVAL").amounts + u = t - (((w + e) + s) + p) + results = (w, e, s, p, t, u) + _require(all(np.isfinite(a).all() for a in results), "ACCOUNTING_NONFINITE") + age = income.array("A_AGE")[positions] + nominal, yn = income.array("PAW_VAL")[positions], income.array("PAW_YN")[positions] + buffers = [ + a.astype(" KernelResult: + from . import graph_asec_prepared as bridge + + bridge._phase(context) + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.NONE + and not node.sources, + "ACCOUNTING_NODE", + ) + _require( + node.inputs == (Slice("person", ("source_year",)),) + and node.outputs == reported_income_declarations(), + "ACCOUNTING_DECLARATIONS", + ) + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "ACCOUNTING_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "ACCOUNTING_ARTIFACT_OUTPUTS", + ) + values = { + name: bridge._artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("selection", US_ASEC_SELECTION_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + selection_producer = bridge._same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + source_producer = bridge._same_producer( + context, ("prepared_receipt", "income_observations") + ) + document = bridge._document(values["frame_context"]) + person = context.tables["person"] + bridge._check_identity(document, "person", person) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + prepared = _parse(values["prepared_receipt"].payload) + selection = _parse(values["selection"].payload) + _require( + canonical_json(prepared) == values["prepared_receipt"].payload + and canonical_json(selection) == values["selection"].payload + and selection["schema"] == bridge.SELECTION_SCHEMA, + "ACCOUNTING_RECEIPTS", + ) + _require( + selection["parent"]["producer_key"] == source_producer + and selection["parent"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "ACCOUNTING_SELECTION_PARENT", + ) + for entity, declared in document["entities"].items(): + _require( + all( + selection["entities"][entity][key] == declared[key] + for key in ("rows", "ordered_ids_sha256") + ), + "ACCOUNTING_SELECTION_IDENTITY", + ) + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["selection"].payload), + ) + _require( + selected.header_data["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and selected.header_data["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "ACCOUNTING_MONEY_COORDINATES", + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person.source_year.dtype == np.dtype("int64"), + "ACCOUNTING_SOURCE_YEAR_DTYPE", + ) + result = derive_reported_income( + selected, + income, + person_ids=person.person_id.to_numpy(), + income_years=person.source_year.to_numpy(), + person_positions_sha256=selection["person_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["selection"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=selection_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person.person_id.to_numpy(), name="person_id") + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype="float64" + ) + for name in RESULT_COLUMNS + } + document["entities"]["person"]["columns"].extend(RESULT_COLUMNS) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": bridge.ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(STAGE), + "accounting_sha256": _sha(payload), + "arithmetic_order": ARITHMETIC_ORDER, + "person_rows": len(person), + "release_eligible": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py new file mode 100644 index 000000000..db6a84363 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py @@ -0,0 +1,1136 @@ +"""The five-node ASEC slice: source, selection, leaves, engine and reported income. + +One directory source holds the reviewed restoration inputs. The CREATE kernel +prepares the whole authenticated population from it; a FILTER kernel draws a +seeded whole-household engineering sample and slices the money body to exactly +the selected coordinates; a pure kernel derives the corrected monetary CPS +leaves from that slice alone; and a final kernel runs the real PolicyEngine-US +adapter for the four outputs whose complete input closure this slice produces. + +Two boundaries are load-bearing. The authenticated ``ReadyCurrentMoney`` never +leaves the CREATE process: what travels is its canonical encoded body, typed as +a subset artifact that no ``type(x) is ReadyCurrentMoney`` check accepts. And +engine outputs are formula-owned, so they are retained in an evaluation +artifact and never written back as population cells. + +Nothing here is a release, a calibration, a representative sample, or a tax or +benefit score. Every receipt says so. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + +import numpy as np +import pandas as pd + +from microcosm.build.frame_sampling import EXACT_COUNT_RULE, sample_frame_households +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, Weights +from microcosm.frame.adapters.policyengine_us import ( + PolicyEngineUSEngine, + PolicyEngineUSVariableMetadataIndex, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_housing_status_source as housing_source +from . import graph_context +from .asec_current_money import _sha +from .asec_current_money_graph_resources import load_graph_current_money_consumers +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + decode_selected_current_money, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_engine_evaluation import ( + ADMITTED_ROOTS, + BLOCKED_ENGINE_OUTPUTS, + US_ASEC_ENGINE_EVALUATION_TYPE, + admit_engine_outputs, + encode_engine_evaluation, + engine_runtime_identity, + materialize_engine_outputs, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + REPORTED_INCOME_NODE, + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + USAsecReportedIncomeKernel, + reported_income_declarations, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ASEC_PREPARED_STAGE = "asec_prepared_v3" +ASEC_PREPARED_PHASE = "prepare_asec_current_money_slice" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASEC_PREPARED_SOURCE_NAME = "asec_prepared" +ASEC_PREPARED_SOURCE = SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three housing " + "cohort HDFs and three official Census PERSON CSV members.", +) +ASEC_PREPARED_DEPENDENCIES = STAGE_DEPENDENCIES[ASEC_PREPARED_STAGE] + +PREFIX = "asec_prepared" +CREATE_NODE = f"{PREFIX}.create" +SELECT_NODE = f"{PREFIX}.select_households" +LEAVES_NODE = f"{PREFIX}.cps_carried_current" +ENGINE_NODE = f"{PREFIX}.engine_current_money" + +#: The one household column the selection and engine nodes declare. They need +#: the household table and its design weights; the executor exposes an entity's +#: weights only to a node that declares that entity. This is the first column +#: the reviewed housing attachment guarantees, so a changed attachment fails +#: closed here instead of silently reading something else. It is a carrier: it +#: never reaches the corrected leaves and never reaches the engine frame. +HOUSEHOLD_WEIGHT_CARRIER = housing_source.ATTACHED_COLUMNS[0] + +SELECTION_SCHEMA = "microcosm.us.asec_household_selection.v2" +SELECTION_SOURCE_LABEL = "US ASEC prepared current money" +SELECTION_STRATUM_COLUMN = "source_year" +LEAF_DTYPE = "float64" +_ENTITIES = US_SCHEMA.entities +_PERSON_ID = US_SCHEMA.entity_id_column("person") + + +class PreparedGraphError(ValueError): + """A declaration, binding or identity this slice refuses to execute on.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _artifact(context: KernelContext, name: str, expected): + """Return one declared typed artifact, refusing an aliased or retyped edge.""" + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and len(edges) == 1 + and edges[0].type == expected + and value.type == expected, + f"ARTIFACT_EDGE:{name}", + ) + _require( + isinstance(value.producer_key, str) and len(value.producer_key) == 64, + f"ARTIFACT_PRODUCER:{name}", + ) + return value + + +def _same_producer(context: KernelContext, names: Sequence[str]) -> str: + """Require a set of edges to come from one actual producing node execution.""" + producers = {context.artifacts[name].producer_key for name in names} + edges = {edge.name: edge.producer for edge in context.node.artifact_inputs} + _require(len(producers) == 1, "ARTIFACT_PRODUCER_SPLIT") + _require(len({edges[name] for name in names}) == 1, "ARTIFACT_PRODUCER_SPLIT") + return next(iter(producers)) + + +def _phase(context: KernelContext, extra: tuple[str, ...] = ()) -> None: + _require(set(context.params) == {"phase", *extra}, "NODE_PARAMS") + _require(context.params["phase"] == ASEC_PREPARED_PHASE, "NODE_PHASE") + + +def _document(value) -> dict: + document = graph_context._decode(value.payload) + graph_context._mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require(document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY") + return document + + +def _identity(ids: np.ndarray, entity: str) -> dict[str, object]: + """Row identity of an id vector, using the graph context's own digest.""" + column = US_SCHEMA.entity_id_column(entity) + return graph_context._row_identity( + pd.DataFrame({column: np.asarray(ids, dtype="int64")}), entity + ) + + +def _check_identity(document: dict, entity: str, table: pd.DataFrame) -> None: + declared = document["entities"][entity] + actual = graph_context._row_identity(table, entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"CONTEXT_IDENTITY:{entity}", + ) + _require( + not set(table.columns) - set(declared["columns"]), f"CONTEXT_COLUMNS:{entity}" + ) + + +def _group_ids(person: pd.DataFrame, group: str) -> np.ndarray: + """The exact id inventory a group table holds for these persons. + + ``Frame`` validates that a group table's ids are the sorted distinct values + of the person membership column, and ``Frame.select`` prunes to exactly + that set. Reconstructing the inventory from membership is therefore the + same operation, not an approximation; every reconstruction below is checked + against the producer's typed context before it is used. + """ + membership = person[US_SCHEMA.membership_column(group)].to_numpy(dtype="int64") + return np.unique(membership) + + +def _minimal_frame(person: pd.DataFrame, household: pd.DataFrame, weights, strata): + """A person/household view for the sampler; no group beyond household.""" + columns = [_PERSON_ID, US_SCHEMA.membership_column("household")] + extra = [name for name in (SELECTION_STRATUM_COLUMN,) if name in person] + return Frame( + { + "person": person.loc[:, columns + extra].copy(deep=True), + "household": household.loc[ + :, [US_SCHEMA.entity_id_column("household")] + ].copy(deep=True), + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + strata.copy(deep=True), + ) + + +def _household_strata(person: pd.DataFrame, household_ids: np.ndarray) -> np.ndarray: + """One declared source-year stratum per household row; mixed years refuse.""" + years = person[SELECTION_STRATUM_COLUMN] + _require(pd.api.types.is_integer_dtype(years.dtype), "STRATUM_DTYPE") + _require(not bool(years.isna().any()), "STRATUM_MISSING") + frame = pd.DataFrame( + { + "household": person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ), + "year": years.to_numpy(dtype="int64"), + } + ) + distinct = frame.drop_duplicates() + _require(not bool(distinct["household"].duplicated().any()), "MIXED_YEAR_HOUSEHOLD") + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require(set(lookup) == set(household_ids.tolist()), "STRATUM_COVERAGE") + return np.asarray( + [str(lookup[int(value)]) for value in household_ids], dtype=object + ) + + +class _PreparedKernel(KernelBase): + """Every kernel of this slice shares one reviewed implementation scope.""" + + def implementation_hash(self) -> str: + return implementation_hash(ASEC_PREPARED_STAGE) + + +class USAsecPreparedCreateKernel(_PreparedKernel): + """Prepare the whole authenticated ASEC population from one source directory.""" + + ref = "us.asec_prepared.create@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME,), "NODE_SOURCES") + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + prepared = prepare_asec_current_money_population( + context.sources[ASEC_PREPARED_SOURCE_NAME] + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "PREPARED_COLUMN_INVENTORY", + ) + receipt = prepared.receipt + consumers = load_graph_current_money_consumers() + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require(receipt["file_roster"] == list(PREPARED_SOURCE_FILES), "FILE_ROSTER") + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": graph_context.encode_us_frame_context(prepared.frame), + "current_money": prepared.money_payload, + "prepared_receipt": prepared.receipt_payload, + "housing_universe": prepared.housing_universe_payload, + "income_observations": prepared.income_observations_payload, + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "prepared": receipt, + "current_money_consumers": consumers, + "release_eligible": False, + }, + ) + + +class USAsecPreparedSelectionKernel(_PreparedKernel): + """Draw the seeded whole-household sample and slice the money body to it.""" + + ref = "us.asec_prepared.select_households@3" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.FILTER, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("fraction", "seed")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.outputs and not node.sources, "NODE_OUTPUTS") + _require(node.structural is StructuralDelta.FILTER, "NODE_STRUCTURAL") + _require(node.mass == "declared", "NODE_MASS") + _require( + set(context.tables) == {"person", "household"} + and set(context.artifacts) + == { + "frame_context", + "current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_INPUTS", + ) + _require( + node.inputs + == ( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice( + "household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS) + ), + ), + "NODE_SLICES", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + money = _artifact(context, "current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + producer_key = _same_producer( + context, + ("frame_context", "current_money", "prepared_receipt", "housing_universe"), + ) + document = _document(bound) + person, household = context.tables["person"], context.tables["household"] + for entity, table in (("person", person), ("household", household)): + _check_identity(document, entity, table) + _require( + document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY" + ) + _require( + context.weights["household"].kind.value == "design", "WEIGHT_AUTHORITY" + ) + prepared = json.loads(receipt_value.payload) + _require( + canonical_json(prepared) == receipt_value.payload, "PREPARED_RECEIPT_JSON" + ) + _require( + prepared["source_kind"] == PREPARED_SOURCE_KIND + and prepared["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + _require( + prepared["entity_rows"]["person"] == document["entities"]["person"]["rows"] + and prepared["entity_rows"]["household"] + == document["entities"]["household"]["rows"], + "PREPARED_RECEIPT_ROWS", + ) + body = parse_current_money_body( + money.payload, + expected_header_sha256=prepared["money_header_sha256"], + expected_content_sha256=prepared["money_content_sha256"], + field_entities=tuple(tuple(item) for item in prepared["field_entities"]), + ) + _require( + body.person_rows == document["entities"]["person"]["rows"] + and body.household_rows == document["entities"]["household"]["rows"], + "BODY_ROW_ALIGNMENT", + ) + household_ids = household[US_SCHEMA.entity_id_column("household")].to_numpy( + dtype="int64" + ) + strata = _household_strata(person, household_ids) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + verify_graph_housing_rows( + household, + universe, + positions=np.arange(len(household), dtype="int64"), + income_years=strata.astype("int64"), + ) + view = _minimal_frame( + person, household, context.weights["household"], context.strata + ) + sampled, selection = sample_frame_households( + view, + fraction=context.params["fraction"], + seed=context.params["seed"], + source_name=SELECTION_SOURCE_LABEL, + unit_strata=strata, + unit_noun="household", + floor_context="the ASEC prepared current-money engineering sample", + ) + selected_household_ids = sampled.table("household")[ + US_SCHEMA.entity_id_column("household") + ].to_numpy(dtype="int64") + membership = person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ) + keep = np.isin(membership, selected_household_ids) + person_positions = np.flatnonzero(keep).astype("int64") + household_positions = np.flatnonzero( + np.isin(household_ids, selected_household_ids) + ).astype("int64") + _require(len(person_positions) > 0, "EMPTY_SELECTION") + _require( + np.array_equal(household_ids[household_positions], selected_household_ids), + "HOUSEHOLD_POSITIONS", + ) + # Whole households only: no selected household may leave a member behind. + _require( + np.array_equal( + person[_PERSON_ID].to_numpy(dtype="int64")[keep], + sampled.person[_PERSON_ID].to_numpy(dtype="int64"), + ), + "WHOLE_HOUSEHOLD", + ) + parent_weights = np.asarray( + context.weights["household"].values, dtype="float64" + ) + selected_weights = np.asarray( + sampled.weights_for("household").values, dtype="float64" + ) + _require( + selected_weights.tobytes() == parent_weights[household_positions].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + entities = {"person": person[_PERSON_ID].to_numpy(dtype="int64")[keep]} + for group in US_SCHEMA.group_entities: + entities[group] = _group_ids(person.loc[keep], group) + _require( + np.array_equal(entities["household"], selected_household_ids), + "SELECTED_HOUSEHOLD_INVENTORY", + ) + identities = { + entity: _identity(ids, entity) for entity, ids in entities.items() + } + before_mass = view.stratum_mass() + after_mass = sampled.stratum_mass() + payload = canonical_json( + { + "schema": SELECTION_SCHEMA, + "phase": ASEC_PREPARED_PHASE, + "release_eligible": False, + "representative": False, + "claim": "engineering_sample_only_no_district_or_calibration_claim", + "stratum_column": SELECTION_STRATUM_COLUMN, + "stratum_grain": "household_source_year", + "exact_count_rule": EXACT_COUNT_RULE, + "mass_normalization": "none", + "selection": _json(selection), + "entities": { + entity: { + "rows": identities[entity]["rows"], + "ordered_ids_sha256": identities[entity]["ordered_ids_sha256"], + } + for entity in _ENTITIES + }, + "person_positions_sha256": _sha(person_positions.tobytes()), + "household_positions_sha256": _sha(household_positions.tobytes()), + "parent": { + "producer_key": producer_key, + "frame_context_sha256": _sha(bound.payload), + "prepared_receipt_sha256": _sha(receipt_value.payload), + "money_header_sha256": prepared["money_header_sha256"], + "money_content_sha256": prepared["money_content_sha256"], + "housing_universe_payload_sha256": _sha(universe_value.payload), + }, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + }, + } + ) + selected = select_current_money( + body, + person_positions=person_positions, + household_positions=household_positions, + prepared_receipt_sha256=_sha(receipt_value.payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in _ENTITIES: + document["entities"][entity].update(identities[entity]) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID), + dtype=bool, + ), + artifacts={ + "frame_context": canonical_json(document), + "selection": payload, + "selected_current_money": encode_selected_current_money(selected), + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "selection_sha256": _sha(payload), + "selected_money_sha256": _sha(encode_selected_current_money(selected)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(key): float(value) for key, value in before_mass.items() + }, + "stratum_after": { + str(key): float(value) for key, value in after_mass.items() + }, + }, + }, + ) + + +def _json(value: object) -> object: + """Normalize a receipt for canonical JSON without inventing string casts.""" + return graph_context._json_data(value) + + +class USAsecPreparedLeavesKernel(_PreparedKernel): + """Derive the corrected monetary CPS leaves from the selected money alone.""" + + ref = "us.asec_prepared.cps_carried_current@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + "NODE_SLICES", + ) + _require(node.outputs == owned_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + selection_value = _artifact(context, "selection", US_ASEC_SELECTION_TYPE) + money = _artifact( + context, "selected_current_money", US_ASEC_SELECTED_MONEY_TYPE + ) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + _same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + _same_producer(context, ("prepared_receipt", "housing_universe")) + document = _document(bound) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + for entity, table in (("person", person), ("spm_unit", spm_unit)): + _check_identity(document, entity, table) + selection = json.loads(selection_value.payload) + _require(canonical_json(selection) == selection_value.payload, "SELECTION_JSON") + _require(selection["schema"] == SELECTION_SCHEMA, "SELECTION_SCHEMA") + prepared = json.loads(receipt_value.payload) + _require( + selection["parent"]["prepared_receipt_sha256"] + == _sha(receipt_value.payload) + and selection["parent"]["producer_key"] == receipt_value.producer_key, + "SELECTION_PARENT_BINDING", + ) + _require( + selection["parent"]["housing_universe_payload_sha256"] + == _sha(universe_value.payload), + "SELECTION_HOUSING_UNIVERSE_BINDING", + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + household = context.tables["household"] + _check_identity(document, "household", household) + positions = ( + pd.Index(universe.array("household_id")) + .get_indexer(household.household_id.to_numpy()) + .astype("int64") + ) + _require( + _sha(positions.tobytes()) == selection["household_positions_sha256"], + "SELECTED_HOUSING_UNIVERSE_POSITIONS", + ) + verify_graph_housing_rows(household, universe, positions=positions) + # The complete selected-coordinate check: the selection artifact, the + # producer's typed context and the actual rows must name one row set. + for entity in _ENTITIES: + declared = document["entities"][entity] + _require( + selection["entities"][entity]["rows"] == declared["rows"] + and selection["entities"][entity]["ordered_ids_sha256"] + == declared["ordered_ids_sha256"], + f"SELECTION_IDENTITY:{entity}", + ) + selected = decode_selected_current_money( + money.payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(receipt_value.payload), + expected_selection_sha256=_sha(selection_value.payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and header["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "SELECTED_COORDINATES", + ) + _require( + selected.person_rows == len(person) + and selected.household_rows == document["entities"]["household"]["rows"], + "SELECTED_ROW_ALIGNMENT", + ) + incumbents = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require(not incumbents, "OWNED_LEAF_INCUMBENT") + routing = person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)] + spm_ids = spm_unit[US_SCHEMA.entity_id_column("spm_unit")].to_numpy( + dtype="int64" + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=routing, + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values, index=person_index, dtype=LEAF_DTYPE) + for name, values in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series(values, index=spm_index, dtype=LEAF_DTYPE) + for name, values in leaves.spm_unit.items() + } + ) + for entity, column in _owned_coordinates(): + document["entities"][entity]["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "contract": cps_carried_current_leaf_contract(), + "selected_money_sha256": _sha(money.payload), + "selection_sha256": _sha(selection_value.payload), + "person_rows": len(person), + "spm_unit_rows": len(spm_ids), + "release_eligible": False, + }, + ) + + +class USAsecPreparedEngineKernel(_PreparedKernel): + """Run the real PolicyEngine-US adapter for the admitted formula-owned roots.""" + + ref = "us.asec_prepared.engine_current_money@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=( + *ASEC_PREPARED_DEPENDENCIES, + "policyengine-us", + "policyengine-core", + ), + ) + + def implementation_hash(self) -> str: + # The executor checks this identity before cache lookup. Runtime or + # baseline parameter drift therefore refuses even when run() is unused. + return _sha( + canonical_json( + { + "implementation": super().implementation_hash(), + "runtime": engine_runtime_identity(), + } + ) + ) + + def __init__(self, engine=None) -> None: + # Only a real adapter instance is accepted; a substitute would make the + # evaluation artifact a claim about something other than the engine. + if engine is not None and type(engine) is not PolicyEngineUSEngine: + raise PreparedGraphError("ENGINE_ADAPTER_TYPE") + self._engine = engine + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("period", "engine_version")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + # Engine outputs are formula-owned; this node owns no population cell. + _require(not node.outputs, "ENGINE_OWNS_NO_CELLS") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + "NODE_SLICES", + ) + _require(set(context.artifacts) == {"frame_context"}, "NODE_ARTIFACT_INPUTS") + _require( + node.artifact_outputs + == (ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + document = _document(bound) + for entity in ("person", "spm_unit", "household"): + _check_identity(document, entity, context.tables[entity]) + for entity, column in _owned_coordinates(): + _require( + column in document["entities"][entity]["columns"], + f"LEAF_NOT_IN_CONTEXT:{entity}.{column}", + ) + frame = self._engine_frame(context, document) + index = PolicyEngineUSVariableMetadataIndex() + contracts = admit_engine_outputs( + index, produced_leaves=CPS_CARRIED_CURRENT_PERSON_LEAVES + ) + _require( + all( + contract.engine_version == context.params["engine_version"] + for contract in contracts + ), + "DECLARED_ENGINE_VERSION", + ) + evaluation = materialize_engine_outputs( + frame, + PolicyEngineUSEngine() if self._engine is None else self._engine, + contracts, + period=context.params["period"], + context_bindings={ + "frame_context_sha256": _sha(bound.payload), + "frame_context_producer_key": bound.producer_key, + "produced_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + }, + ) + payload = encode_engine_evaluation(evaluation) + return KernelResult( + artifacts={"engine_evaluation": payload}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "engine": { + "package": evaluation.header["engine_package"], + "version": evaluation.header["engine_version"], + "period": evaluation.header["period"], + "roots": list(ADMITTED_ROOTS), + "blocked_roots": list(BLOCKED_ENGINE_OUTPUTS), + "closures": evaluation.header["closures"], + "recorded_defaults": [], + "aggregates": evaluation.header["aggregates"], + }, + "evaluation_sha256": _sha(payload), + "cells_written": 0, + "release_eligible": False, + "claim": "engineering_evidence_only_no_tax_or_benefit_score", + }, + ) + + def _engine_frame(self, context: KernelContext, document: dict) -> Frame: + """Rebuild the selected frame carrying only leaves and id carriers. + + Every raw ASEC column, every routing code and the household carrier are + left behind: what the engine receives is exactly the corrected leaves + plus the structure they are indexed by. + """ + person = context.tables["person"] + keep = [ + _PERSON_ID, + *(US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities), + *CPS_CARRIED_CURRENT_PERSON_LEAVES, + ] + _require(set(person.columns) == set(keep), "ENGINE_PERSON_COLUMNS") + tables = {"person": person.loc[:, keep].copy(deep=True)} + for group in US_SCHEMA.group_entities: + ids = _group_ids(person, group) + declared = document["entities"][group] + actual = _identity(ids, group) + _require( + all(declared[key] == value for key, value in actual.items()), + f"ENGINE_GROUP_IDENTITY:{group}", + ) + tables[group] = pd.DataFrame({US_SCHEMA.entity_id_column(group): ids}) + for group in ("household", "spm_unit"): + view = context.tables[group] + _require( + np.array_equal( + view[US_SCHEMA.entity_id_column(group)].to_numpy(dtype="int64"), + tables[group][US_SCHEMA.entity_id_column(group)].to_numpy(), + ), + f"ENGINE_GROUP_VIEW:{group}", + ) + spm = context.tables["spm_unit"] + for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: + tables["spm_unit"][name] = spm[name].to_numpy(dtype="float64", copy=True) + weights = context.weights["household"] + _require(weights.kind.value == "design", "ENGINE_WEIGHT_AUTHORITY") + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + ) + + +def _owned_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def owned_leaf_declarations() -> tuple[Owned, ...]: + """The cells the corrected-leaves node owns, in one canonical order.""" + return tuple( + Owned(entity, column, LEAF_DTYPE) for entity, column in _owned_coordinates() + ) + + +def us_asec_prepared_graph( + columns: Sequence[Owned], + *, + fraction: float, + seed: int, + period: int = 2024, + engine_version: str = "1.819.0", +) -> Graph: + """Declare the five-node slice over the caller's prepared column inventory.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + _require( + ("household", HOUSEHOLD_WEIGHT_CARRIER) in inventory, "MISSING_WEIGHT_CARRIER" + ) + missing = sorted( + name + for name in (SELECTION_STRATUM_COLUMN, *CPS_CARRIED_CURRENT_ROUTING_COLUMNS) + if ("person", name) not in inventory + ) + _require(not missing, f"MISSING_DECLARED_INPUTS:{missing}") + _require( + all(("household", name) in inventory for name in HOUSEHOLD_EVIDENCE_COLUMNS), + "MISSING_HOUSING_UNIVERSE_INPUTS", + ) + # A nominal incumbent can be neither preserved nor silently rewritten. + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if (entity, column) in inventory + ) + incumbent.extend( + f"person.{name}" for name in RESULT_COLUMNS if ("person", name) in inventory + ) + _require(not incumbent, f"OWNED_LEAF_INCUMBENT:{incumbent}") + parent_artifacts = ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "current_money", + CREATE_NODE, + "current_money", + US_ASEC_CURRENT_MONEY_BODY_TYPE, + ), + ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ), + ArtifactInput( + "housing_universe", + CREATE_NODE, + "housing_universe", + US_ASEC_HOUSING_UNIVERSE_TYPE, + ), + ) + create = Node( + id=CREATE_NODE, + kernel=USAsecPreparedCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME,), + outputs=tuple(columns), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + ) + select = Node( + id=SELECT_NODE, + kernel=USAsecPreparedSelectionKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="declared", + inputs=( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS)), + ), + params={"phase": ASEC_PREPARED_PHASE, "fraction": fraction, "seed": seed}, + artifact_inputs=parent_artifacts, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USAsecPreparedLeavesKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + outputs=owned_leaf_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", SELECT_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "selection", SELECT_NODE, "selection", US_ASEC_SELECTION_TYPE + ), + ArtifactInput( + "selected_current_money", + SELECT_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + parent_artifacts[2], + parent_artifacts[3], + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + engine = Node( + id=ENGINE_NODE, + kernel=USAsecPreparedEngineKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + params={ + "phase": ASEC_PREPARED_PHASE, + "period": period, + "engine_version": engine_version, + }, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE), + ), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USAsecReportedIncomeKernel.ref, + population=SELECT_NODE, + inputs=(Slice("person", ("source_year",)),), + outputs=reported_income_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + *leaves.artifact_inputs[:3], + parent_artifacts[2], + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return Graph( + country="us", + sources=(ASEC_PREPARED_SOURCE,), + nodes=(create, select, leaves, engine, reported), + ) + + +def us_asec_prepared_registry(engine=None) -> KernelRegistry: + """Register exactly this slice's five kernels.""" + registry = KernelRegistry() + registry.register(USAsecPreparedCreateKernel()) + registry.register(USAsecPreparedSelectionKernel()) + registry.register(USAsecPreparedLeavesKernel()) + registry.register(USAsecPreparedEngineKernel(engine)) + registry.register(USAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ASEC_PREPARED_CODEC", + "ASEC_PREPARED_DEPENDENCIES", + "ASEC_PREPARED_PHASE", + "ASEC_PREPARED_SOURCE", + "ASEC_PREPARED_SOURCE_NAME", + "ASEC_PREPARED_STAGE", + "CREATE_NODE", + "ENGINE_NODE", + "HOUSEHOLD_WEIGHT_CARRIER", + "LEAVES_NODE", + "PreparedGraphError", + "SELECTION_SCHEMA", + "SELECT_NODE", + "USAsecPreparedCreateKernel", + "USAsecPreparedEngineKernel", + "USAsecPreparedLeavesKernel", + "USAsecPreparedSelectionKernel", + "owned_leaf_declarations", + "us_asec_prepared_graph", + "us_asec_prepared_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py new file mode 100644 index 000000000..c9b1276fe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py @@ -0,0 +1,830 @@ +"""The combined-survey PUF-support clone, as a real graph ``EXPAND`` stage. + +What this stage is +------------------ + +It takes an already combined ASEC + ACS population — every entity native, every +``*_support_clone_index`` zero, household weights already IMPORTANCE — and +returns the two-role host the PUF detail pass needs: the untouched native role +plus exactly one full-attachment detail clone of every household, with each +incoming household weight split evenly across the pair. + +It is only that. No PUF donor is read, no quantile forest is fitted, no draw is +taken, no amount is placed, no tax or benefit value is computed, and no prior +wage stage runs. The donor sampling weight ``S006`` never touches a host weight: +this stage never sees a donor at all. The profile is development; nothing here +is release eligible. + +Why it is an ``EXPAND`` and not a Frame handed to the executor +------------------------------------------------------------- + +:func:`~.puf_support.clone_us_frame_for_puf_support` already performs this +operation directly, and it stays the behavioral authority: the kernel calls it +and validates its output with +:func:`~.puf_support.validate_puf_clone_attachment`, exactly as the direct path +does. What the kernel returns to the executor, though, is not that Frame. It +returns what :class:`~microcosm.graph.KernelResult` declares for a structural +node — per-entity clone lineage, the declared cell overlays, and explicit +weights — and ``microcosm.graph.population._patch_expand`` does the structural +work: it carries every column from each copied row, remaps every copied +person's five memberships onto the copied groups, appends the strata, installs +the weights and records the mass ledger. A Frame smuggled through as an opaque +artifact would make the executor's lineage, membership, storage, weight and +mass checks vacuous, so this stage does not do that. + +The ID relationship between the two paths +----------------------------------------- + +The executor's contract is that every base row survives with its own id and new +rows get new ids. The direct operator honours the same rule on this input: for a +preassembled frame it copies the native block unchanged and shifts only the +detail block by a decimal multiplier. The two paths therefore agree on raw ids +here — but the stage does not depend on that. Lineage is derived from the +operator's own output through the ``(entity_source_id, clone_index)`` bijection, +with the positional pairing checked independently, and the receipt publishes a +digest of that bijection. A comparison against the direct operator is a +comparison by declared source and role, not an assumption about integers. + +What the node key binds +----------------------- + +The kernel's implementation hash covers this module, the clone operator's module +and the provenance owner's module, plus the pinned ``numpy``/``pandas`` +versions, so an edit to the operator moves every cached identity. The node's own +key additionally binds its declared parameters, the artifact keys of the +provenance columns it reads and — through ``population_input['base']`` — the +whole base population version: its amounts, its ids, its row order, its +provenance and its incoming weights. Changing any of those changes this node's +key, so warm reuse can never serve a clone of a different population. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import token_for_dtype + +from . import puf_support as _puf_support_module +from . import support_provenance as _support_provenance_module +from .puf_support import ( + clone_us_frame_for_puf_support, + validate_puf_clone_attachment, +) +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + PUF_TAX_DETAIL_CLONE_INDEX, + PUF_TAX_DETAIL_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +__all__ = [ + "COMBINED_CLONE_ATTACHMENT_FRACTION", + "COMBINED_CLONE_ATTACHMENT_SEED", + "COMBINED_CLONE_CELL_DTYPE", + "COMBINED_CLONE_CLAIM_NODE", + "COMBINED_CLONE_NODE", + "COMBINED_CLONE_PHASE", + "COMBINED_CLONE_PREFIX", + "COMBINED_CLONE_PROFILE", + "COMBINED_CLONE_ROLES", + "COMBINED_CLONE_WEIGHT_ENTITY", + "COMBINED_CLONE_WEIGHT_KIND", + "COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR", + "CombinedCloneError", + "USCombinedSurveyCloneClaimKernel", + "USCombinedSurveyCloneExpandKernel", + "combined_clone_provenance_columns", + "register_us_combined_survey_clone_kernels", + "us_combined_survey_clone_nodes", +] + + +class CombinedCloneError(ValueError): + """A combined-survey clone declaration or input violates the contract.""" + + +#: Descriptive phase label carried in the node params and every receipt. +COMBINED_CLONE_PHASE = "us_combined_survey_puf_support_clone" +COMBINED_CLONE_PREFIX = "combined_survey_puf_support_clone" +COMBINED_CLONE_NODE = COMBINED_CLONE_PREFIX +COMBINED_CLONE_CLAIM_NODE = f"{COMBINED_CLONE_PREFIX}.owned" + +#: This stage is an engineering intermediate, not a candidate for release. +COMBINED_CLONE_PROFILE = "development" + +#: The two operator roles the clone writes: the incoming native role keeps +#: clone index 0, the single detail copy takes ``PUF_TAX_DETAIL_CLONE_INDEX``. +#: These are operator roles, never source channels: the ASEC and ACS source +#: channels are carried unchanged into *both* roles. +COMBINED_CLONE_ROLES = (BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_SUPPORT_CHANNEL) + +#: Full one-copy attachment only. A seeded partial arm is a separate, explicitly +#: declared control and is deliberately not reachable from this stage. +COMBINED_CLONE_ATTACHMENT_FRACTION = 1.0 +COMBINED_CLONE_ATTACHMENT_SEED = 0 + +#: Each incoming household weight is split across exactly two roles. +COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR = 2 +COMBINED_CLONE_WEIGHT_ENTITY = "household" +COMBINED_CLONE_WEIGHT_KIND = WeightKind.IMPORTANCE + +#: The clone-index columns are the only cells this stage writes. +COMBINED_CLONE_CELL_DTYPE = "int64" + +_PARAM_KEYS = ( + "clone_attachment_fraction", + "clone_attachment_seed", + "clone_roles", + "expand_cells", + "expand_weight_entity", + "expand_weight_kind", + "phase", + "profile", + "release_eligible", + "source_channels", + "weight_split_denominator", +) + +_LINEAGE_DIGEST_DOMAIN = b"microcosm.us.combined-survey-clone-lineage.v1\0" + + +def combined_clone_provenance_columns(entity: str) -> tuple[str, ...]: + """The four provenance columns this stage reads on ``entity``. + + Order is fixed so a node declaration is a stable, reviewable literal. + """ + + return ( + support_source_id_column(entity), + spine_source_id_column(entity), + support_channel_column(entity), + support_clone_index_column(entity), + ) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CombinedCloneError(f"{COMBINED_CLONE_PHASE}: {reason}") + + +def _expected_expand_cells() -> tuple[tuple[str, str, str], ...]: + return tuple( + (entity, support_clone_index_column(entity), COMBINED_CLONE_CELL_DTYPE) + for entity in US_SCHEMA.entities + ) + + +def _validated_source_channels(value: object) -> tuple[str, ...]: + _require(isinstance(value, tuple), "SOURCE_CHANNELS_TYPE") + channels = tuple(value) # type: ignore[arg-type] + _require( + len(channels) >= 2 + and len(set(channels)) == len(channels) + and all(isinstance(name, str) and bool(name) for name in channels), + "SOURCE_CHANNELS_VALUE", + ) + _require(tuple(sorted(channels)) == channels, "SOURCE_CHANNELS_ORDER") + return channels + + +def _digest(payload: object) -> str: + return hashlib.sha256(_LINEAGE_DIGEST_DOMAIN + canonical_json(payload)).hexdigest() + + +def _int64_ids(table: pd.DataFrame, column: str, *, reason: str) -> np.ndarray: + values = table[column] + _require(str(values.dtype) == "int64", reason) + return values.to_numpy(dtype=np.int64, copy=True) + + +def _entity_lineage( + before: Frame, + after: Frame, + entity: str, +) -> tuple[pd.Series, dict[str, object]]: + """Derive one entity's new-id -> copied-id lineage and its receipt facts. + + The pairing is by ``(entity_source_id, clone_index)``: the detail row whose + assembly-unique source id is *s* copies the native row whose source id is + *s*. Nothing here reads or reconstructs an ID-offset rule. The positional + pairing the operator happens to produce is checked against that bijection + rather than trusted in its place. + """ + + id_column = US_SCHEMA.entity_id_column(entity) + source_column = support_source_id_column(entity) + clone_column = support_clone_index_column(entity) + + before_table = before.table(entity) + after_table = after.table(entity) + before_ids = _int64_ids(before_table, id_column, reason=f"BASE_ID_DTYPE:{entity}") + after_ids = _int64_ids(after_table, id_column, reason=f"CLONE_ID_DTYPE:{entity}") + clone_index = after_table[clone_column].to_numpy(dtype=np.int64, copy=True) + source_ids = _int64_ids( + after_table, source_column, reason=f"CLONE_SOURCE_ID_DTYPE:{entity}" + ) + + unexpected = sorted( + set(int(value) for value in np.unique(clone_index)) + - {0, PUF_TAX_DETAIL_CLONE_INDEX} + ) + _require(not unexpected, f"CLONE_ROLE_UNEXPECTED:{entity}:{unexpected}") + + native = clone_index == 0 + detail = clone_index == PUF_TAX_DETAIL_CLONE_INDEX + _require(int(native.sum()) == len(before_ids), f"NATIVE_ROW_COUNT:{entity}") + _require(int(detail.sum()) == len(before_ids), f"DETAIL_ROW_COUNT:{entity}") + _require( + np.array_equal(after_ids[native], before_ids), f"NATIVE_IDS_PRESERVED:{entity}" + ) + + native_sources = source_ids[native] + detail_sources = source_ids[detail] + _require( + len(np.unique(native_sources)) == len(native_sources), + f"NATIVE_SOURCE_IDS_UNIQUE:{entity}", + ) + _require( + len(np.unique(detail_sources)) == len(detail_sources), + f"DETAIL_SOURCE_IDS_UNIQUE:{entity}", + ) + native_by_source = pd.Index(native_sources) + positions = native_by_source.get_indexer(detail_sources) + _require(bool((positions >= 0).all()), f"DETAIL_SOURCE_ID_UNMATCHED:{entity}") + + target_ids = after_ids[detail] + copied_ids = before_ids[positions] + _require( + not len(np.intersect1d(target_ids, before_ids)), + f"TARGET_ID_COLLIDES_WITH_BASE:{entity}", + ) + _require( + len(np.unique(target_ids)) == len(target_ids), f"TARGET_IDS_UNIQUE:{entity}" + ) + # Independent cross-check: the operator emits the detail block row-aligned + # to the native block, so position i of the detail block must resolve to + # position i of the native block. A disagreement means the bijection and + # the physical layout describe different pairings; refuse rather than pick. + _require( + np.array_equal(positions, np.arange(len(positions), dtype=np.int64)), + f"LINEAGE_POSITION_DISAGREES:{entity}", + ) + + lineage = pd.Series( + copied_ids, + index=pd.Index(target_ids, name=id_column, dtype=before_table[id_column].dtype), + dtype=before_table[id_column].dtype, + name=id_column, + ) + facts = { + "base_rows": int(len(before_ids)), + "expanded_rows": int(len(after_ids)), + "native_rows": int(native.sum()), + "detail_rows": int(detail.sum()), + "lineage_sha256": _digest( + [ + [int(source), int(PUF_TAX_DETAIL_CLONE_INDEX), int(target)] + for source, target in zip(detail_sources, target_ids, strict=True) + ] + ), + "native_source_ids_sha256": _digest( + [int(value) for value in native_sources.tolist()] + ), + } + return lineage, facts + + +def _channel_mass(frame: Frame, channels: Sequence[str]) -> dict[str, float]: + """Weighted household mass per source channel, for the receipt only.""" + + table = frame.table(COMBINED_CLONE_WEIGHT_ENTITY) + values = np.asarray( + frame.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).values, dtype=np.float64 + ) + labels = table[support_channel_column(COMBINED_CLONE_WEIGHT_ENTITY)].astype(str) + return { + channel: float(values[labels.eq(channel).to_numpy()].sum()) + for channel in channels + } + + +class USCombinedSurveyCloneExpandKernel(KernelBase): + """Run the registered clone operator and hand the executor its lineage.""" + + ref = "us.combined_survey.puf_support_clone.expand@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.EXPAND, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + # The behavior of this node is this module plus the operator it calls + # and the provenance owner whose columns define both roles. Hashing + # only this file would let an operator edit reuse a stale clone. + return source_hash( + type(self), + _puf_support_module, + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + # ------------------------------------------------------------------ + + def _validated_params(self, context: KernelContext) -> tuple[str, ...]: + params = context.params + _require(set(params) == set(_PARAM_KEYS), f"NODE_PARAMS:{sorted(params)}") + _require(params["phase"] == COMBINED_CLONE_PHASE, "NODE_PHASE") + _require(params["profile"] == COMBINED_CLONE_PROFILE, "NODE_PROFILE") + _require(params["release_eligible"] is False, "NODE_RELEASE_ELIGIBLE") + _require( + params["clone_attachment_fraction"] == COMBINED_CLONE_ATTACHMENT_FRACTION, + "NODE_ATTACHMENT_FRACTION", + ) + _require( + params["clone_attachment_seed"] == COMBINED_CLONE_ATTACHMENT_SEED, + "NODE_ATTACHMENT_SEED", + ) + _require(tuple(params["clone_roles"]) == COMBINED_CLONE_ROLES, "NODE_ROLES") + _require( + params["weight_split_denominator"] + == COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "NODE_WEIGHT_SPLIT", + ) + _require( + params["expand_weight_entity"] == COMBINED_CLONE_WEIGHT_ENTITY, + "NODE_WEIGHT_ENTITY", + ) + _require( + params["expand_weight_kind"] == COMBINED_CLONE_WEIGHT_KIND.value, + "NODE_WEIGHT_KIND", + ) + _require( + tuple(params["expand_cells"]) == _expected_expand_cells(), + "NODE_EXPAND_CELLS", + ) + return _validated_source_channels(params["source_channels"]) + + def _validated_declaration(self, context: KernelContext) -> None: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(node.structural is StructuralDelta.EXPAND, "NODE_STRUCTURAL") + _require(node.entrants is False, "NODE_ENTRANTS") + _require(node.mass == "conserve", "NODE_MASS") + _require(not node.sources, "NODE_SOURCES") + _require(not node.artifact_inputs, "NODE_ARTIFACT_INPUTS") + _require(not node.artifact_outputs, "NODE_ARTIFACT_OUTPUTS") + declared = { + (slice_.entity, column) + for slice_ in node.inputs + for column in slice_.columns + } + expected = { + (entity, column) + for entity in US_SCHEMA.entities + for column in combined_clone_provenance_columns(entity) + } + _require(declared == expected, f"NODE_INPUTS:{sorted(declared ^ expected)}") + _require( + all(slice_.rows == "all" for slice_ in node.inputs), "NODE_INPUT_ROW_MASK" + ) + + def _minimal_frame( + self, context: KernelContext, channels: tuple[str, ...] + ) -> Frame: + """Rebuild the combined host from the declared slices only. + + The frame carries the four provenance columns, the structural id and + membership columns the executor always projects, the household weights + and the strata — nothing else. Every amount the population holds is + carried by the executor from the copied rows, so the operator does not + need to see one, and this kernel deliberately cannot read one. + """ + + missing = [ + entity for entity in US_SCHEMA.entities if entity not in context.tables + ] + _require(not missing, f"MISSING_SLICES:{missing}") + tables = { + entity: context.tables[entity].copy(deep=True) + for entity in US_SCHEMA.entities + } + _require( + COMBINED_CLONE_WEIGHT_ENTITY in context.weights, "MISSING_HOUSEHOLD_WEIGHTS" + ) + household_weights = context.weights[COMBINED_CLONE_WEIGHT_ENTITY] + _require( + household_weights.kind is COMBINED_CLONE_WEIGHT_KIND, + f"INCOMING_WEIGHT_KIND:{household_weights.kind.value}", + ) + self._require_inherited_weights(context, tables, household_weights) + metadata = spine_assembly_manifest(tables, channels=channels) + frame = Frame( + tables, + US_SCHEMA, + {COMBINED_CLONE_WEIGHT_ENTITY: household_weights}, + context.strata.copy(deep=True), + metadata=metadata, + ) + observed = { + str(value) + for entity in US_SCHEMA.entities + for value in frame.table(entity)[support_channel_column(entity)].unique() + } + _require(observed == set(channels), f"SOURCE_CHANNELS_LIVE:{sorted(observed)}") + return frame + + @staticmethod + def _require_inherited_weights( + context: KernelContext, + tables: Mapping[str, pd.DataFrame], + household_weights: Weights, + ) -> None: + """Refuse a base that weights anything but the household explicitly. + + ``KernelContext.weights`` resolves inheritance, so it cannot by itself + say which entity stores a vector. What it can say is whether every + other entity's effective weights are exactly the household weights + broadcast through membership; anything else is a base this stage would + silently mis-split, because the executor carries a non-weight-entity's + weights unhalved while the operator halves them. The residual case — a + stored vector numerically equal to the inherited one — cannot pass the + executor's ``conserve`` person-mass ledger either. + """ + + household = tables[COMBINED_CLONE_WEIGHT_ENTITY] + by_id = pd.Series( + np.asarray(household_weights.values, dtype=np.float64), + index=pd.Index(household["household_id"].to_numpy()), + ) + person = tables[US_SCHEMA.person_entity] + person_weights = by_id.reindex( + person[US_SCHEMA.membership_column(COMBINED_CLONE_WEIGHT_ENTITY)].to_numpy() + ).to_numpy(dtype=np.float64) + for entity, weights in context.weights.items(): + if entity == COMBINED_CLONE_WEIGHT_ENTITY: + continue + values = np.asarray(weights.values, dtype=np.float64) + if entity == US_SCHEMA.person_entity: + expected = person_weights + else: + member = pd.DataFrame( + { + "group": person[US_SCHEMA.membership_column(entity)].to_numpy(), + "weight": person_weights, + } + ) + grouped = member.groupby("group")["weight"] + _require( + bool((grouped.nunique() == 1).all()), + f"AMBIGUOUS_INHERITED_WEIGHTS:{entity}", + ) + expected = ( + grouped.first() + .reindex(tables[entity][US_SCHEMA.entity_id_column(entity)]) + .to_numpy(dtype=np.float64) + ) + _require( + values.shape == expected.shape and np.array_equal(values, expected), + f"NON_INHERITED_WEIGHTS:{entity}", + ) + + # ------------------------------------------------------------------ + + def run(self, context: KernelContext) -> KernelResult: + self._validated_declaration(context) + channels = self._validated_params(context) + before = self._minimal_frame(context, channels) + + after = clone_us_frame_for_puf_support( + before, + channels=COMBINED_CLONE_ROLES, + clone_attachment_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + clone_attachment_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + authority = validate_puf_clone_attachment( + after, + boundary=f"{COMBINED_CLONE_PHASE} clone output", + expected_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + expected_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + _require( + authority["authority_form"] == "full_clone_identity_no_manifest", + "ATTACHMENT_AUTHORITY_FORM", + ) + + lineage: dict[str, pd.Series] = {} + entity_facts: dict[str, object] = {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = _entity_lineage( + before, after, entity + ) + + columns: dict[tuple[str, str], pd.Series] = {} + for entity, column, dtype in _expected_expand_cells(): + incumbent = before.table(entity)[column] + _require( + token_for_dtype(incumbent.dtype) == dtype, + f"CELL_DTYPE:{entity}.{column}", + ) + table = after.table(entity) + values = table[column] + _require(str(values.dtype) == dtype, f"CLONE_CELL_DTYPE:{entity}.{column}") + columns[(entity, column)] = pd.Series( + values.to_numpy(dtype=np.int64, copy=True), + index=pd.Index( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(copy=True), + name=US_SCHEMA.entity_id_column(entity), + ), + name=column, + dtype=dtype, + ) + + weights = self._split_weights(before, after, lineage) + receipt = self._receipt(before, after, channels, authority, entity_facts) + return KernelResult( + columns=columns, expand=lineage, weights=weights, receipt=receipt + ) + + @staticmethod + def _split_weights( + before: Frame, after: Frame, lineage: Mapping[str, pd.Series] + ) -> Weights: + """Household weights in the executor's target order, pair-sum checked.""" + + entity = COMBINED_CLONE_WEIGHT_ENTITY + id_column = US_SCHEMA.entity_id_column(entity) + after_weights = after.weights_for(entity) + _require(after_weights.kind is COMBINED_CLONE_WEIGHT_KIND, "CLONE_WEIGHT_KIND") + by_id = pd.Series( + np.asarray(after_weights.values, dtype=np.float64), + index=pd.Index(after.table(entity)[id_column].to_numpy(copy=True)), + ) + base_ids = before.table(entity)[id_column].to_numpy(copy=True) + target_ids = np.concatenate( + [base_ids, lineage[entity].index.to_numpy(copy=True)] + ) + ordered = by_id.reindex(target_ids) + _require(not bool(ordered.isna().any()), "WEIGHT_TARGET_ALIGNMENT") + values = ordered.to_numpy(dtype=np.float64) + + incoming = np.asarray( + before.weights_for(entity).values, dtype=np.float64, copy=True + ) + native = values[: len(base_ids)] + detail = values[len(base_ids) :] + # Require an exact pair sum rather than a tolerance that could hide + # a rescale. Halving can lose the low bit of a subnormal double; + # such a weight refuses here instead of silently losing mass. + _require(np.array_equal(native + detail, incoming), "PAIR_WEIGHT_SUM") + _require(np.array_equal(native, detail), "PAIR_WEIGHT_SYMMETRY") + return Weights(values, after_weights.kind) + + @staticmethod + def _receipt( + before: Frame, + after: Frame, + channels: tuple[str, ...], + authority: Mapping[str, object], + entity_facts: Mapping[str, object], + ) -> dict[str, object]: + """Counts, digests, labels and platform-sensitive mass summaries. + + Floating reductions here and in ``_channel_mass`` are receipt evidence, + outside the BITWISE population-output promise. Their last bits can + differ across platforms, so receipt/manifest hashes are scoped too. + """ + + return { + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "operator": { + "module": _puf_support_module.__name__, + "clone": clone_us_frame_for_puf_support.__name__, + "validator": validate_puf_clone_attachment.__name__, + }, + "source_channels": list(channels), + "clone_roles": list(COMBINED_CLONE_ROLES), + "clone_attachment": { + "fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "seed": COMBINED_CLONE_ATTACHMENT_SEED, + "authority_form": authority["authority_form"], + "eligible_household_count": int(authority["eligible_household_count"]), + "realized_household_count": int(authority["realized_household_count"]), + "exact_count_rule": authority["exact_count_rule"], + "selected_household_source_ids_sha256": authority[ + "selected_household_source_ids_sha256" + ], + }, + "entities": {entity: entity_facts[entity] for entity in US_SCHEMA.entities}, + "provenance_counts": { + "base": spine_provenance_counts( + before, boundary=f"{COMBINED_CLONE_PHASE} base" + ), + "expanded": spine_provenance_counts( + after, boundary=f"{COMBINED_CLONE_PHASE} expanded" + ), + }, + "household_weights": { + "kind": COMBINED_CLONE_WEIGHT_KIND.value, + # The operator divides by its role count. This literal is + # the re-validated declaration of that two-role contract, + # not a separate arithmetic control for the operator. + "split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "base_total": float( + before.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "expanded_total": float( + after.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "base_by_source_channel": _channel_mass(before, channels), + "expanded_by_source_channel": _channel_mass(after, channels), + }, + "donor_inputs": { + "puf_donor_read": False, + "donor_sampling_weight_in_host_weights": False, + "draws_or_tax_benefit_values_written": False, + "prior_wage_stage": False, + }, + } + + +class USCombinedSurveyCloneClaimKernel(KernelBase): + """Own the clone-index cells the ``EXPAND`` materialized, unchanged. + + A copied row's clone index differs from its source's, which the executor + only accepts when a same-version claimant declares that coordinate a + full-cell rewrite. This kernel is that claimant: it re-emits exactly the + values the ``EXPAND`` installed, so ownership becomes explicit without any + second opinion about what the clone index is. + """ + + ref = "us.combined_survey.puf_support_clone.claim@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + ) + + def implementation_hash(self) -> str: + return source_hash( + type(self), + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + expected = { + (entity, support_clone_index_column(entity)) + for entity in US_SCHEMA.entities + } + owned = {(output.entity, output.column) for output in node.outputs} + _require(owned == expected, f"CLAIM_OUTPUTS:{sorted(owned ^ expected)}") + _require( + all(output.rewrite and output.rows == "all" for output in node.outputs), + "CLAIM_REWRITE", + ) + columns: dict[tuple[str, str], pd.Series] = {} + for output in node.outputs: + table = context.tables[output.entity] + id_column = US_SCHEMA.entity_id_column(output.entity) + _require( + str(table[output.column].dtype) == output.dtype, + f"CLAIM_DTYPE:{output.entity}.{output.column}", + ) + columns[(output.entity, output.column)] = pd.Series( + table[output.column].to_numpy(dtype=np.int64, copy=True), + index=pd.Index(table[id_column].to_numpy(copy=True), name=id_column), + name=output.column, + dtype=output.dtype, + ) + return KernelResult( + columns=columns, + receipt={ + "phase": COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{entity}.{column}" for entity, column in owned + ), + }, + ) + + +def us_combined_survey_clone_nodes( + columns: Sequence[Owned], + *, + base: str, + source_channels: Sequence[str], + prefix: str = COMBINED_CLONE_PREFIX, +) -> tuple[Node, ...]: + """Declare the clone ``EXPAND`` and the claim node that owns its cells. + + Args: + columns: The base population version's full column inventory. Only the + four provenance columns of each entity are read; the inventory is + required so a missing or wrongly typed provenance column is a + declaration-time refusal rather than a run-time surprise. + base: The population version holding the combined native host. + source_channels: The exact source channels the host carries, sorted. + Both survey arms must be named; this is what the kernel then + requires the live rows to match. + prefix: Node-id prefix, so a caller may declare more than one profile. + + Returns: + ``(expand_node, claim_node)``. The claim node's population is the + expand node, so the claimed cells are owned inside the cloned version. + """ + + _require(bool(base), "BASE_REQUIRED") + channels = _validated_source_channels(tuple(source_channels)) + inventory = {(owned.entity, owned.column): owned for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + for column in combined_clone_provenance_columns(entity): + owned = inventory.get((entity, column)) + _require(owned is not None, f"MISSING_PROVENANCE_COLUMN:{entity}.{column}") + cells = _expected_expand_cells() + for entity, column, dtype in cells: + _require( + inventory[(entity, column)].dtype == dtype, + f"CLONE_INDEX_DTYPE:{entity}.{column}", + ) + + expand_node = f"{prefix}" + claim_node = f"{prefix}.owned" + expand = Node( + id=expand_node, + kernel=USCombinedSurveyCloneExpandKernel.ref, + inputs=tuple( + Slice(entity, combined_clone_provenance_columns(entity)) + for entity in US_SCHEMA.entities + ), + params={ + "clone_attachment_fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "clone_attachment_seed": COMBINED_CLONE_ATTACHMENT_SEED, + "clone_roles": COMBINED_CLONE_ROLES, + "expand_cells": cells, + "expand_weight_entity": COMBINED_CLONE_WEIGHT_ENTITY, + "expand_weight_kind": COMBINED_CLONE_WEIGHT_KIND.value, + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "source_channels": channels, + "weight_split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + }, + structural=StructuralDelta.EXPAND, + base=base, + mass="conserve", + description=( + "Attach one whole-household PUF-detail clone to the combined " + "ASEC/ACS host and split every household weight across the pair." + ), + ) + claim = Node( + id=claim_node, + kernel=USCombinedSurveyCloneClaimKernel.ref, + outputs=tuple( + Owned(entity, column, dtype, rewrite=True) + for entity, column, dtype in cells + ), + population=expand_node, + params={"phase": COMBINED_CLONE_PHASE}, + description="Own the clone-index cells the clone stage materialized.", + ) + return (expand, claim) + + +def register_us_combined_survey_clone_kernels(registry: KernelRegistry) -> None: + """Register both kernels of the bounded combined-survey clone stage.""" + + registry.register(USCombinedSurveyCloneExpandKernel()) + registry.register(USCombinedSurveyCloneClaimKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py new file mode 100644 index 000000000..0686001aa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py @@ -0,0 +1,1400 @@ +"""Bind the carried prepared-ASEC evidence to the ASEC rows of a composed population. + +:mod:`.graph_composed_population` reads both authenticated sources whole and +carries the prepared arm's four typed evidence artifacts plus its own +``asec_frame_context`` unchanged. Those artifacts are **positional over the whole +prepared ASEC population**, not over the composed rows, so the accepted +prepared-slice kernels cannot attach: their contracts require an empty context +metadata, an empty mass history, a design household weight and prepared-receipt +rows equal to the population's rows. A harmonized composed population satisfies +none of them. Those refusals are real, are reproduced in the test module, and +nothing here weakens them. + +What this module adds is the missing edge and nothing else: one node that +resolves which composed rows came from the ASEC arm and which source row each +one is, and publishes that resolution as a typed artifact together with a +:class:`~.asec_current_money_selection.SelectedCurrentMoney` sliced to exactly +those source coordinates by the same reviewed selector the prepared slice uses. + +How rows are resolved +--------------------- +By source arm and the arm's own pre-remap identifier, never by composed row +position and never by the post-offset ``*_source_id``. For each entity the node +reads ``*_support_channel`` to find the arm and ``*_spine_source_id`` for the +arm's original ID, then locates each original ID in the arm-ordered identity the +carried evidence itself declares: ``income_observations``' ``person_id`` for +persons and ``housing_universe``' ``household_id`` for households. Both buffers +are bound to the prepared receipt before they are used, so that ordering is +receipt-authenticated rather than assumed. The resolution refuses an empty, +duplicated, unresolved or non-monotone mapping, and refuses a cohort +disagreement: the composed ``source_year`` must equal the source ``income_year`` +at the resolved positions, and the income artifact's own three declared +crosschecks must agree cell by cell. + +Nothing is assumed about where the arm's rows sit. They need not be a prefix, +need not be contiguous, and group tables may be sorted or interleaved with the +other arm's; the mask is read, never constructed from a row range. Every +reconstruction is then checked back against ``source_origin``: the origin +document's per-entity provenance digests are recomputed against the actual +composed tables, so a document that describes a different frame refuses. + +Why this module may read source provenance +------------------------------------------ +Resolving "which arm did this row come from" *is* this node's operation, so it +is a reviewed source-spine provenance owner. It applies no population treatment: +it writes one boolean arm-membership cell per bound entity and no measurement. +The measurements live in :mod:`.graph_composed_asec_measures`, consume this +node's typed artifact and its declared row mask, and stay source-blind. +""" + +from __future__ import annotations + +import json +import struct +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import ( + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + _json, + _parse, + _sha, +) +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_prepared_source import PREPARED_SOURCE_KIND +from .graph_asec_income import ( + US_ASEC_INCOME_OBSERVATIONS_TYPE, + bind_income_observations, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_contracts import ( + BIND_NODE as BIND_NODE, + US_COMPOSED_ASEC_BINDING_TYPE as US_COMPOSED_ASEC_BINDING_TYPE, + US_COMPOSED_ASEC_ARM_ROWS_TYPE as US_COMPOSED_ASEC_ARM_ROWS_TYPE, +) +from .graph_composed_population import ( + CREATE_NODE, + US_COMPOSED_SOURCE_ORIGIN_TYPE, + bind_composed_source_origin, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_ASEC_STAGE = "composed_asec_binding_v1" +COMPOSED_ASEC_PHASE = "bind_composed_asec_evidence" +COMPOSED_ASEC_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_ASEC_STAGE] + + +#: The arm this binding resolves. The other arm is named only so an unexpected +#: third channel refuses; no ACS row is read, written or measured here. +ARM_CHANNEL = BASE_ASEC_SUPPORT_CHANNEL +_OTHER_CHANNEL = ACS_STACKED_SUPPORT_CHANNEL + +#: The declared boolean arm-membership cell, entity-prefixed because ``Frame`` +#: requires column names to be globally unique across entity tables. Every +#: measurement node writes under this mask, so the executor — not a convention — +#: is what keeps the opposite arm's storage byte-identical: +#: ``population._patch_columns`` refuses any change outside an ``Owned`` mask, +#: missingness included. +ARM_ROW_SUFFIX = "composed_asec_row" +ARM_ROW_DTYPE = "bool" + + +def arm_row_column(entity: str) -> str: + """Return the entity-prefixed arm-membership cell name.""" + _require(entity in US_SCHEMA.entities, f"ARM_ROW_ENTITY:{entity}") + return f"{entity}_{ARM_ROW_SUFFIX}" + + +#: The entities a measurement of this arm addresses: persons, the households +#: whose carried housing evidence is verified against the source, and the SPM +#: units the corrected childcare leaf is grained on. +BOUND_ENTITIES = ("person", "household", "spm_unit") +#: The two entities the carried evidence declares an ordered source identity for. +_ID_ENTITIES = ("person", "household") + +BINDING_SCHEMA = "microcosm.us.composed-asec-binding.v1" + +#: The income artifact's own declared crosschecks, verified row by row between +#: the composed cells and the source buffers at the resolved positions. +PERSON_COORDINATE_CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +COHORT_COLUMN = "source_year" + +#: Demographic columns a later calibration wants on both arms and which this +#: stage deliberately does not bind: each names the exact source column the +#: reviewed ASEC mapping needs, the mapping itself, and what the native ACS arm +#: uses instead. Nothing is imputed, aged or defaulted. +#: +#: Whether the prepared arm carries the source column is an **observed property +#: of the population in hand**, recorded per run rather than asserted: the +#: invented fixture parent has neither column, and the genuine prepared arm has +#: both. Presence is not a licence — binding either column still needs a +#: reviewed cross-arm mapping decision that this stage may not make — so what +#: fails closed here is an attempted **write** of one of these columns +#: (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``), not the source column's presence. Keying +#: the refusal on presence would have refused every genuine run while proving +#: nothing about what this stage binds. +#: Each entry is (column, required source columns, diagnostic-only source +#: columns, the explicit source contract that would supply it, the mappings +#: deliberately not adopted as proof, the caveats on reading presence as +#: observation, the ACS arm's own observed mapping). +#: The required columns are the ones an *explicit* binding needs, which is not +#: the same as the columns some existing mapping happens to read: the root +#: adjudication of 2026-09-06 refused ``P_SEQ == 1`` as semantic proof of +#: headship and required sex to carry its allocation provenance, so those +#: mappings are recorded here as not adopted rather than as the requirement. +_UNBOUND_DEMOGRAPHICS = ( + ( + "is_female", + ("A_SEX", "AXSEX"), + (), + "asec_demographic_source: A_SEX printed codes 1 = Male and 2 = Female, " + "admitted only with AXSEX printed codes 0 = No change or 4 = Allocated; " + "any other token leaves the person unbound", + ( + ( + "cps_carried.derive_us_cps_carried_inputs (is_female = A_SEX == 2)", + "maps every token other than 2 onto male, so an unprinted code " + "would be admitted as male, and it reads no allocation flag", + ), + ), + ( + "A_SEX reaches the prepared person roster as a carried source " + "column, but its presence attests the column, not that every " + "delivered token is one of the two printed codes", + ), + "SEX == 2 on the native ACS arm, whose pinned dictionary prints the " + "same 1 = Male / 2 = Female convention", + ), + ( + "is_household_head", + ("A_EXPRRP",), + ("P_SEQ",), + "asec_demographic_source: A_EXPRRP printed codes 1 = Reference person " + "with relatives and 2 = Reference person without relatives, with " + "exactly one such person per household and every household relationship " + "token inside the printed named codes", + ( + ( + "relationship_inputs (is_household_head = P_SEQ == 1)", + "P_SEQ's dictionary entry labels no code as reference person, " + "and the ordering statement behind it is expressly limited to " + "the ASCII file while this arm is restored from the CSV member", + ), + ( + "A_FAMREL == 1", + "a family relationship scoped to primary-family membership, " + "which a subfamily reference person does not carry; household " + "and family roles stay separate", + ), + ( + "asec_pool relationship recode (A_LINENO == 1)", + "a separately assigned Basic-CPS roster line number with no " + "source guarantee of agreement with any ASEC sequence", + ), + ), + ( + "asec_pool._with_relationship_recode derives A_EXPRRP from " + "A_LINENO == 1 whenever the locked input omits it, so the column's " + "presence on a prepared arm does not certify an observed source " + "value for that cohort", + "the native ACS arm's A_EXPRRP is likewise derived by acs_pums " + "from RELSHIPP, and its group-quarters households carry the " + "nonrelative code 14 rather than any reference-person code", + ), + "RELSHIPP == 20 observed on the native ACS arm's housing units only; " + "RELSHIPP 37 and 38 are the group-quarters populations and carry no " + "observed reference person", + ), +) +#: The population columns this stage records as unbound, and therefore may not +#: own on any node. Public because the measurement module's own declarations are +#: checked against it. +UNBOUND_DEMOGRAPHIC_COLUMNS = tuple(column for column, *_rest in _UNBOUND_DEMOGRAPHICS) + +_ARTIFACT_TYPES = ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + ("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) +#: The prepared context and the four evidence artifacts are positional over one +#: prepared population, so they must come from one producing node execution. +_SOURCE_EDGES = ( + "asec_frame_context", + "current_money", + "housing_universe", + "income_observations", + "prepared_receipt", +) +#: Every edge the composed CREATE produces, the origin document included. The +#: origin is positional over that same CREATE execution, so it is held to the +#: same producer identity as the evidence. Its content is independently +#: recomputed against the actual tables (``ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME``) +#: and that check is unchanged — but recomputation cannot tell two CREATE +#: executions describing identical tables apart, so the producer key the +#: document records would otherwise be carried unverified. +_CREATE_EDGES = (*_SOURCE_EDGES, "source_origin") + +_BINDING_KEYS = frozenset( + { + "arm", + "arm_channel", + "arm_row_columns", + "certified", + "claim", + "cohorts", + "composed", + "composed_is_whole_arm", + "demographics", + "inputs", + "origin", + "phase", + "prepared_arm_rows", + "release_eligible", + "resolution", + "schema", + } +) +_ARM_KEYS = frozenset({"rows", "composed_ordered_ids_sha256", "mask_sha256"}) +_ARM_ID_KEYS = _ARM_KEYS | { + "original_ordered_ids_sha256", + "source_positions_sha256", +} + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def refuse_unbound_demographic_writes(columns: Iterable[str]) -> None: + """Refuse if this stage would own a demographic its binding records as open. + + This is the fail-closed half of the recorded gap. The binding document says + ``is_female`` and ``is_household_head`` are not bound on this population; + that statement is only honest while no node of the stage writes them, so + every owned-column roster of the stage passes through here. A reviewed + mapping that started producing one of them would refuse at declaration time + rather than silently turning a recorded gap into an unreviewed binding. + """ + for column in columns: + _require( + column not in UNBOUND_DEMOGRAPHIC_COLUMNS, + f"ASEC_DEMOGRAPHIC_COLUMN_BOUND:{column}", + ) + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +@dataclass(frozen=True) +class ComposedAsecBinding: + """One resolved arm binding: the row masks, the source rows and its document.""" + + document: Mapping[str, object] + context_document: Mapping[str, object] + masks: Mapping[str, np.ndarray] + positions: Mapping[str, np.ndarray] + original_ids: Mapping[str, np.ndarray] + income_years: np.ndarray + selected_money: bytes + + @property + def payload(self) -> bytes: + return canonical_json(self.document) + + +def arm_mask(table: pd.DataFrame, entity: str) -> np.ndarray: + """Which rows of a composed entity table came from the ASEC arm.""" + column = support_channel_column(entity) + _require(column in table, f"ARM_CHANNEL_COLUMN:{entity}") + channels = table[column] + _require(not bool(channels.isna().any()), f"ARM_CHANNEL_MISSING:{entity}") + values = channels.astype(str).to_numpy() + _require( + set(values.tolist()) <= {ARM_CHANNEL, _OTHER_CHANNEL}, + f"ARM_CHANNEL_UNKNOWN:{entity}", + ) + mask = values == ARM_CHANNEL + _require(bool(mask.any()), f"ARM_EMPTY:{entity}") + return mask + + +def _original_ids(table: pd.DataFrame, entity: str, mask: np.ndarray) -> np.ndarray: + """The arm's own pre-remap identifiers, never the post-offset source IDs.""" + column = spine_source_id_column(entity) + _require(column in table, f"ARM_SPINE_SOURCE_ID:{entity}") + return _int64_view(table.loc[mask, column], reason=f"ARM_SPINE_SOURCE_ID:{entity}") + + +def _resolved_positions( + original_ids: np.ndarray, arm_ids: np.ndarray, *, entity: str +) -> np.ndarray: + """Locate each original ID in the source's own ordered identity. + + Every failure mode refuses instead of dropping a row: a source identity that + repeats an ID is ambiguous, an original ID the source never declares is + unresolved, and a non-increasing mapping cannot be a whole-household slice of + the source buffers, which the reviewed selector requires to be strictly + ordered. The order is a property of the actual mapping, checked here; it is + never imposed by sorting the rows. + """ + _require(len(original_ids) > 0, f"BINDING_EMPTY:{entity}") + _require( + len(np.unique(arm_ids)) == len(arm_ids), f"BINDING_AMBIGUOUS_SOURCE:{entity}" + ) + _require( + len(np.unique(original_ids)) == len(original_ids), + f"BINDING_AMBIGUOUS_ROWS:{entity}", + ) + positions = pd.Index(arm_ids).get_indexer(original_ids).astype("int64") + _require(bool((positions >= 0).all()), f"BINDING_UNRESOLVED:{entity}") + _require( + len(positions) < 2 or bool((np.diff(positions) > 0).all()), + f"BINDING_NON_MONOTONE:{entity}", + ) + return positions + + +def _int64_view(series: pd.Series, *, reason: str) -> np.ndarray: + """Narrow the stack's nullable storage back to int64 on this arm's own rows. + + Stacking widens an arm-specific integer column to ``Int64`` so the opposite + arm's absence stays a declared null. On the arm's own rows nothing may be + missing and the reviewed verifiers compare native ``int64`` buffers, so the + narrowing is explicit and refuses rather than filling. + """ + _require(pd.api.types.is_integer_dtype(series.dtype), f"{reason}_DTYPE") + _require(not bool(series.isna().any()), f"{reason}_MISSING") + return series.to_numpy(dtype="int64") + + +def _arm_household_evidence( + household: pd.DataFrame, mask: np.ndarray, original_ids: np.ndarray +) -> pd.DataFrame: + """The arm's carried housing evidence restated under its own source identity. + + ``verify_graph_housing_rows`` compares native source buffers against a + household view keyed by ``household_id``. On a composed population that + column is the assembly-remapped identity, so this view restates the arm's own + pre-remap identity instead. The evidence columns themselves are carried + unchanged; only the stack's nullable storage is narrowed. + """ + view = pd.DataFrame(index=pd.RangeIndex(int(mask.sum()))) + view["household_id"] = original_ids + for column in HOUSEHOLD_EVIDENCE_COLUMNS: + _require(column in household, f"ARM_HOUSING_EVIDENCE:{column}") + view[column] = _int64_view( + household.loc[mask, column], reason=f"ARM_HOUSING_EVIDENCE:{column}" + ) + return view + + +def _arm_household_cohorts( + person: pd.DataFrame, + person_mask: np.ndarray, + income_years: np.ndarray, + composed_household_ids: np.ndarray, +) -> np.ndarray: + """One source income year per ASEC household; a mixed household refuses. + + The household evidence is cohort-scoped, so the housing verifier is given the + year the arm's own persons declare rather than the year the source buffer + would have supplied — a household whose members disagree is a real source + contract failure and must not be reduced away. + """ + membership = person.loc[ + person_mask, US_SCHEMA.membership_column("household") + ].to_numpy(dtype="int64") + pairs = pd.DataFrame({"household": membership, "year": income_years}) + distinct = pairs.drop_duplicates() + _require( + not bool(distinct["household"].duplicated().any()), "ARM_MIXED_YEAR_HOUSEHOLD" + ) + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require( + set(lookup) == set(composed_household_ids.tolist()), + "ARM_HOUSEHOLD_COHORT_COVERAGE", + ) + return np.asarray( + [lookup[int(value)] for value in composed_household_ids], dtype="int64" + ) + + +def _composed_document(payload: bytes) -> dict: + """Decode the composed population's typed context without relaxing anything. + + The prepared slice's ``_document`` refuses exactly the three things a + composed population always has — assembly metadata, a mass history and an + importance household weight — so it cannot be reused. This decoder keeps the + canonical-bytes, normative-metadata and mass-record validation that + ``graph_context`` owns and asserts the composed shape positively. + """ + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "COMPOSED_CONTEXT_CANONICAL") + _require(set(document["weight_sources"]) == {"household"}, "COMPOSED_WEIGHT_ENTITY") + return document + + +def _prepared_document(payload: bytes) -> dict: + """The prepared arm's own pre-stack context, which the evidence is ordered in.""" + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "PREPARED_CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require( + document["weight_sources"] == {"household": "design"}, + "PREPARED_WEIGHT_AUTHORITY", + ) + return document + + +def _origin_entity_digests(tables: Mapping[str, pd.DataFrame]) -> dict: + """Recompute the origin document's per-entity mapping from the actual rows. + + ``bind_composed_source_origin`` deliberately verifies no digest against any + population, because it receives only bytes. A consumer that relies on the + mapping has to recompute it against the frame in hand; this is that + recomputation, in the producer's own order and encoding. + """ + digests = {} + for entity in US_SCHEMA.entities: + table = tables[entity] + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + for column in (channel_column, spine_column, source_column): + _require(column in table, f"ORIGIN_RECOMPUTE_COLUMN:{entity}.{column}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + digests[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + return digests + + +def _demographics(prepared_columns: Sequence[str], cohorts: Mapping[str, int]) -> dict: + """What a cross-arm demographic can honestly claim on this population today. + + ``age`` is bound through the reviewed corrected-leaf mapping whose only input + is the routing column ``A_AGE``. Its ASEC observation is the interview + household one year after the income year — the reference period the income + artifact declares and this node verifies cell by cell — while the native ACS + arm's ``age`` is observed in its own vintage. The two are **not** harmonized + here and no person is aged from any date; the periods are recorded per cohort + so a calibration decides rather than inherits an assumption. + + ``is_female`` and ``is_household_head`` stay unbound. What their entries say + about the source is **observed on the prepared arm actually in hand**, and + is stated both ways: which required source columns this population carries + and which it does not. That observation is evidence, not authority — + ``bound_by_this_stage`` is ``False`` either way, and + :func:`refuse_unbound_demographic_writes` is what keeps it true. + + Each entry also records the explicit source contract that would supply the + column, the mappings deliberately **not** adopted as proof of it, and the + diagnostic-only columns that are crosschecked against that contract and + never substituted for it. + """ + present = set(prepared_columns) + unbound = [] + for ( + column, + sources, + diagnostics, + contract, + not_adopted, + caveats, + acs_mapping, + ) in _UNBOUND_DEMOGRAPHICS: + found = sorted(name for name in sources if name in present) + absent = sorted(name for name in sources if name not in present) + unbound.append( + { + "column": column, + "required_asec_source_columns": list(sources), + "present_in_prepared_arm": found, + "absent_from_prepared_arm": absent, + "diagnostic_only_source_columns": list(diagnostics), + "diagnostic_only_present_in_prepared_arm": sorted( + name for name in diagnostics if name in present + ), + "diagnostic_only_is_semantic_proof": False, + "bound_by_this_stage": False, + "explicit_asec_source_contract": contract, + "asec_mappings_not_adopted": [ + {"mapping": mapping, "refusal": refusal} + for mapping, refusal in not_adopted + ], + "presence_certifies_observation": False, + "presence_caveats": list(caveats), + "acs_arm_mapping": acs_mapping, + "reason": ( + "required_asec_source_column_absent_from_this_prepared_population" + if absent + else "no_reviewed_cross_arm_mapping_decision_for_this_column" + ), + "resolution": "independent_source_decision_required", + } + ) + bound = [ + { + "column": "age", + "dtype": "float64", + "asec_source_column": "A_AGE", + "mapping": "derive_cps_carried_current_leaves", + "observed_period_kind": "interview_household_one_year_after_income_year", + "observed_period_by_cohort": { + year: {"income_year": int(year), "asec_survey_year": int(year) + 1} + for year in sorted(cohorts) + }, + "aged_from_a_date": False, + "cross_arm_period_harmonized": False, + "acs_arm_note": ( + "the native ACS arm's age is carried unchanged in its own " + "vintage and is never rewritten by this binding" + ), + } + ] + refuse_unbound_demographic_writes(item["column"] for item in bound) + return {"bound": bound, "unbound": unbound} + + +def resolve_composed_asec_binding( + tables: Mapping[str, pd.DataFrame], + *, + context_payload: bytes, + origin_payload: bytes, + prepared_context_payload: bytes, + money_payload: bytes, + receipt_payload: bytes, + housing_payload: bytes, + income_payload: bytes, + producers: Mapping[str, str], +) -> ComposedAsecBinding: + """Resolve the ASEC arm's composed rows against the carried source evidence.""" + context = _composed_document(context_payload) + origin = bind_composed_source_origin(origin_payload) + prepared_context = _prepared_document(prepared_context_payload) + receipt = json.loads(receipt_payload) + _require(canonical_json(receipt) == receipt_payload, "PREPARED_RECEIPT_JSON") + _require( + receipt["source_kind"] == PREPARED_SOURCE_KIND + and receipt["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + for entity in _ID_ENTITIES: + _require( + int(receipt["entity_rows"][entity]) + == int(prepared_context["entities"][entity]["rows"]), + f"PREPARED_EVIDENCE_ROWS:{entity}", + ) + income = bind_income_observations(income_payload, prepared_receipt=receipt) + universe = bind_housing_universe(housing_payload, prepared_receipt=receipt) + body = parse_current_money_body( + money_payload, + expected_header_sha256=receipt["money_header_sha256"], + expected_content_sha256=receipt["money_content_sha256"], + field_entities=tuple(tuple(item) for item in receipt["field_entities"]), + ) + _require( + body.person_rows == int(receipt["entity_rows"]["person"]) + and body.household_rows == int(receipt["entity_rows"]["household"]), + "BODY_ROW_ALIGNMENT", + ) + + for entity in US_SCHEMA.entities: + _require(entity in tables, f"COMPOSED_TABLE_MISSING:{entity}") + declared = context["entities"][entity] + actual = _row_identity(tables[entity], entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"COMPOSED_CONTEXT_IDENTITY:{entity}", + ) + _require( + _origin_entity_digests(tables) == origin["assembled"], + "ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME", + ) + + masks = {entity: arm_mask(tables[entity], entity) for entity in US_SCHEMA.entities} + alignment = origin["asec_evidence_alignment"] + for entity in US_SCHEMA.entities: + _require( + int(alignment["composed_rows"][entity]) == int(masks[entity].sum()), + f"ORIGIN_COMPOSED_ROWS:{entity}", + ) + whole = bool(alignment["composed_is_whole_arm"]) + _require( + whole == (alignment["composed_rows"] == alignment["arm_rows"]), + "ORIGIN_WHOLE_ARM_FLAG", + ) + for entity in _ID_ENTITIES: + _require( + int(alignment["evidence_rows"][entity]) + == int(receipt["entity_rows"][entity]), + f"ORIGIN_EVIDENCE_ROWS:{entity}", + ) + _require( + int(masks[entity].sum()) <= int(receipt["entity_rows"][entity]), + f"ARM_OVERFLOW:{entity}", + ) + + original = { + entity: _original_ids(tables[entity], entity, masks[entity]) + for entity in _ID_ENTITIES + } + arm_identity = { + "person": income.array("person_id").astype("int64"), + "household": universe.array("household_id").astype("int64"), + } + for entity in _ID_ENTITIES: + _require( + len(arm_identity[entity]) == int(receipt["entity_rows"][entity]), + f"ARM_IDENTITY_ROWS:{entity}", + ) + positions = { + entity: _resolved_positions( + original[entity], arm_identity[entity], entity=entity + ) + for entity in _ID_ENTITIES + } + + person = tables["person"] + income_years = _int64_view( + person.loc[masks["person"], COHORT_COLUMN], reason="ARM_COHORT" + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions["person"]]), + "BINDING_COHORT_MISMATCH", + ) + for column in PERSON_COORDINATE_CROSSCHECKS: + _require(column in person, f"ARM_CROSSCHECK_COLUMN:{column}") + _require( + np.array_equal( + _int64_view( + person.loc[masks["person"], column], + reason=f"ARM_CROSSCHECK:{column}", + ), + income.array(column)[positions["person"]], + ), + f"BINDING_CROSSCHECK:{column}", + ) + composed_household_ids = ( + tables["household"] + .loc[masks["household"], US_SCHEMA.entity_id_column("household")] + .to_numpy(dtype="int64") + ) + verify_graph_housing_rows( + _arm_household_evidence( + tables["household"], masks["household"], original["household"] + ), + universe, + positions=positions["household"], + income_years=_arm_household_cohorts( + person, masks["person"], income_years, composed_household_ids + ), + ) + + identities = { + entity: _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): original[entity]}), entity + ) + for entity in _ID_ENTITIES + } + cohorts = { + str(int(year)): int((income_years == year).sum()) + for year in np.unique(income_years).tolist() + } + arm = {} + for entity in BOUND_ENTITIES: + table = tables[entity] + id_column = US_SCHEMA.entity_id_column(entity) + composed_ids = table.loc[masks[entity], id_column].to_numpy(dtype="int64") + record = { + "rows": int(masks[entity].sum()), + "composed_ordered_ids_sha256": _row_identity( + pd.DataFrame({id_column: composed_ids}), entity + )["ordered_ids_sha256"], + "mask_sha256": _sha(masks[entity].tobytes()), + } + if entity in _ID_ENTITIES: + record["original_ordered_ids_sha256"] = identities[entity][ + "ordered_ids_sha256" + ] + record["source_positions_sha256"] = _sha(positions[entity].tobytes()) + arm[entity] = record + + document = { + "schema": BINDING_SCHEMA, + "phase": COMPOSED_ASEC_PHASE, + "release_eligible": False, + "certified": False, + "claim": "engineering_binding_only_no_calibration_or_district_claim", + "arm_channel": ARM_CHANNEL, + "arm_row_columns": { + entity: arm_row_column(entity) for entity in BOUND_ENTITIES + }, + "resolution": { + "keys": ["support_channel", "spine_source_id", COHORT_COLUMN], + "source_identity": { + "household": "housing_universe.household_id", + "person": "income_observations.person_id", + }, + "crosschecks": list(PERSON_COORDINATE_CROSSCHECKS), + "uses_composed_row_position": False, + "uses_post_offset_source_id": False, + }, + "composed": { + entity: { + "rows": int(context["entities"][entity]["rows"]), + "ordered_ids_sha256": context["entities"][entity]["ordered_ids_sha256"], + } + for entity in US_SCHEMA.entities + }, + "arm": arm, + "prepared_arm_rows": { + entity: int(receipt["entity_rows"][entity]) for entity in _ID_ENTITIES + }, + "composed_is_whole_arm": whole, + "cohorts": cohorts, + "demographics": _demographics( + prepared_context["entities"]["person"]["columns"], cohorts + ), + "inputs": { + "asec_frame_context_sha256": _sha(prepared_context_payload), + "current_money_sha256": _sha(money_payload), + "frame_context_sha256": _sha(context_payload), + "housing_universe_sha256": _sha(housing_payload), + "income_observations_sha256": _sha(income_payload), + "money_content_sha256": receipt["money_content_sha256"], + "money_header_sha256": receipt["money_header_sha256"], + "prepared_receipt_sha256": _sha(receipt_payload), + "producers": {name: producers[name] for name in sorted(producers)}, + "source_origin_sha256": _sha(origin_payload), + }, + "origin": { + "preparation_sha256": origin["preparation_sha256"], + "sample_fraction": float(origin["sampling"]["sample_fraction"]), + "sample_seed": int(origin["sampling"]["sample_seed"]), + }, + } + payload = canonical_json(document) + selected = select_current_money( + body, + person_positions=positions["person"], + household_positions=positions["household"], + prepared_receipt_sha256=_sha(receipt_payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in BOUND_ENTITIES: + context["entities"][entity]["columns"].append(arm_row_column(entity)) + return ComposedAsecBinding( + document=document, + context_document=context, + masks={entity: masks[entity] for entity in BOUND_ENTITIES}, + positions=positions, + original_ids=original, + income_years=income_years, + selected_money=encode_selected_current_money(selected), + ) + + +def bind_composed_asec_document(payload: bytes) -> dict: + """Decode the binding artifact as a typed shape, refusing a malformed one. + + Like the origin binder this checks canonical bytes and declared shape only. + It verifies no digest against any population; a consumer holding the frame + recomputes what it needs. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "BINDING_CANONICAL") + _require(isinstance(document, dict), "BINDING_DOCUMENT") + _require(set(document) == _BINDING_KEYS, "BINDING_KEYS") + _require(document["schema"] == BINDING_SCHEMA, "BINDING_SCHEMA") + _require(document["phase"] == COMPOSED_ASEC_PHASE, "BINDING_PHASE") + _require(document["arm_channel"] == ARM_CHANNEL, "BINDING_ARM_CHANNEL") + _require( + document["arm_row_columns"] + == {entity: arm_row_column(entity) for entity in BOUND_ENTITIES}, + "BINDING_ARM_ROW_COLUMNS", + ) + _require( + document["release_eligible"] is False and document["certified"] is False, + "BINDING_RELEASE_CLAIM", + ) + _require( + set(document["composed"]) == set(US_SCHEMA.entities), + "BINDING_COMPOSED_ENTITIES", + ) + _require(set(document["arm"]) == set(BOUND_ENTITIES), "BINDING_ARM_ENTITIES") + for entity, record in document["arm"].items(): + expected = _ARM_ID_KEYS if entity in _ID_ENTITIES else _ARM_KEYS + _require(set(record) == expected, f"BINDING_ARM_KEYS:{entity}") + _require( + type(record["rows"]) is int and record["rows"] > 0, + f"BINDING_ARM_ROWS:{entity}", + ) + _require( + all( + _is_digest(value) + for key, value in record.items() + if key.endswith("_sha256") + ), + f"BINDING_ARM_DIGEST:{entity}", + ) + resolution = document["resolution"] + _require( + resolution["uses_composed_row_position"] is False + and resolution["uses_post_offset_source_id"] is False + and resolution["keys"] == ["support_channel", "spine_source_id", COHORT_COLUMN] + and resolution["crosschecks"] == list(PERSON_COORDINATE_CROSSCHECKS), + "BINDING_RESOLUTION", + ) + demographics = document["demographics"] + _require( + set(demographics) == {"bound", "unbound"} + and bool(demographics["bound"]) + and all(item["aged_from_a_date"] is False for item in demographics["bound"]), + "BINDING_DEMOGRAPHICS", + ) + # What the unbound entries must satisfy is that they are unbound and that + # they say the whole truth about the source, not that the source lacked the + # column: a genuine prepared arm does carry some of these, and a document + # recording that is correct. Every required column is accounted for exactly + # once, on one side or the other. A diagnostic-only column is held apart + # from the requirement and is never allowed to claim it proves anything. + _require( + [item["column"] for item in demographics["unbound"]] + == list(UNBOUND_DEMOGRAPHIC_COLUMNS) + and all( + item["bound_by_this_stage"] is False + and sorted( + [*item["present_in_prepared_arm"], *item["absent_from_prepared_arm"]] + ) + == sorted(item["required_asec_source_columns"]) + and not set(item["present_in_prepared_arm"]) + & set(item["absent_from_prepared_arm"]) + and item["diagnostic_only_is_semantic_proof"] is False + and not set(item["diagnostic_only_source_columns"]) + & set(item["required_asec_source_columns"]) + and set(item["diagnostic_only_present_in_prepared_arm"]) + <= set(item["diagnostic_only_source_columns"]) + and bool(item["explicit_asec_source_contract"]) + and bool(item["asec_mappings_not_adopted"]) + and item["presence_certifies_observation"] is False + and bool(item["presence_caveats"]) + and all( + bool(entry["mapping"]) and bool(entry["refusal"]) + for entry in item["asec_mappings_not_adopted"] + ) + for item in demographics["unbound"] + ), + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + # Presence varies with the prepared roster. The required columns and their + # semantics do not: a transported document cannot redefine this registry. + declared_presence = [ + name + for item in demographics["unbound"] + for name in ( + *item["present_in_prepared_arm"], + *item["diagnostic_only_present_in_prepared_arm"], + ) + ] + _require( + demographics["unbound"] == _demographics(declared_presence, {})["unbound"], + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + refuse_unbound_demographic_writes(item["column"] for item in demographics["bound"]) + _require( + all( + _is_digest(value) + for key, value in document["inputs"].items() + if key.endswith("_sha256") + ) + and all( + _is_digest(value) for value in document["inputs"]["producers"].values() + ), + "BINDING_INPUT_DIGESTS", + ) + _require( + type(document["composed_is_whole_arm"]) is bool + and bool(document["cohorts"]) + and all( + type(count) is int and count > 0 for count in document["cohorts"].values() + ), + "BINDING_COHORTS", + ) + return document + + +#: The resolved source rows, as a framed binary artifact rather than a JSON list. +#: The measurement nodes need the arm's original IDs and its source positions to +#: reconstruct the reviewed accounting, and they must get them without reading a +#: provenance column. This carries exactly those four int64 buffers, bound to the +#: binding document that already digests every one of them. +ARM_ROWS_MAGIC = b"MCCASECR\x01" +ARM_ROWS_KIND = "microcosm.us.composed_asec_arm_rows.v1" +_ARM_ROWS_BUFFERS = ( + ("person", "source_positions"), + ("person", "original_ids"), + ("household", "source_positions"), + ("household", "original_ids"), +) +_ARM_ROWS_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "binding_sha256", + "buffers", + "rows", + "dtype", + } +) +ARM_ROWS_PAYLOAD_MAX_BYTES = ( + len(ARM_ROWS_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 16 * (MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) +_ARM_ROWS_TOKEN = object() + + +@dataclass(frozen=True) +class BoundComposedAsecArmRows: + """The arm's resolved source rows, admitted only against its binding document.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _ARM_ROWS_TOKEN, "ARM_ROWS_CONSTRUCTOR") + + def array(self, entity: str, name: str) -> np.ndarray: + _require((entity, name) in _ARM_ROWS_BUFFERS, "ARM_ROWS_BUFFER") + return np.frombuffer( + self.buffers[_ARM_ROWS_BUFFERS.index((entity, name))], dtype=" tuple[bytes, tuple[bytes, ...]]: + values = {"source_positions": positions, "original_ids": original_ids} + buffers = tuple( + np.ascontiguousarray(values[name][entity], dtype=" bytes: + """Frame the resolved source rows with a transport checksum over their bytes.""" + header, buffers = _arm_rows_parts( + positions, original_ids, binding_sha256=binding_sha256 + ) + payload = ( + ARM_ROWS_MAGIC + struct.pack(" BoundComposedAsecArmRows: + """Admit the resolved rows only against the binding document that digests them. + + Every buffer is re-digested here in the binding document's own encodings: the + original IDs through the graph context's ordered-identity digest and the + source positions through their raw bytes. A payload that does not reproduce + both, for the exact row counts the binding declares, is refused. + """ + _require( + type(payload) is bytes + and len(ARM_ROWS_MAGIC) + 4 + 32 < len(payload) <= ARM_ROWS_PAYLOAD_MAX_BYTES, + "ARM_ROWS_SIZE", + ) + _require(payload.startswith(ARM_ROWS_MAGIC), "ARM_ROWS_MAGIC") + _require(_sha(payload[:-32]) == payload[-32:].hex(), "ARM_ROWS_CHECKSUM") + size = struct.unpack_from("= 0).all()) + and (len(places) < 2 or bool((np.diff(places) > 0).all())), + f"ARM_ROWS_ORDER:{entity}", + ) + declared = binding_document["arm"][entity] + _require( + _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): ids}), entity + )["ordered_ids_sha256"] + == declared["original_ordered_ids_sha256"], + f"ARM_ROWS_IDENTITY:{entity}", + ) + _require( + _sha(places.tobytes()) == declared["source_positions_sha256"], + f"ARM_ROWS_POSITIONS:{entity}", + ) + return bound + + +def arm_row_declarations() -> tuple[Owned, ...]: + """The boolean arm-membership cells this node owns, in one canonical order.""" + return tuple( + Owned(entity, arm_row_column(entity), ARM_ROW_DTYPE) + for entity in BOUND_ENTITIES + ) + + +def bind_node_inputs() -> tuple[Slice, ...]: + """The declared views the resolution reads, and nothing else.""" + provenance = tuple( + ( + support_channel_column(entity), + spine_source_id_column(entity), + support_source_id_column(entity), + ) + for entity in US_SCHEMA.entities + ) + extra = { + "person": (COHORT_COLUMN, *PERSON_COORDINATE_CROSSCHECKS), + "household": HOUSEHOLD_EVIDENCE_COLUMNS, + } + return tuple( + Slice(entity, (*columns, *extra.get(entity, ()))) + for entity, columns in zip(US_SCHEMA.entities, provenance, strict=True) + ) + + +def bind_node_artifact_inputs(*, population_context: str) -> tuple[ArtifactInput, ...]: + """The population's own context plus the composed CREATE's carried evidence.""" + return ( + ArtifactInput( + "frame_context", population_context, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *( + ArtifactInput(name, CREATE_NODE, name, kind) + for name, kind in _ARTIFACT_TYPES + if name != "frame_context" + ), + ) + + +_BIND_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ArtifactOutput("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), +) + + +def composed_asec_bind_node(*, population: str, population_context: str) -> Node: + """Declare the one binding node over an already composed population version.""" + return Node( + id=BIND_NODE, + kernel=USComposedAsecBindKernel.ref, + population=population, + inputs=bind_node_inputs(), + outputs=arm_row_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=bind_node_artifact_inputs( + population_context=population_context + ), + artifact_outputs=_BIND_ARTIFACTS, + ) + + +class USComposedAsecBindKernel(KernelBase): + """Resolve the ASEC arm's composed rows and slice its evidence to them.""" + + ref = "us.composed_population.asec_bind@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == bind_node_inputs(), "NODE_SLICES") + _require(node.outputs == arm_row_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) == {name for name, _kind in _ARTIFACT_TYPES}, + "NODE_ARTIFACT_INPUTS", + ) + _require(node.artifact_outputs == _BIND_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + _require( + { + edge.producer + for edge in node.artifact_inputs + if edge.name in _CREATE_EDGES + } + == {CREATE_NODE}, + "NODE_EVIDENCE_PRODUCER", + ) + values = { + name: _artifact(context, name, kind) for name, kind in _ARTIFACT_TYPES + } + create_producer = _same_producer(context, _CREATE_EDGES) + binding = resolve_composed_asec_binding( + {entity: context.tables[entity] for entity in US_SCHEMA.entities}, + context_payload=values["frame_context"].payload, + origin_payload=values["source_origin"].payload, + prepared_context_payload=values["asec_frame_context"].payload, + money_payload=values["current_money"].payload, + receipt_payload=values["prepared_receipt"].payload, + housing_payload=values["housing_universe"].payload, + income_payload=values["income_observations"].payload, + producers={ + "population": values["frame_context"].producer_key, + "prepared_source": create_producer, + "source_origin": values["source_origin"].producer_key, + }, + ) + columns = {} + for entity in BOUND_ENTITIES: + id_column = US_SCHEMA.entity_id_column(entity) + index = pd.Index( + context.tables[entity][id_column].to_numpy(), name=id_column + ) + columns[(entity, arm_row_column(entity))] = pd.Series( + binding.masks[entity], index=index, dtype=ARM_ROW_DTYPE + ) + payload = binding.payload + return KernelResult( + columns=columns, + artifacts={ + "frame_context": canonical_json(binding.context_document), + "asec_binding": payload, + "arm_rows": encode_composed_asec_arm_rows( + binding.positions, + binding.original_ids, + binding_sha256=_sha(payload), + ), + "selected_current_money": binding.selected_money, + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "binding_sha256": _sha(payload), + "selected_money_sha256": _sha(binding.selected_money), + "arm_rows": { + entity: int(binding.masks[entity].sum()) + for entity in BOUND_ENTITIES + }, + "cohorts": binding.document["cohorts"], + "demographics": binding.document["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +__all__ = [ + "ARM_CHANNEL", + "ARM_ROW_DTYPE", + "ARM_ROW_SUFFIX", + "BINDING_SCHEMA", + "BIND_NODE", + "BOUND_ENTITIES", + "COHORT_COLUMN", + "COMPOSED_ASEC_DEPENDENCIES", + "COMPOSED_ASEC_PHASE", + "COMPOSED_ASEC_STAGE", + "PERSON_COORDINATE_CROSSCHECKS", + "UNBOUND_DEMOGRAPHIC_COLUMNS", + "US_COMPOSED_ASEC_ARM_ROWS_TYPE", + "US_COMPOSED_ASEC_BINDING_TYPE", + "BoundComposedAsecArmRows", + "ComposedAsecBinding", + "USComposedAsecBindKernel", + "arm_mask", + "arm_row_column", + "arm_row_declarations", + "bind_composed_asec_arm_rows", + "bind_composed_asec_document", + "bind_node_artifact_inputs", + "bind_node_inputs", + "composed_asec_bind_node", + "encode_composed_asec_arm_rows", + "refuse_unbound_demographic_writes", + "resolve_composed_asec_binding", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py new file mode 100644 index 000000000..2a937fd55 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py @@ -0,0 +1,650 @@ +"""Corrected ASEC monetary and demographic measurements over composed ASEC rows. + +Two pure nodes hang off :mod:`.graph_composed_asec_binding`. Each writes only +under the binding's declared boolean arm mask, so the executor holds every +native ACS cell — value and missingness alike — byte-identical: a masked +``Owned`` is checked by ``population._patch_columns``, which refuses any change +to non-owned storage. Where a column is new the opposite arm keeps a declared +null; nothing is inferred as zero. + +Neither kernel reads a source-channel or spine-source-ID column. They receive +the arm's rows through the declared mask and the arm's source coordinates +through the typed ``arm_rows`` artifact, both already resolved and digested by +the binding node. The arithmetic is the accepted one, unchanged and imported: +:func:`~.cps_carried_current.derive_cps_carried_current_leaves` for the +corrected leaves (which is also where ``age`` comes from, mapped from ``A_AGE``) +and :func:`~.graph_asec_income.derive_reported_income` for the six reported +observations. Raw source observations and their quality/zero/allocation axes +stay inside those artifacts; only the computed quantities become cells. + +Nothing here is a release, a calibration, a transfer or a score. +""" + +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + decode_selected_current_money, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + bind_income_observations, + derive_reported_income, + encode_reported_income, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_asec_binding import ( + BIND_NODE, + COHORT_COLUMN, + COMPOSED_ASEC_DEPENDENCIES, + COMPOSED_ASEC_PHASE, + COMPOSED_ASEC_STAGE, + US_COMPOSED_ASEC_ARM_ROWS_TYPE, + US_COMPOSED_ASEC_BINDING_TYPE, + USComposedAsecBindKernel, + arm_row_column, + bind_composed_asec_arm_rows, + bind_composed_asec_document, + composed_asec_bind_node, + refuse_unbound_demographic_writes, +) +from .graph_composed_contracts import ( + LEAVES_NODE as LEAVES_NODE, + REPORTED_INCOME_NODE as REPORTED_INCOME_NODE, +) +from .graph_composed_population import ( + COMPOSED_SOURCES, + CREATE_NODE, + GEOGRAPHY_PHASE, + HARMONIZE_NODE, + composed_population_nodes, + composed_population_registry, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_geography import LOOKUP_SOURCES, us_geography_nodes +from .graph_implementation import implementation_hash, implementation_manifest + +LEAF_DTYPE = "float64" + +#: Corrected leaves the native ACS arm already carries under the same name. +#: On those the masked write fills the ASEC rows and the executor holds every +#: ACS cell byte-identical; on every other leaf the column is new and the ACS +#: rows stay declared-null. Any other incumbent is unreviewed and refuses. +ACS_SHARED_LEAVES = ( + "age", + "employment_income_before_lsr", + "self_employment_income_before_lsr", +) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _leaf_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def composed_leaf_declarations() -> tuple[Owned, ...]: + """The corrected-leaf cells, owned only at the bound arm's rows. + + The binding document records ``is_female`` and ``is_household_head`` as + unbound on this population. That record stays honest only while no node of + the stage writes them, so the declaration itself refuses a leaf roster that + would (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``) rather than letting a reviewed + mapping quietly turn a recorded gap into an unreviewed binding. + """ + coordinates = _leaf_coordinates() + refuse_unbound_demographic_writes(column for _entity, column in coordinates) + return tuple( + Owned(entity, column, LEAF_DTYPE, rows=arm_row_column(entity)) + for entity, column in coordinates + ) + + +def composed_reported_income_declarations() -> tuple[Owned, ...]: + """The six reported-income cells, owned only at the bound arm's rows.""" + refuse_unbound_demographic_writes(RESULT_COLUMNS) + return tuple( + Owned("person", name, LEAF_DTYPE, rows=arm_row_column("person")) + for name in RESULT_COLUMNS + ) + + +def _document(value) -> dict: + """Decode the producing node's context, keeping every graph-context check.""" + document = _decode(value.payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + return document + + +def _bound_population(document: dict, binding: dict) -> None: + """The context and the binding must describe one composed population.""" + for entity in US_SCHEMA.entities: + declared = document["entities"][entity] + recorded = binding["composed"][entity] + _require( + declared["rows"] == recorded["rows"] + and declared["ordered_ids_sha256"] == recorded["ordered_ids_sha256"], + f"BINDING_POPULATION_IDENTITY:{entity}", + ) + for entity, column in binding["arm_row_columns"].items(): + _require( + column == arm_row_column(entity) + and column in document["entities"][entity]["columns"], + f"BINDING_ARM_COLUMN:{entity}", + ) + + +def _arm_rows(table: pd.DataFrame, entity: str, binding: dict) -> np.ndarray: + """The masked view's own ids, checked against the binding's recorded arm.""" + id_column = US_SCHEMA.entity_id_column(entity) + ids = table[id_column].to_numpy(dtype="int64") + recorded = binding["arm"][entity] + _require(len(ids) == recorded["rows"], f"ARM_VIEW_ROWS:{entity}") + _require( + _row_identity(pd.DataFrame({id_column: ids}), entity)["ordered_ids_sha256"] + == recorded["composed_ordered_ids_sha256"], + f"ARM_VIEW_IDENTITY:{entity}", + ) + column = arm_row_column(entity) + _require( + column in table and bool(table[column].to_numpy().all()), + f"ARM_VIEW_MASK:{entity}", + ) + return ids + + +def _selected_money(binding: dict, prepared: dict, values): + """Decode the arm's money slice against the binding it was cut for.""" + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["asec_binding"].payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == binding["arm"]["person"]["original_ordered_ids_sha256"] + and header["household_identity_sha256"] + == binding["arm"]["household"]["original_ordered_ids_sha256"], + "SELECTED_ARM_COORDINATES", + ) + _require( + selected.person_rows == binding["arm"]["person"]["rows"] + and selected.household_rows == binding["arm"]["household"]["rows"], + "SELECTED_ARM_ROWS", + ) + return selected + + +def _binding(values) -> dict: + binding = bind_composed_asec_document(values["asec_binding"].payload) + _require( + binding["inputs"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "BINDING_PREPARED_RECEIPT", + ) + _require( + binding["inputs"]["producers"]["prepared_source"] + == values["prepared_receipt"].producer_key, + "BINDING_SOURCE_PRODUCER", + ) + return binding + + +def _prepared(value) -> dict: + """The producing CREATE's own receipt, re-read as canonical bytes.""" + document = json.loads(value.payload) + _require(canonical_json(document) == value.payload, "PREPARED_RECEIPT_JSON") + _require(document["release_eligible"] is False, "PREPARED_RECEIPT_SCHEMA") + return document + + +def _new_columns(document: dict, coordinates) -> None: + """Record only the coordinates this stage actually adds to the population.""" + for entity, column in coordinates: + columns = document["entities"][entity]["columns"] + if column not in columns: + columns.append(column) + + +class USComposedAsecLeavesKernel(KernelBase): + """Derive the corrected monetary and age leaves over the bound arm's rows.""" + + ref = "us.composed_population.asec_cps_carried_current@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _leaves_inputs(), "NODE_SLICES") + _require(node.outputs == composed_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ) + } + _same_producer( + context, + ("frame_context", "asec_binding", "arm_rows", "selected_current_money"), + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _leaf_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require( + set(incumbent) <= {f"person.{name}" for name in ACS_SHARED_LEAVES}, + f"UNREVIEWED_LEAF_INCUMBENT:{incumbent}", + ) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + person_ids = _arm_rows(person, "person", binding) + spm_ids = _arm_rows(spm_unit, "spm_unit", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + _require( + len(rows.array("person", "original_ids")) == len(person_ids), + "ARM_ROWS_PERSON_ALIGNMENT", + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)], + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values_, index=person_index, dtype=LEAF_DTYPE) + for name, values_ in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series( + values_, index=spm_index, dtype=LEAF_DTYPE + ) + for name, values_ in leaves.spm_unit.items() + } + ) + _new_columns(document, _leaf_coordinates()) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "contract": cps_carried_current_leaf_contract(), + "binding_sha256": _sha(values["asec_binding"].payload), + "selected_money_sha256": _sha(values["selected_current_money"].payload), + "arm_person_rows": int(len(person_ids)), + "arm_spm_unit_rows": int(len(spm_ids)), + "acs_shared_incumbents": incumbent, + "demographics": binding["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +class USComposedAsecReportedIncomeKernel(KernelBase): + """Reconstruct the six reported observations over the bound arm's rows.""" + + ref = "us.composed_population.asec_reported_income@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _reported_income_inputs(), "NODE_SLICES") + _require( + node.outputs == composed_reported_income_declarations(), "NODE_OUTPUTS" + ) + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + binding_producer = _same_producer( + context, ("asec_binding", "arm_rows", "selected_current_money") + ) + source_producer = _same_producer( + context, ("prepared_receipt", "income_observations") + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + person = context.tables["person"] + person_ids = _arm_rows(person, "person", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person[COHORT_COLUMN].dtype == np.dtype("int64"), "ARM_SOURCE_YEAR_DTYPE" + ) + result = derive_reported_income( + selected, + income, + # The arm's own original person identity, resolved and digested by + # the binding node; the composed row ids never enter the accounting. + person_ids=rows.array("person", "original_ids"), + income_years=person[COHORT_COLUMN].to_numpy(dtype="int64"), + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["asec_binding"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=binding_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype=LEAF_DTYPE + ) + for name in RESULT_COLUMNS + } + _new_columns(document, tuple(("person", name) for name in RESULT_COLUMNS)) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "accounting_sha256": _sha(payload), + "binding_sha256": _sha(values["asec_binding"].payload), + "arm_person_rows": int(len(person_ids)), + "release_eligible": False, + "certified": False, + }, + ) + + +def _leaves_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (*CPS_CARRIED_CURRENT_ROUTING_COLUMNS, arm_row_column("person")), + rows=arm_row_column("person"), + ), + Slice( + "spm_unit", + (arm_row_column("spm_unit"),), + rows=arm_row_column("spm_unit"), + ), + ) + + +def _reported_income_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (COHORT_COLUMN, arm_row_column("person")), + rows=arm_row_column("person"), + ), + ) + + +def composed_asec_measure_nodes(*, population: str, population_context: str): + """Bind the arm, then the corrected leaves, then the reported observations.""" + bind = composed_asec_bind_node( + population=population, population_context=population_context + ) + binding_artifacts = ( + ArtifactInput( + "asec_binding", BIND_NODE, "asec_binding", US_COMPOSED_ASEC_BINDING_TYPE + ), + ArtifactInput( + "arm_rows", BIND_NODE, "arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE + ), + ArtifactInput( + "selected_current_money", + BIND_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + ) + prepared_receipt = ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USComposedAsecLeavesKernel.ref, + population=population, + inputs=_leaves_inputs(), + outputs=composed_leaf_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", BIND_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USComposedAsecReportedIncomeKernel.ref, + population=population, + inputs=_reported_income_inputs(), + outputs=composed_reported_income_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return (bind, leaves, reported) + + +def composed_asec_population_graph( + columns, + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """The composed development population plus this arm's bound measurements.""" + composed = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph( + country="us", + sources=COMPOSED_SOURCES, + nodes=( + *composed, + *composed_asec_measure_nodes( + population=HARMONIZE_NODE, population_context=HARMONIZE_NODE + ), + ), + ) + geography_nodes = us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *composed, + *geography_nodes, + *composed_asec_measure_nodes( + population=f"{GEOGRAPHY_PHASE}.boundary", + population_context=GEOGRAPHY_PHASE, + ), + ), + ) + + +def composed_asec_population_registry(*, geography: bool = True) -> KernelRegistry: + """The composed registry plus exactly this stage's three kernels.""" + registry = composed_population_registry(geography=geography) + registry.register(USComposedAsecBindKernel()) + registry.register(USComposedAsecLeavesKernel()) + registry.register(USComposedAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ACS_SHARED_LEAVES", + "LEAF_DTYPE", + "LEAVES_NODE", + "REPORTED_INCOME_NODE", + "USComposedAsecLeavesKernel", + "USComposedAsecReportedIncomeKernel", + "composed_asec_measure_nodes", + "composed_asec_population_graph", + "composed_asec_population_registry", + "composed_leaf_declarations", + "composed_reported_income_declarations", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py new file mode 100644 index 000000000..2f49f6e89 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py @@ -0,0 +1,16 @@ +"""Source-free declarations shared by composed producers and their consumers. + +These are the existing legacy producer IDs and artifact types. Importing their +names must not select legacy source assembly, evaluation, or country resources. +The defining producers re-export the same objects for compatibility; operations +and source authority remain in those producers. +""" + +from microcosm.graph import ArtifactType + +CREATE_NODE = "composed_population.prepare" +BIND_NODE = "composed_population.asec_bind" +LEAVES_NODE = "composed_population.asec_cps_carried_current" +REPORTED_INCOME_NODE = "composed_population.asec_reported_income" +US_COMPOSED_ASEC_BINDING_TYPE = ArtifactType("microcosm.us.composed_asec_binding", 1) +US_COMPOSED_ASEC_ARM_ROWS_TYPE = ArtifactType("microcosm.us.composed_asec_arm_rows", 1) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py new file mode 100644 index 000000000..b07496f8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py @@ -0,0 +1,751 @@ +"""Compose the prepared ASEC population with the native ACS population. + +This is a separately versioned seam. The reviewed ``asec_raw_stage`` assembly +kernel in :mod:`.graph_sources` keeps its kernel ref, contract, params, sources +and node ids unchanged; nothing here rewrites them. What changes is only which +authenticated ASEC source the ASEC arm reads: the reviewed prepared +current-money directory (``us-asec-prepared-current-money-v3``) instead of the +raw-stage checkpoint. + +Node *keys* are a different matter and do move. Every stage manifest binds the +inventory and the bytes of ``graph_implementation.py``, which is in every +stage's module list, and declaring a new stage necessarily edits both. So every +existing US node key changes and every existing US cache is invalidated — +exactly as when any other attested module changes. + +Why this is a two-source ``CREATE`` and not two parent populations +------------------------------------------------------------------ +``Node.base`` is a single population version. A ``CREATE`` node may declare no +base at all, a structural node may not declare a second ``population``, and a +tuple ``base`` is refused as "not a structural node". So the executor cannot +structurally combine two parent populations; an ``EXPAND`` over one arm can +only invent entrant rows, never bind the other arm's rows or lineage. The +composed population is therefore one ``CREATE`` over two declared sources — +exactly the shape :class:`~.graph_sources.USAssemblyCreateKernel` already uses. +The refusals are recorded in the review root's two-parent probe. + +The stack itself is not reimplemented. Sampling, stacking, provenance and +allocation stay in :mod:`.stacked_spine` / :mod:`.spine_assembly`, and the +harmonization node reuses :class:`~.graph_sources.USSpineHarmonizeKernel` +unchanged by emitting the same two typed producer artifacts it declares. + +Nothing here is a release, a calibration, a certified population, a transfer or +a score. It is an engineering intermediate whose receipts say so. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import ( + CANONICAL_STRING_DTYPE, + canonicalize_frame_string_dtypes, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .graph_asec_income import US_ASEC_INCOME_OBSERVATIONS_TYPE +from .graph_asec_prepared import ( + ASEC_PREPARED_CODEC, + ASEC_PREPARED_SOURCE_NAME, + PreparedGraphError, +) +from .graph_composed_contracts import CREATE_NODE as CREATE_NODE +from .graph_context import US_FRAME_CONTEXT_TYPE, _json_data, encode_us_frame_context +from .graph_geography import ( + GEOGRAPHY_PHASE, + LOOKUP_SOURCES, + register_us_geography_kernels, + us_geography_nodes, +) +from .graph_housing_universe import US_ASEC_HOUSING_UNIVERSE_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import ( + ACS_CODEC, + ASSEMBLY_PHASE, + US_STACK_PREPARATION_TYPE, + USSpineHarmonizeKernel, + _canonical_assembly, + frame_column_declarations, + load_graph_acs, + us_source_codecs, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL, prepare_stacked_spine +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_STAGE = "composed_population_v1" +COMPOSED_PHASE = "compose_prepared_asec_native_acs" +ACS_NATIVE_SOURCE_NAME = "acs_native" +COMPOSED_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_STAGE] + +PREFIX = "composed_population" +HARMONIZE_NODE = PREFIX + +US_COMPOSED_SOURCE_ORIGIN_TYPE = ArtifactType( + "microcosm.us.composed_population_source_origin", 1 +) + +COMPOSED_SOURCES = ( + SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three " + "housing cohort HDFs and three official Census PERSON CSV members.", + ), + SourceRef( + ACS_NATIVE_SOURCE_NAME, + ACS_CODEC, + "2024 ACS one-year national PUMS archives.", + ), +) + +#: Native source-record identity carried by each arm, on both entities. These +#: are the columns a later node must be able to bind back to an actual archive +#: member, so the origin artifact digests them in addition to — never instead of +#: — the arm's own pre-remap IDs. Every declared column must be present: a +#: silently skipped one would leave an arm unbindable while still receipting. +NATIVE_IDENTITY_COLUMNS = { + BASE_ASEC_SUPPORT_CHANNEL: ( + ("household", "asec_H_SEQ"), + ("person", "PERIDNUM"), + ("person", "PH_SEQ"), + ), + ACS_STACKED_SUPPORT_CHANNEL: ( + ("household", "SERIALNO"), + ("person", "source_household_id"), + ("person", "source_person_id"), + ), +} +#: The per-arm cohort/vintage axis. ASEC restores three income years into one +#: prepared population, so a single scalar vintage would misdescribe it. +COHORT_COLUMN = "source_year" + +_SOURCE_ORIGIN_SCHEMA = "microcosm.us.composed-source-origin.v1" + + +@dataclass(frozen=True) +class ComposedPopulationResult: + """A direct-call composition and everything its CREATE kernel emits.""" + + frame: Frame + preparation: object + dtype_transitions: tuple[dict[str, str], ...] + storage_bridge: tuple[dict[str, str], ...] + source_origin: dict[str, object] + asec_frame_context: bytes + money_payload: bytes + receipt_payload: bytes + housing_universe_payload: bytes + income_observations_payload: bytes + prepared_receipt: Mapping[str, object] + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def bridge_prepared_string_storage(frame: Frame) -> tuple[dict[str, str], ...]: + """Put the prepared arm on the pre-assembly string storage, in place. + + The one explicit schema difference between the two arms. The prepared + source hands back graph column storage (``string[python, pd.NA]``) because + its own CREATE writes graph cells directly; every pre-assembly build + boundary, and therefore the native ACS arm, uses ``CANONICAL_STRING_DTYPE`` + (``string[python, nan]``). ``_stack_spine_tables`` requires shared columns + to have identical dtypes, so one arm has to move. Moving the prepared arm + down leaves the ACS arm byte-identical to today's production assembly, and + :func:`~.graph_sources._canonical_assembly` promotes the assembled frame + back to graph storage afterwards under its own null/value equality checks. + + This is a physical storage bridge only: no value, no missingness and no + column is added, dropped or filled. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + dtype = table[column].dtype + if isinstance(dtype, pd.StringDtype) and dtype != CANONICAL_STRING_DTYPE: + transitions.append( + { + "entity": entity, + "column": column, + "from": str(dtype), + "to": str(CANONICAL_STRING_DTYPE), + } + ) + before = { + (item["entity"], item["column"]): frame.table(item["entity"])[item["column"]] + .isna() + .to_numpy(copy=True) + for item in transitions + } + canonicalize_frame_string_dtypes( + frame, boundary="US composed population source bridge", in_place=True + ) + for (entity, column), missing in before.items(): + actual = frame.table(entity)[column] + _require( + actual.dtype == CANONICAL_STRING_DTYPE + and np.array_equal(actual.isna().to_numpy(), missing), + f"STRING_BRIDGE_MISSINGNESS:{entity}.{column}", + ) + return tuple(transitions) + + +def _arm_origin(source: Frame, channel: str) -> dict[str, object]: + """Digest one arm's native identity and cohort axis before assembly.""" + identity = {} + for entity, column in NATIVE_IDENTITY_COLUMNS[channel]: + table = source.table(entity) + _require(column in table, f"ARM_NATIVE_IDENTITY:{channel}.{entity}.{column}") + series = table[column] + identity[f"{entity}.{column}"] = { + "dtype": str(series.dtype), + "rows": int(len(series)), + "missing": int(series.isna().sum()), + "ordered_sha256": _sha( + canonical_json( + [ + None if value is None or value is pd.NA else str(value) + for value in series.tolist() + ] + ) + ), + } + person = source.table("person") + _require(COHORT_COLUMN in person, f"ARM_COHORT_COLUMN:{channel}") + years = person[COHORT_COLUMN] + counts = years.value_counts(dropna=False).to_dict() + cohorts = { + ("missing" if pd.isna(year) else str(int(year))): int(count) + for year, count in counts.items() + } + cohorts = {key: cohorts[key] for key in sorted(cohorts)} + return { + "channel": channel, + "rows": {entity: int(source.n(entity)) for entity in source.entities}, + "household_mass": float(source.weights_for("household").total), + "weight_kind": source.weights_for("household").kind.value, + "native_identity": identity, + "person_cohorts": cohorts, + "columns": { + entity: sorted(source.table(entity).columns) for entity in source.entities + }, + } + + +def _source_origin( + frame: Frame, + sources: Mapping[str, Frame], + *, + sampling: Mapping[str, object], + preparation_sha256: str, + prepared_receipt: Mapping[str, object], +) -> dict[str, object]: + """Bind each output row to the arm and the arm's own pre-remap source ID. + + The mapping is read back off the assembled frame's own receipt-validated + support-provenance columns, so it records what assembly actually did rather + than restating what the caller intended. Both provenance IDs are digested: + ``*_spine_source_id`` is the arm's raw pre-remap identifier and is what + binds an output row back to its source row, while ``*_source_id`` is the + assembly-unique pre-clone identifier the clone stage will later offset. At + CREATE the latter equals the entity ID, so digesting it alone would bind + nothing. + """ + mapping: dict[str, object] = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + _require(channel_column in table, f"ORIGIN_CHANNEL_MISSING:{entity}") + _require(spine_column in table, f"ORIGIN_SPINE_SOURCE_ID_MISSING:{entity}") + _require(source_column in table, f"ORIGIN_SOURCE_ID_MISSING:{entity}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + mapping[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + asec = sources[BASE_ASEC_SUPPORT_CHANNEL] + composed_asec = { + entity: int( + ( + frame.table(entity)[support_channel_column(entity)] + == BASE_ASEC_SUPPORT_CHANNEL + ).sum() + ) + for entity in US_SCHEMA.entities + } + return { + "schema": _SOURCE_ORIGIN_SCHEMA, + "phase": COMPOSED_PHASE, + "preparation_sha256": preparation_sha256, + "sampling": { + "sample_fraction": float(sampling["sample_fraction"]), + "sample_seed": int(sampling["sample_seed"]), + "survey_samples": { + channel: { + key: sample[key] + for key in ( + "realized_household_count", + "selected_household_ids_sha256", + "incoming_household_mass", + "sampled_household_mass", + ) + } + for channel, sample in sampling["survey_samples"].items() + }, + }, + # The four carried prepared artifacts describe the whole prepared ASEC + # population, not the composed rows. They coincide only when the draw + # keeps the arm entire. Recording both counts makes the difference + # readable instead of leaving a later consumer to assume alignment. + "asec_evidence_alignment": { + "basis": "full_prepared_population", + "evidence_rows": { + entity: int(count) + for entity, count in sorted(prepared_receipt["entity_rows"].items()) + }, + "arm_rows": {entity: int(asec.n(entity)) for entity in asec.entities}, + "composed_rows": composed_asec, + "composed_is_whole_arm": all( + composed_asec[entity] == int(asec.n(entity)) + for entity in US_SCHEMA.entities + ), + }, + "arms": { + channel: _arm_origin(source, channel) + for channel, source in sorted(sources.items()) + }, + "assembled": mapping, + "release_eligible": False, + } + + +def compose_from_sources( + asec_prepared_path, + acs_path, + *, + sample_fraction: float, + sample_seed: int, +) -> ComposedPopulationResult: + """The direct-call parity oracle the CREATE kernel shares. + + Source preparation is deliberately separate from sampling: the prepared + ASEC directory and the ACS archives are read whole, and only then does + :func:`prepare_stacked_spine` draw its declared seeded sample. Reading a + fraction of a source is not what this does and would not cost a fraction + of a whole-source read. + """ + prepared = prepare_asec_current_money_population(asec_prepared_path) + asec = prepared.frame + # The carried evidence is positional over the whole prepared population. + # If the receipt and the arm ever disagreed, everything downstream that + # reads the evidence by position would be silently misaligned. + _require( + { + entity: int(count) + for entity, count in prepared.receipt["entity_rows"].items() + } + == {entity: int(asec.n(entity)) for entity in prepared.receipt["entity_rows"]}, + "PREPARED_EVIDENCE_ROWS", + ) + acs = load_graph_acs(acs_path) + storage_bridge = bridge_prepared_string_storage(asec) + # Encoded before stacking, so the identity it declares is the prepared + # arm's own ordered identity, which is what the evidence is positional in. + asec_context = encode_us_frame_context(asec) + arms = { + BASE_ASEC_SUPPORT_CHANNEL: asec, + ACS_STACKED_SUPPORT_CHANNEL: acs, + } + stacked = prepare_stacked_spine( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(stacked.frame, (asec, acs)) + preparation_payload = canonical_json(_json_data(stacked.receipt)) + return ComposedPopulationResult( + frame=stacked.frame, + preparation=stacked.receipt, + dtype_transitions=transitions, + storage_bridge=storage_bridge, + source_origin=_source_origin( + stacked.frame, + arms, + sampling=stacked.receipt["sampling"], + preparation_sha256=_sha(preparation_payload), + prepared_receipt=prepared.receipt, + ), + asec_frame_context=asec_context, + money_payload=prepared.money_payload, + receipt_payload=prepared.receipt_payload, + housing_universe_payload=prepared.housing_universe_payload, + income_observations_payload=prepared.income_observations_payload, + prepared_receipt=prepared.receipt, + ) + + +class USComposedPopulationCreateKernel(KernelBase): + """Read both authenticated sources whole and stack them into one spine.""" + + ref = "us.composed_population.prepare@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=COMPOSED_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + # Runs before cache lookup, including resume=require, exactly as the + # raw-stage assembly kernel does: a renamed or substituted public + # loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash(COMPOSED_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require( + set(context.params) == {"phase", "sample_fraction", "sample_seed"}, + "NODE_PARAMS", + ) + _require(context.params["phase"] == COMPOSED_PHASE, "NODE_PHASE") + _require( + tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + "NODE_SOURCES", + ) + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require(node.artifact_outputs == _CREATE_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + result = compose_from_sources( + context.sources[ASEC_PREPARED_SOURCE_NAME], + context.sources[ACS_NATIVE_SOURCE_NAME], + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + receipt = result.prepared_receipt + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require( + receipt["file_roster"] == list(PREPARED_SOURCE_FILES), + "PREPARED_FILE_ROSTER", + ) + _require( + frame_column_declarations(result.frame) == node.outputs, + "COMPOSED_COLUMN_INVENTORY", + ) + return KernelResult( + frame=result.frame, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.preparation)), + "source_origin": canonical_json(result.source_origin), + "asec_frame_context": result.asec_frame_context, + "current_money": result.money_payload, + "prepared_receipt": result.receipt_payload, + "housing_universe": result.housing_universe_payload, + "income_observations": result.income_observations_payload, + }, + receipt={ + "phase": COMPOSED_PHASE, + "implementation": implementation_manifest(COMPOSED_STAGE), + "preparation": result.preparation, + "prepared": receipt, + "dtype_transitions": result.dtype_transitions, + "storage_bridge": result.storage_bridge, + "release_eligible": False, + "certified": False, + }, + ) + + +_CREATE_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ArtifactOutput("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + # The prepared arm's own typed context, encoded before stacking. The four + # evidence artifacts below are positional over that population, so a later + # binding node needs its ordered identity to align them to composed rows. + ArtifactOutput("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) + + +#: Top-level keys the origin document must carry, exactly. +_ORIGIN_KEYS = frozenset( + { + "schema", + "phase", + "preparation_sha256", + "sampling", + "asec_evidence_alignment", + "arms", + "assembled", + "release_eligible", + } +) +_ORIGIN_ENTITY_KEYS = frozenset( + { + "rows_by_channel", + "ordered_channels_sha256", + "ordered_spine_source_ids_sha256", + "ordered_source_to_output_sha256", + } +) + + +def bind_composed_source_origin(payload: bytes) -> dict[str, object]: + """Decode the origin artifact as a typed shape, refusing a malformed one. + + This checks canonical bytes and the document's declared shape: its exact + key set, its schema/phase, that both arms and every US entity are present + with well-formed digests and counts, and that it makes no release claim. It + verifies **no** digest against any population — it cannot, since it receives + only bytes. A consumer that needs the mapping to be true of a frame in hand + must recompute the digests against that frame. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "ORIGIN_CANONICAL") + _require(isinstance(document, dict), "ORIGIN_DOCUMENT") + _require(set(document) == _ORIGIN_KEYS, "ORIGIN_KEYS") + _require(document["schema"] == _SOURCE_ORIGIN_SCHEMA, "ORIGIN_SCHEMA") + _require(document["phase"] == COMPOSED_PHASE, "ORIGIN_PHASE") + _require(_is_digest(document["preparation_sha256"]), "ORIGIN_PREPARATION_DIGEST") + _require(document["release_eligible"] is False, "ORIGIN_RELEASE_CLAIM") + channels = {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + _require(set(document["arms"]) == channels, "ORIGIN_ARMS") + sampling = document["sampling"] + _require( + set(sampling) == {"sample_fraction", "sample_seed", "survey_samples"} + and set(sampling["survey_samples"]) == channels, + "ORIGIN_SAMPLING", + ) + alignment = document["asec_evidence_alignment"] + _require( + set(alignment) + == { + "basis", + "evidence_rows", + "arm_rows", + "composed_rows", + "composed_is_whole_arm", + } + and alignment["basis"] == "full_prepared_population" + and isinstance(alignment["composed_is_whole_arm"], bool), + "ORIGIN_ALIGNMENT", + ) + _require(set(document["assembled"]) == set(US_SCHEMA.entities), "ORIGIN_ENTITIES") + for entity, record in document["assembled"].items(): + _require(set(record) == _ORIGIN_ENTITY_KEYS, f"ORIGIN_ENTITY_KEYS:{entity}") + _require( + all( + _is_digest(record[key]) + for key in _ORIGIN_ENTITY_KEYS + if key.endswith("_sha256") + ), + f"ORIGIN_ENTITY_DIGEST:{entity}", + ) + _require( + set(record["rows_by_channel"]) <= channels + and all( + isinstance(count, int) and count >= 0 + for count in record["rows_by_channel"].values() + ), + f"ORIGIN_ENTITY_ROWS:{entity}", + ) + return document + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +def composed_population_nodes( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> tuple[Node, ...]: + """The two composition nodes: whole-source CREATE, then reused harmonize.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + _require( + (entity, support_channel_column(entity)) in inventory, + f"MISSING_SUPPORT_CHANNEL:{entity}", + ) + _require( + (entity, support_source_id_column(entity)) in inventory, + f"MISSING_SUPPORT_SOURCE_ID:{entity}", + ) + return ( + Node( + id=CREATE_NODE, + kernel=USComposedPopulationCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + outputs=tuple(columns), + params={ + "phase": COMPOSED_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=_CREATE_ARTIFACTS, + ), + Node( + id=HARMONIZE_NODE, + kernel=USSpineHarmonizeKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + # The reused production harmonization kernel owns this phase name; + # composing a different ASEC source does not make its operation a + # different one, and its contract is deliberately not re-versioned. + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", + CREATE_NODE, + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + ) + + +def composed_population_graph( + columns: Sequence[Owned], + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """Declare the composed development population, never a release pool.""" + nodes = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph(country="us", sources=COMPOSED_SOURCES, nodes=nodes) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *nodes, + *us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ), + ), + ) + + +def composed_population_registry(*, geography: bool = True) -> KernelRegistry: + """Register exactly the kernels this graph runs; two of three are reused.""" + registry = KernelRegistry() + registry.register(USComposedPopulationCreateKernel()) + registry.register(USSpineHarmonizeKernel()) + if geography: + register_us_geography_kernels(registry) + return registry + + +__all__ = [ + "ACS_NATIVE_SOURCE_NAME", + "COHORT_COLUMN", + "COMPOSED_DEPENDENCIES", + "COMPOSED_PHASE", + "COMPOSED_SOURCES", + "COMPOSED_STAGE", + "CREATE_NODE", + "GEOGRAPHY_PHASE", + "HARMONIZE_NODE", + "NATIVE_IDENTITY_COLUMNS", + "US_COMPOSED_SOURCE_ORIGIN_TYPE", + "ComposedPopulationResult", + "USComposedPopulationCreateKernel", + "bind_composed_source_origin", + "bridge_prepared_string_storage", + "compose_from_sources", + "composed_population_graph", + "composed_population_nodes", + "composed_population_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py new file mode 100644 index 000000000..c1732455a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py @@ -0,0 +1,278 @@ +"""Identity-bound metadata for real US operators in the population graph. + +Population cells, weights and strata remain executor-owned. This typed artifact +carries the metadata and prior mass records that existing US operators require, +so they cannot silently reload context from an unrelated mutable checkpoint. +Reconstruction uses only the node's declared table views; it grants no access to +undeclared columns or a live population object. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord +from microcosm.graph import ArtifactType, KernelContext +from microcosm.graph.canonical import canonical_json + +US_FRAME_CONTEXT_TYPE = ArtifactType("microcosm.us.frame_context", 1) + +_FIELDS = {"schema_version", "entities", "metadata", "mass_log", "weight_sources"} +_ENTITY_FIELDS = {"columns", "rows", "id_dtype", "ordered_ids_sha256"} +_MASS_FIELDS = {"entity", "old_total", "new_total", "declared_factor", "reason"} + +# These five receipts are read as authority by the existing US operators. +# Operational observations belong in run receipts, outside this keyed edge. +# An additional normative receipt requires an explicit contract revision here. +US_NORMATIVE_METADATA_KEYS = frozenset( + { + "us_spine_assembly_manifest", + "us_stacked_spine_manifest", + "us_puf_clone_attachment_manifest", + "us_late_producer_transition_authority", + "acs_pums_earnings_universe_application", + } +) +_RUN_FIELDS = frozenset( + { + "created_at", + "updated_at", + "timestamp", + "started_at", + "finished_at", + "elapsed_seconds", + "wall_time_seconds", + "duration_seconds", + "peak_rss", + "hostname", + "host", + "pid", + "run_id", + "path", + "checkpoint_dir", + "output_dir", + "timings", + "cache_hit", + "cache_hits", + } +) + + +def _normative_metadata(metadata: Mapping[str, object]) -> object: + unknown = set(metadata) - US_NORMATIVE_METADATA_KEYS + if unknown: + raise ValueError( + f"US graph context has undeclared metadata: {sorted(unknown)}." + ) + + def validate(value: object) -> None: + if isinstance(value, Mapping): + if set(value) & _RUN_FIELDS: + raise ValueError("US graph context cannot contain run-level metadata.") + for item in value.values(): + validate(item) + elif isinstance(value, list | tuple): + for item in value: + validate(item) + + validate(metadata) + return _json_data(metadata) + + +def _json_data(value: object) -> object: + """Normalize NumPy metadata without coercing unknown objects to strings.""" + if isinstance(value, np.generic): + return _json_data(value.item()) + if isinstance(value, np.ndarray): + return _json_data(value.tolist()) + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise ValueError("US graph context metadata requires string mapping keys.") + return {key: _json_data(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_json_data(item) for item in value] + # canonical_json validates scalar types and refuses nonfinite numbers. + return value + + +def _row_identity(table: pd.DataFrame, entity: str) -> dict[str, object]: + column = US_SCHEMA.entity_id_column(entity) + if column not in table: + raise ValueError(f"US graph context {entity} identity column is missing.") + ids = table[column] + if not pd.api.types.is_integer_dtype(ids.dtype) or ids.isna().any(): + raise ValueError(f"US graph context {entity} identity must be integral.") + if len(ids) and (int(ids.min()) < 0 or int(ids.max()) > np.iinfo(np.int64).max): + raise ValueError(f"US graph context {entity} identity exceeds int64 bounds.") + values = ids.to_numpy(dtype=" bytes: + """Encode current US metadata and row identities, without population cells.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph context requires the US entity schema.") + entities = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + if entity in US_SCHEMA.group_entities and ( + not isinstance(table.index, pd.RangeIndex) + or table.index.start != 0 + or table.index.stop != len(table) + or table.index.step != 1 + or table.index.name is not None + ): + # Version 1 carries group IDs, not pandas index descriptors. An + # identity FILTER uses Frame.select, which resets group indices. + # Refuse an unsupported representation before emitting authority + # that downstream person-only nodes cannot independently inspect. + raise ValueError( + f"US graph context {entity} requires a default unnamed RangeIndex." + ) + if any(not isinstance(column, str) for column in table.columns): + raise ValueError("US graph context column names must be strings.") + entities[entity] = { + "columns": list(table.columns), + **_row_identity(table, entity), + } + document = { + "schema_version": US_FRAME_CONTEXT_TYPE.schema_version, + "entities": entities, + "metadata": _normative_metadata(frame.metadata), + "mass_log": [_json_data(asdict(record)) for record in frame.mass_log], + "weight_sources": { + entity: frame.weights_for(entity).kind.value + for entity in frame.weighted_entities + }, + } + return canonical_json(document) + + +def _decode(payload: bytes) -> dict[str, object]: + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate US graph context field {key!r}.") + result[key] = value + return result + + try: + document = json.loads(payload, object_pairs_hook=unique_pairs) + canonical_json(document) + except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as error: + raise ValueError( + "US graph context must be finite, unambiguous JSON." + ) from error + if ( + not isinstance(document, dict) + or set(document) != _FIELDS + or type(document["schema_version"]) is not int + or document["schema_version"] != US_FRAME_CONTEXT_TYPE.schema_version + ): + raise ValueError("US graph context has an unsupported schema.") + entities = document["entities"] + if not isinstance(entities, dict) or set(entities) != set(US_SCHEMA.entities): + raise ValueError("US graph context must describe exactly the US entities.") + for entity, value in entities.items(): + if not isinstance(value, dict) or set(value) != _ENTITY_FIELDS: + raise ValueError(f"US graph context {entity} identity is malformed.") + columns = value["columns"] + if ( + not isinstance(columns, list) + or any(not isinstance(column, str) or not column for column in columns) + or len(set(columns)) != len(columns) + or US_SCHEMA.entity_id_column(entity) not in columns + or type(value["rows"]) is not int + or value["rows"] < 0 + ): + raise ValueError(f"US graph context {entity} schema is malformed.") + if not isinstance(document["metadata"], dict): + raise ValueError("US graph context metadata must be an object.") + _normative_metadata(document["metadata"]) + weight_sources = document["weight_sources"] + if ( + not isinstance(weight_sources, dict) + or not weight_sources + or set(weight_sources) - set(US_SCHEMA.entities) + or any(not isinstance(kind, str) for kind in weight_sources.values()) + ): + raise ValueError("US graph context weight sources are malformed.") + if not isinstance(document["mass_log"], list): + raise ValueError("US graph context mass_log must be an array.") + return document + + +def _mass_records(raw: list[object]) -> tuple[MassChangeRecord, ...]: + records = [] + for value in raw: + if not isinstance(value, dict) or set(value) != _MASS_FIELDS: + raise ValueError("US graph context mass record is malformed.") + if value["entity"] not in US_SCHEMA.entities or not isinstance( + value["reason"], str + ): + raise ValueError("US graph context mass record has invalid authority.") + for field in ("old_total", "new_total", "declared_factor"): + number = value[field] + if field == "declared_factor" and number is None: + continue + if type(number) not in (int, float) or number < 0: + raise ValueError("US graph context mass record is invalid.") + records.append(MassChangeRecord(**value)) + return tuple(records) + + +def us_frame_from_context( + context: KernelContext, *, artifact_alias: str = "frame_context" +) -> Frame: + """Restore an isolated US operator view from declared cells and typed context. + + The complete row identity of each entity must match the artifact. Columns + omitted by the node stay omitted. Callers needing a narrower row population + must declare a structural graph stage and publish its own context first. + """ + artifact = context.artifacts.get(artifact_alias) + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US graph context artifact has a missing or incorrect type.") + document = _decode(artifact.payload) + if set(context.tables) != set(US_SCHEMA.entities): + raise ValueError("US graph context requires declared views for every entity.") + tables = {} + for entity in US_SCHEMA.entities: + table = context.tables[entity] + declared = document["entities"][entity] + actual = _row_identity(table, entity) + if any(declared[key] != value for key, value in actual.items()): + raise ValueError(f"US graph context {entity} identity does not match.") + if set(table.columns) - set(declared["columns"]): + raise ValueError(f"US graph context {entity} includes unknown columns.") + ordered = [column for column in declared["columns"] if column in table] + tables[entity] = table.loc[:, ordered].copy(deep=True) + weights = {} + for entity, kind in document["weight_sources"].items(): + value = context.weights.get(entity) + if value is None or value.kind.value != kind: + raise ValueError( + f"US graph context has no matching {entity} weight source." + ) + weights[entity] = value + return Frame( + tables, + US_SCHEMA, + weights, + context.strata.copy(deep=True), + mass_log=_mass_records(document["mass_log"]), + metadata=document["metadata"], + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py new file mode 100644 index 000000000..e1ce9a0f3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py @@ -0,0 +1,407 @@ +"""Current survey-first E00900 diagnostic placement, not donor admission. + +The source-qualified host is the accepted ACS+ASEC allocation and whole-household +clone. Donor qualification belongs to a separately reviewed upstream producer. +This module writes a development tax-unit diagnostic, never an engine input. +The mandatory materialized check also runs after replay, using the retained live +preparation and the real executor-observed Population. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + KernelResult, + Node, + Owned, + Slice, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import graph_puf_diagnostic_consumer as host +from . import survey_population_replay as replay + +shared, detail, codec, require = host.shared, host.detail, host.codec, host.require +SCOPE = "current_survey_e00900_development_not_tax_inputs" +PLACEMENT_TYPE = ArtifactType("microcosm.us.current_survey_puf_placement", 1) +PLACEMENT_NODE = "puf_diagnostic.current_survey_placement" +APPLY_PREFIX = "puf_diagnostic.current_survey_apply" +ATTACH_NODE = "puf_diagnostic.current_survey_attach" +SEED = 578 + + +def _inputs(frame): + values = [] + for entity in US_SCHEMA.entities: + structural = {US_SCHEMA.entity_id_column(entity)} + if entity == US_SCHEMA.person_entity: + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + values.append( + Slice(entity, tuple(c for c in frame.table(entity) if c not in structural)) + ) + return tuple(values) + + +def _edges(): + return ( + ArtifactInput( + "projection", + host.CURRENT_PROJECTION_NODE, + "projection", + host.CURRENT_PROJECTION_TYPE, + ), + ArtifactInput( + "matrix", + host.CURRENT_MATRIX_NODE, + "matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ), + ) + + +def _attach_edges(): + return ( + *_edges(), + ArtifactInput("placement", PLACEMENT_NODE, "placement", PLACEMENT_TYPE), + ArtifactInput( + "raw_draw", APPLY_PREFIX + ".000", "raw_draw", codec.RAW_TARGET_TYPE + ), + ArtifactInput( + "apply_state", APPLY_PREFIX + ".000", "apply_state", MATRIX_APPLY_STATE_TYPE + ), + ) + + +def _placement_node(frame): + require( + detail.MASK not in frame.table("tax_unit") + and detail.OUTPUT not in frame.table("tax_unit"), + "SURVEY_TRANSFER_ALREADY_PRESENT", + ) + return Node( + PLACEMENT_NODE, + CurrentSurveyPlacementKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=_inputs(frame), + outputs=(Owned("tax_unit", detail.MASK, "bool"),), + params={"scope": SCOPE}, + artifact_inputs=_edges(), + artifact_outputs=(ArtifactOutput("placement", PLACEMENT_TYPE),), + ) + + +def _attach_node(frame): + inputs = tuple( + Slice( + s.entity, (*s.columns, detail.MASK) if s.entity == "tax_unit" else s.columns + ) + for s in _inputs(frame) + ) + return Node( + ATTACH_NODE, + CurrentSurveyAttachKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=inputs, + outputs=(Owned("tax_unit", detail.OUTPUT, "float64", rows=detail.MASK),), + params={"scope": SCOPE}, + artifact_inputs=_attach_edges(), + ) + + +def _binding(projection, matrix, matrix_producer_key, frame, mask): + return codec.encode_json( + { + "scope": SCOPE, + "projection_sha256": codec.sha(projection), + "matrix_sha256": codec.sha(matrix), + "matrix_producer_key": matrix_producer_key, + "features": list(detail.FEATURES), + "targets": [detail.TARGET], + "output": detail.OUTPUT, + "selected_ids": frame.table("tax_unit") + .tax_unit_id.to_numpy()[mask] + .tolist(), + "source_admission": detail.SOURCE_ADMISSION, + "release_eligible": False, + } + ) + + +def _mask_result(frame, mask): + return KernelResult( + columns={ + ("tax_unit", detail.MASK): pd.Series( + mask, index=frame.table("tax_unit").tax_unit_id, dtype=bool + ), + } + ) + + +def _raw_values(matrix, matrix_producer_key, raw_draw, apply_state): + packet = decode_matrix_apply_state(apply_state) + require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "SURVEY_TRANSFER_MATRIX_PRODUCER", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + tuple(chain.completed_targets) == (detail.TARGET,) + and tuple(chain.predictors) == detail.FEATURES + and application["seed"] == SEED + and application["raw_targets"] + == [ + { + "target": detail.TARGET, + "sha256": codec.sha(raw_draw), + } + ], + "SURVEY_TRANSFER_RAW_STATE", + ) + prepared = model_input.decode_recipient_matrix(matrix) + require( + chain.entity == prepared.entity == "tax_unit" + and tuple(chain.targets) == (detail.TARGET,) + and chain.recipient_index + == shared.qrf._index_identity(prepared.features.index), + "SURVEY_TRANSFER_RAW_INDEX", + ) + values = codec.read_raw_target( + raw_draw, target=detail.TARGET, index=prepared.features.index + ) + require(np.isfinite(values).all(), "SURVEY_TRANSFER_NONFINITE_DRAW") + return pd.Series(values, index=prepared.entity_ids, dtype="float64") + + +class _CurrentTransferKernel(host._CurrentSurveyKernel): + def __init__(self, preparation, allocated_population, clone_population): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + + def implementation_hash(self): + # Independent extension closure; do not relabel or refresh survey inventory. + return codec.sha( + codec.encode_json( + { + "host": super().implementation_hash(), + "transfer": source_hash( + sys.modules[__name__], + population_ops, + replay, + sys.modules[decode_matrix_apply_state.__module__], + sys.modules[legacy_qrf_apply_matrix_nodes.__module__], + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _qualified(self, context): + expected_projection, expected_matrix, mask, evidence = ( + host.qualify_current_survey_host( + self.preparation, self.allocated_population, self.clone_population + ) + ) + projection = shared.artifact( + context, "projection", host.CURRENT_PROJECTION_TYPE + ) + matrix = shared.artifact(context, "matrix", model_input.RECIPIENT_MATRIX_TYPE) + require( + projection.payload == expected_projection + and matrix.payload == expected_matrix, + "SURVEY_TRANSFER_HOST_ARTIFACTS", + ) + return projection, matrix, mask, evidence + + +class CurrentSurveyPlacementKernel(_CurrentTransferKernel): + ref = "us.puf_diagnostic.current_survey_placement@1" + + def run(self, context): + require( + context.node == _placement_node(self.clone_population.frame) + and not context.sources + and set(context.artifacts) == {"projection", "matrix"}, + "SURVEY_TRANSFER_PLACEMENT_DECLARATION", + ) + projection, matrix, mask, evidence = self._qualified(context) + host._current_context_frame(context, self.clone_population.frame) + result = _mask_result(self.clone_population.frame, mask) + return KernelResult( + columns=result.columns, + artifacts={ + "placement": _binding( + projection.payload, + matrix.payload, + matrix.producer_key, + self.clone_population.frame, + mask, + ) + }, + receipt={ + **evidence, + "scope": SCOPE, + "source_admission": detail.SOURCE_ADMISSION, + }, + ) + + +class CurrentSurveyAttachKernel(_CurrentTransferKernel): + ref = "us.puf_diagnostic.current_survey_attach@1" + + def run(self, context): + require( + context.node == _attach_node(self.clone_population.frame) + and not context.sources + and set(context.artifacts) + == {"projection", "matrix", "placement", "raw_draw", "apply_state"}, + "SURVEY_TRANSFER_ATTACH_DECLARATION", + ) + projection, matrix, mask, evidence = self._qualified(context) + placement = shared.artifact(context, "placement", PLACEMENT_TYPE) + raw = shared.artifact(context, "raw_draw", codec.RAW_TARGET_TYPE) + state = shared.artifact(context, "apply_state", MATRIX_APPLY_STATE_TYPE) + shared.siblings(context, ("raw_draw", "apply_state")) + require( + placement.payload + == _binding( + projection.payload, + matrix.payload, + matrix.producer_key, + self.clone_population.frame, + mask, + ), + "SURVEY_TRANSFER_PLACEMENT_BINDING", + ) + # Comparison-only normal patch; no new source/successor owner is issued. + masked = population_ops.patch( + self.clone_population, + _placement_node(self.clone_population.frame), + _mask_result(self.clone_population.frame, mask), + ) + host._current_context_frame(context, masked.frame) + values = _raw_values( + matrix.payload, matrix.producer_key, raw.payload, state.payload + ) + return KernelResult( + columns={("tax_unit", detail.OUTPUT): values}, + receipt={ + **evidence, + "scope": SCOPE, + "source_admission": detail.SOURCE_ADMISSION, + "raw_sha256": codec.sha(raw.payload), + "native_output_cells_written": 0, + "host_weights_changed": False, + }, + ) + + +def current_survey_transfer_nodes(frame, *, fit_nodes): + """Connect the independently qualified donor fits to the accepted host matrix. + + This E00900 mechanism does not admit a donor source. A future complete PUF + chain uses the same maintained train/apply family with its full ordered + targets and separately reviewed person/tax-unit finalization contract. + """ + require( + type(fit_nodes) is tuple and len(fit_nodes) == 1, "SURVEY_TRANSFER_FIT_CHAIN" + ) + fit = fit_nodes[0] + expected = legacy_qrf_train_nodes( + "__comparison__", + population=fit.population, + entity="tax_unit", + predictors=detail.FEATURES, + targets=(detail.TARGET,), + seed=SEED, + phase=SCOPE, + n_estimators=2, + zero_atol=0, + )[0] + from dataclasses import replace + + require(fit == replace(expected, id=fit.id), "SURVEY_TRANSFER_FIT_DECLARATION") + applies = legacy_qrf_apply_matrix_nodes( + APPLY_PREFIX, + population=host.survey_clone.COMBINED_CLONE_NODE, + fit_nodes=fit_nodes, + matrix_producer=host.CURRENT_MATRIX_NODE, + seed=SEED, + phase=SCOPE, + ) + return (_placement_node(frame), *applies, _attach_node(frame)) + + +def verify_materialized_current_survey_transfer( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, + matrix_producer_key, + placement, + raw_draw, + apply_state, +): + """Mandatory after actual typed-artifact/key/store checks, cold and replay. + + Inputs are the actual loaded payloads and producer key, not receipt labels. + Last source I/O requalifies the retained host. Pure comparison then verifies + every native/clone cell, axis, metadata value, weight, owner and mass ledger. + Reconstructed expected populations are comparison values, never source owners. + """ + require(codec._hash(matrix_producer_key), "SURVEY_TRANSFER_MATRIX_KEY") + expected_projection, expected_matrix, mask, evidence = ( + host.qualify_current_survey_host( + preparation, allocated_population, clone_population + ) + ) + require( + projection == expected_projection and matrix == expected_matrix, + "SURVEY_TRANSFER_MATERIALIZED_HOST", + ) + require( + placement + == _binding( + projection, matrix, matrix_producer_key, clone_population.frame, mask + ), + "SURVEY_TRANSFER_PLACEMENT_BINDING", + ) + values = _raw_values(matrix, matrix_producer_key, raw_draw, apply_state) + expected = population_ops.patch( + clone_population, + _placement_node(clone_population.frame), + _mask_result(clone_population.frame, mask), + ) + expected = population_ops.patch( + expected, + _attach_node(clone_population.frame), + KernelResult(columns={("tax_unit", detail.OUTPUT): values}), + ) + replay.same_replayed_population(expected, population) + return {**evidence, "scope": SCOPE, "source_admission": detail.SOURCE_ADMISSION} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py new file mode 100644 index 000000000..dff2f747e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py @@ -0,0 +1,380 @@ +"""Declared lookup import, household assignment, and geography validation. + +The existing PUMA operator keeps native ACS PUMAs and uses its legacy seeded +draw order for missing PUMAs, districts and counties. Reference import and the +post-assignment gate are separate graph nodes. No target values are fitted here. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateReport +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes +from microcosm.graph.kernel import KernelRole +from microcosm.graph.population import dtype_for_token + +from .congressional_district_geography import CONGRESSIONAL_DISTRICT_GEOID_COLUMN +from .congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + decode_congressional_district_vintage_crosswalk, + validate_packaged_congressional_district_vintage_crosswalk, +) +from .graph_context import ( + US_FRAME_CONTEXT_TYPE, + _decode, + _mass_records, + _row_identity, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .puma_ladder import ( + US_PUMA_LADDER_COLUMNS, + assign_us_puma_ladder, + decode_us_puma_ladder, + us_puma_ladder_assignment_summary, + us_puma_ladder_gate, +) + +US_PUMA_LOOKUP_TYPE = ArtifactType("microcosm.us.puma_ladder_npz", 1) +US_CD_CROSSWALK_TYPE = ArtifactType("microcosm.us.cd_crosswalk_csv", 1) +GEOGRAPHY_PHASE = "assign_us_puma_ladder" +LOOKUP_SOURCES = ( + SourceRef( + "us_puma_ladder_2020", + "raw-bytes-v1", + "Census population-weighted PUMA overlaps.", + ), + SourceRef( + "us_cd_crosswalk_117_119", + "raw-bytes-v1", + "Pinned 117th-to-119th population crosswalk.", + ), +) + + +def _lookups(ladder_bytes: bytes, crosswalk_bytes: bytes): + ladder = decode_us_puma_ladder(ladder_bytes) + crosswalk = decode_congressional_district_vintage_crosswalk(crosswalk_bytes) + validate_packaged_congressional_district_vintage_crosswalk(crosswalk) + expected = tuple( + sorted( + int(value.removeprefix(CURRENT_CONGRESSIONAL_DISTRICT_PREFIX)) + for value in crosswalk["target_geography_id"].unique() + ) + ) + actual = tuple(sorted(int(value) for value in np.unique(ladder.cd_overlap_cd))) + if actual != expected: + raise ValueError( + "PUMA ladder and national crosswalk district universes differ." + ) + if ( + ladder.layer_vintages["congressional_district"] + != CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ): + raise ValueError("PUMA ladder has the wrong congressional district vintage.") + return ladder, expected + + +class _USGeographyKernel(KernelBase): + def implementation_hash(self) -> str: + return implementation_hash("geography") + + +class USGeographyLookupKernel(_USGeographyKernel): + ref = "us.production.geography_lookup@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("Unsupported US geography lookup parameters.") + ladder_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_puma_ladder_2020"] + ) + crosswalk_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_cd_crosswalk_117_119"] + ) + ladder, expected = _lookups(ladder_bytes, crosswalk_bytes) + return KernelResult( + artifacts={"ladder": ladder_bytes, "crosswalk": crosswalk_bytes}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "district_count": len(expected), + "puma_count": len(ladder), + "ladder_sha256": hashlib.sha256(ladder_bytes).hexdigest(), + "crosswalk_sha256": hashlib.sha256(crosswalk_bytes).hexdigest(), + }, + ) + + +class USGeographyBoundaryKernel(_USGeographyKernel): + """Preserve all rows while opening the geography rewrite version.""" + + ref = "us.production.geography_boundary@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography boundary requires its registered phase.") + person = context.tables["person"] + return KernelResult( + receipt={"implementation": implementation_manifest("geography")}, + keep=pd.Series( + True, index=pd.Index(person["person_id"], name="person_id"), dtype=bool + ), + ) + + +class USGeographyAssignKernel(_USGeographyKernel): + ref = "us.production.geography_assign@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.PARAM, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != { + "phase": GEOGRAPHY_PHASE, + "seed": 0, + "assign_tract": False, + }: + raise ValueError( + "US geography requires the declared legacy seed and county/CD contract." + ) + artifact = context.artifacts.get("frame_context") + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US geography requires a typed frame context.") + document = _decode(artifact.payload) + _mass_records(document["mass_log"]) + household = context.tables["household"] + declared = document["entities"]["household"] + if any( + declared[key] != value + for key, value in _row_identity(household, "household").items() + ): + raise ValueError("US geography household identity does not match context.") + if set(household.columns) - set(declared["columns"]): + raise ValueError("US geography has undeclared household input columns.") + if ( + document["weight_sources"].get("household") + != context.weights["household"].kind.value + ): + raise ValueError( + "US geography requires the matching household weight source." + ) + ladder, expected = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + table = assign_us_puma_ladder( + household, + ladder, + seed=0, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) + columns = {} + for owned in context.node.outputs: + if ( + owned.entity != "household" + or owned.column not in US_PUMA_LADDER_COLUMNS + ): + raise ValueError("US geography has an unsupported owned output.") + original = table[owned.column] + converted = original.astype(dtype_for_token(owned.dtype)) + if not original.isna().equals(converted.isna()) or not np.array_equal( + original.to_numpy(), converted.to_numpy() + ): + raise ValueError( + "US geography storage conversion changed values or missingness." + ) + table[owned.column] = converted + columns[(owned.entity, owned.column)] = pd.Series( + converted.array, + index=pd.Index(table["household_id"], name="household_id"), + ) + if set(column for _, column in columns) != set(US_PUMA_LADDER_COLUMNS): + raise ValueError( + "US geography must own exactly its three household outputs." + ) + # This operator changes only these three cells on each household. The + # graph boundary preserves every row, weight and membership. Carry the + # authenticated descriptors for untouched entities without copying + # their population tables into this household-only operator. + for column in US_PUMA_LADDER_COLUMNS: + if column not in declared["columns"]: + declared["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "seed": 0, + "assign_tract": False, + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "target_district_count": len(expected), + "summary": us_puma_ladder_assignment_summary( + table, + ladder, + weight_values=context.weights["household"].values, + ), + }, + ) + + +class USGeographyGateKernel(_USGeographyKernel): + ref = "us.production.geography_gate@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography gate requires its registered phase.") + gate = us_puma_ladder_gate( + context.tables["household"], + context.weights["household"].values, + assign_tract=False, + ) + return KernelResult( + receipt={ + "outcome": "pass" if gate.passed else "fail", + "evidence": GateReport((gate,)).to_manifest(), + "scope": "geography_assignment_integrity", + } + ) + + +def us_geography_nodes( + columns: Sequence[Owned], *, base: str, context_producer: str +) -> tuple[Node, ...]: + """Append a declared production geography stage after source assembly. + + This checks assignment integrity, not CD calibration or release readiness. + """ + inventory = {(owned.entity, owned.column): owned for owned in columns} + if len(inventory) != len(columns): + raise ValueError("US geography input column declarations repeat coordinates.") + if ("person", "age") not in inventory: + raise ValueError("US geography requires the assembled person age column.") + if ("household", "state_fips") not in inventory: + raise ValueError("US geography requires household state_fips.") + input_columns = ("state_fips",) + ( + ("puma",) if ("household", "puma") in inventory else () + ) + boundary = f"{GEOGRAPHY_PHASE}.boundary" + lookup = f"{GEOGRAPHY_PHASE}.lookup" + output_types = { + "puma": "string", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN: "int64", + "county_fips": "string", + } + outputs = [] + for column, dtype in output_types.items(): + incumbent = inventory.get(("household", column)) + if incumbent: + if incumbent.dtype not in ( + {"int64", "Int64"} if dtype == "int64" else {"string"} + ): + raise ValueError(f"Unsupported incumbent geography dtype: {column}.") + dtype = incumbent.dtype + outputs.append(Owned("household", column, dtype, rewrite=incumbent is not None)) + return ( + Node( + id=lookup, + kernel=USGeographyLookupKernel.ref, + population=base, + sources=tuple(source.name for source in LOOKUP_SOURCES), + params={"phase": GEOGRAPHY_PHASE}, + artifact_outputs=( + ArtifactOutput("ladder", US_PUMA_LOOKUP_TYPE), + ArtifactOutput("crosswalk", US_CD_CROSSWALK_TYPE), + ), + ), + Node( + id=boundary, + kernel=USGeographyBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + params={"phase": GEOGRAPHY_PHASE}, + ), + Node( + id=GEOGRAPHY_PHASE, + kernel=USGeographyAssignKernel.ref, + population=boundary, + inputs=(Slice("household", input_columns),), + outputs=tuple(outputs), + params={"phase": GEOGRAPHY_PHASE, "seed": 0, "assign_tract": False}, + artifact_inputs=( + ArtifactInput( + "frame_context", + context_producer, + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + Node( + id=f"{GEOGRAPHY_PHASE}.gate", + kernel=USGeographyGateKernel.ref, + population=boundary, + inputs=(Slice("household", ("state_fips", *US_PUMA_LADDER_COLUMNS)),), + params={"phase": GEOGRAPHY_PHASE}, + ), + ) + + +def register_us_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + USGeographyLookupKernel(), + USGeographyBoundaryKernel(), + USGeographyAssignKernel(), + USGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py new file mode 100644 index 000000000..c8bbdf3c0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py @@ -0,0 +1,230 @@ +"""Non-authoritative HU transport bound to CREATE and selected graph ancestry. + +Parsing these bytes never issues AuthenticatedHousingUniverse or Frame authority. +The graph supplies the producer and preparation-receipt custody; this module +checks its native bytes and the exact int64 promotion of selected carried cells. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ArtifactType + +from . import asec_housing_universe as hu +from . import asec_housing_universe_source as source + +US_ASEC_HOUSING_UNIVERSE_TYPE = ArtifactType( + "microcosm.us.asec_bound_housing_universe", 1 +) +HOUSEHOLD_EVIDENCE_COLUMNS = ( + *("asec_" + name for name in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *source.ATTACHED_COLUMNS, +) +_TOKEN = object() +_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "source_authentication", + "release_eligible", + "source_period_kind", + "zero_origin_policy", + "zero_origin_evidence", + "household_rows", + "columns", + "codebook", + "dictionaries", + "definition_sha256", + "source", + "implementation", + "limitations", + } +) + + +ACCEPTED_PREPARED_RECEIPT_SCHEMAS = ("microcosm.us.asec_prepared_receipt.v3",) +ACCEPTED_PREPARED_SOURCE_KINDS = ("us_asec_prepared_current_money_v3",) + + +@dataclass(frozen=True) +class BoundHousingUniverse: + """Immutable native buffers with graph bindings, no full-source authority.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + hu._require(_token is _TOKEN, "BOUND_HU_CONSTRUCTOR") + + def array(self, name: str) -> np.ndarray: + hu._require(name in hu.COLUMNS, "BOUND_HU_COLUMN") + return np.frombuffer( + self.buffers[hu.COLUMNS.index(name)], + dtype=" BoundHousingUniverse: + """Validate transport against a graph-provided receipt, not a source decoder.""" + try: + hu._require( + type(payload) is bytes + and len(hu.MAGIC) + 4 + 32 < len(payload) <= hu.PAYLOAD_MAX_BYTES, + "BOUND_HU_SIZE", + ) + binding = prepared_receipt["housing_universe"] + hu._require( + prepared_receipt["schema"] in ACCEPTED_PREPARED_RECEIPT_SCHEMAS + and prepared_receipt["source_kind"] in ACCEPTED_PREPARED_SOURCE_KINDS + and prepared_receipt["release_eligible"] is False + and binding["parent_kind"] == "HousingStatusAttachedAsec" + and binding["source_period_kind"] == "interview_household_universe" + and binding["native_dtype"] == "uint8" + and binding["graph_dtype"] == "int64" + and binding["aliases"] == list(source.ATTACHED_COLUMNS), + "BOUND_HU_PREPARATION_SCHEMA", + ) + hu._require(hu._sha(payload) == binding["payload_sha256"], "BOUND_HU_PAYLOAD") + hu._require(payload.startswith(hu.MAGIC), "BOUND_HU_MAGIC") + hu._require(hu._sha(payload[:-32]) == payload[-32:].hex(), "BOUND_HU_CHECKSUM") + size = struct.unpack_from(" None: + """Compare graph int64 rows with native evidence; no cast or authority upgrade.""" + hu._require(type(body) is BoundHousingUniverse, "BOUND_HU_TYPE") + hu._require( + type(positions) is np.ndarray + and positions.dtype == np.dtype("int64") + and positions.shape == (len(household),) + and bool( + ((positions >= 0) & (positions < len(body.array("household_id")))).all() + ) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())), + "BOUND_HU_POSITIONS", + ) + hu._require(household.columns.is_unique, "BOUND_HU_TABLE_COLUMNS") + for name, column in ( + ("household_id", "household_id"), + *((n, "asec_" + n) for n in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *zip(hu.DERIVED_COLUMNS, source.ATTACHED_COLUMNS, strict=True), + ): + hu._require(column in household, "BOUND_HU_CARRIED_COLUMN") + values = household[column] + hu._require( + values.dtype == np.dtype("int64") + and np.array_equal(values.to_numpy(), body.array(name)[positions]), + "BOUND_HU_CARRIED_VALUES", + ) + if income_years is not None: + hu._require( + type(income_years) is np.ndarray + and income_years.dtype == np.dtype("int64") + and np.array_equal(income_years, body.array("income_year")[positions]), + "BOUND_HU_COHORTS", + ) + + +__all__ = [ + "BoundHousingUniverse", + "HOUSEHOLD_EVIDENCE_COLUMNS", + "US_ASEC_HOUSING_UNIVERSE_TYPE", + "bind_housing_universe", + "verify_graph_housing_rows", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py new file mode 100644 index 000000000..e715b47de --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py @@ -0,0 +1,532 @@ +"""Reviewed, path-independent implementation identities for the US source graph. + +Whole modules remain the unit of code identity. The packaged inventory is a +reviewed dependency fence, not a Python sandbox or inferred call-graph proof. +Its import/use/resource expressions prevent silently extending a reviewed route +to an unbound helper. Updating that inventory requires a scope review. +""" + +from __future__ import annotations + +import ast +import csv +import hashlib +import importlib.metadata as importlib_metadata +import importlib.util +import inspect +import json +import sys +from collections.abc import Mapping +from functools import lru_cache +from pathlib import Path +from types import MappingProxyType + +import pandas as pd + +STAGE_DEPENDENCIES = MappingProxyType( + { + "asec_codec_v4": ("numpy", "pandas", "h5py"), + "acs_codec_2024": ("numpy", "pandas", "microunit", "PyYAML", "pyarrow"), + "acs_housing_universe_2024": ( + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow", + ), + # Full authenticated source catalogues, one household draw, selected + # native construction and one allocation. This deliberately excludes + # legacy prepared-source income observations and engine resources. + "authenticated_survey_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_prepare": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_harmonize": ("numpy", "pandas"), + "geography": ("numpy", "pandas"), + # The ASEC prepared current-money slice. PolicyEngine-US is deliberately + # absent: its version is a normative parameter of the evaluation node and + # a pinned field of every admitted closure contract, so engine drift + # refuses there. Putting it here instead would make a secrets-free lane + # without the engine extra unable to hash any stage of this graph. + "asec_prepared_v3": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # The composed prepared-ASEC/native-ACS population. It runs both source + # scopes in one CREATE, so its module inventory is their union and its + # dependency set is theirs (identical in both). PolicyEngine-US is + # absent for the same reason it is absent from the prepared scope: no + # engine node lives in this graph. + "composed_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # Binding the carried prepared-ASEC evidence to the composed ASEC rows + # and deriving the reviewed corrected leaves and reported observations + # over them. It runs no source loader of its own, but its resolution and + # every verifier it reuses live inside the composing scope, so its module + # inventory is that scope's plus the two binding modules and its + # dependency set is identical. + "composed_asec_binding_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + } +) +_CODECS = MappingProxyType( + { + "us-asec-raw-stage-v4": ( + "asec_codec_v4", + "microcosm.build.us_runtime.graph_sources:load_graph_asec", + ), + "us-acs-native-2024-v1": ( + "acs_codec_2024", + "microcosm.build.us_runtime.graph_sources:load_graph_acs", + ), + "us-asec-prepared-current-money-v3": ( + "asec_prepared_v3", + "microcosm.build.us_runtime.asec_prepared_source:load_graph_asec_prepared", + ), + "raw-bytes-v1": ("geography", "microcosm.graph.codecs:load_raw_bytes"), + "us-acs-housing-universe-2024-v1": ( + "acs_housing_universe_2024", + "microcosm.build.us_runtime.acs_housing_universe_source:load_graph_acs_housing_universe", + ), + } +) +_RESOURCE_CALLS = frozenset( + { + "open", + "read_bytes", + "read_text", + "read_csv", + "read_parquet", + "read_hdf", + "read_excel", + "read_json", + "read_table", + "read_feather", + "File", + "loadtxt", + "genfromtxt", + "load", + "safe_load", + "files", + "joinpath", + "import_module", + "__import__", + "exec", + "eval", + } +) + + +# These local lazy entry points are dependencies even though no ImportFrom +# binds their names. Record references AND complete calls in their caller scope: +# passing the dispatcher around must not hide its use, and changing a selected +# symbol must require review. This adds tripwires; it grants no scope exemption. +_LOCAL_IMPORT_DISPATCHERS = { + "microcosm.build/us_runtime/graph_sources.py": frozenset( + { + "_stacked_alias", + "__getattr__", + "assemble_stacked_spine", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + } + ), +} + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _digest(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _package_roots() -> dict[str, Path]: + # Resolve packages, never import leaf modules (including optional adapters) + # merely to hash their files. Editable and wheel installs use the same names. + roots = {} + for package in ( + "microcosm.build", + "microcosm.frame", + "microcosm.graph", + "microunit", + ): + spec = importlib.util.find_spec(package) + if spec is None and package == "microunit": + continue # Only the ACS/CREATE scopes require this dependency. + locations = () if spec is None else spec.submodule_search_locations or () + if len(locations) != 1: + raise ValueError( + f"US implementation package inventory unavailable: {package}." + ) + roots[package] = Path(next(iter(locations))).resolve() + return roots + + +@lru_cache(maxsize=256) +def _dependency_details(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + """Static tripwire cached by exact bytes, never by path, mtime or version.""" + tree = ast.parse(payload) + imports, aliases, unbound_imports = set(), {}, set() + package, relative = name.split("/", 1) + module = package + "." + relative.removesuffix(".py").replace("/", ".") + parent = module.rpartition(".")[0] + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name == "*": + raise ValueError("Unclassified wildcard dependency import.") + target = ( + alias.name + if isinstance(node, ast.Import) + else ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), parent + ) + if node.level + else node.module + ) + ) + is_stdlib = target.split(".")[0] in sys.stdlib_module_names + if not is_stdlib: + imports.add(target) + is_bound = target in covered or any( + target == item[1:] or target.startswith(item[1:] + ".") + for item in covered + if item.startswith("*") + ) + if not is_stdlib and not is_bound: + aliases[alias.asname or alias.name.split(".")[0]] = target + unbound_imports.add(f"import:{target}:{alias.name}") + resources, uses = set(), set(unbound_imports) + dispatchers = _LOCAL_IMPORT_DISPATCHERS.get(name, ()) + + class Visitor(ast.NodeVisitor): + scope = "" + + def visit_FunctionDef(self, node): + previous = self.scope + self.scope = f"{previous}.{node.name}" + self.generic_visit(node) + self.scope = previous + + visit_AsyncFunctionDef = visit_FunctionDef # noqa: N815 - AST visitor API + visit_ClassDef = visit_FunctionDef # noqa: N815 - AST visitor API + + def visit_Name(self, node): + if isinstance(node.ctx, ast.Load) and node.id in dispatchers: + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + if isinstance(node.ctx, ast.Load) and node.id in aliases: + uses.add(f"{self.scope}:{aliases[node.id]}:{node.id}") + + def visit_Call(self, node): + name = ( + node.func.attr + if isinstance(node.func, ast.Attribute) + else getattr(node.func, "id", None) + ) + # Bind reflective selection too, including the dispatcher's + # getattr(import_module(...), name) target expression. + if ( + name in _RESOURCE_CALLS + or name in dispatchers + or (dispatchers and name == "getattr") + ): + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + self.generic_visit(node) + + Visitor().visit(tree) + return { + "imports": sorted(imports), + "unbound_uses": sorted(uses), + "resource_accesses": sorted(resources), + } + + +def _covered_imports(name: str, inventory: dict) -> tuple[str, ...]: + """Omit a symbol tripwire only if every scope using this file binds it.""" + scopes = [spec for spec in inventory["stages"].values() if name in spec["modules"]] + modules = set.intersection(*(set(spec["modules"]) for spec in scopes)) + dependencies = set.intersection(*(set(spec["dependencies"]) for spec in scopes)) + covered = set() + for module in modules: + package, relative = module.split("/", 1) + resolved = package + "." + relative.removesuffix(".py").replace("/", ".") + covered.add(resolved.removesuffix(".__init__")) + covered.update( + "*" + ("yaml" if dependency == "PyYAML" else dependency) + for dependency in dependencies + ) + return tuple(sorted(covered)) + + +def _dependency_contract(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + details = _dependency_details(payload, name, covered) + return { + "imports": details["imports"], + "unbound_uses_sha256": _digest(_canonical(details["unbound_uses"])), + "resource_accesses_sha256": _digest(_canonical(details["resource_accesses"])), + } + + +def _inventory(roots: Mapping[str, Path]) -> tuple[dict, bytes]: + payload = ( + roots["microcosm.build"] / "us_runtime/graph_implementation_inventory.json" + ).read_bytes() + value = json.loads(payload) + if value["schema"] != "microcosm.us.implementation-inventory.v1": + raise ValueError("Unsupported US implementation inventory schema.") + if set(value["stages"]) != set(STAGE_DEPENDENCIES): + raise ValueError("US implementation stage dependency inventory differs.") + for name, dependencies in STAGE_DEPENDENCIES.items(): + if value["stages"][name]["dependencies"] != list(dependencies): + raise ValueError(f"US implementation dependency inventory differs: {name}.") + imports = { + name for contract in value["contracts"].values() for name in contract["imports"] + } + if set(value["import_classifications"]) != imports: + raise ValueError("US implementation import classification inventory differs.") + return value, payload + + +def _validate_package_roster(package: str, root: Path, inventory: dict) -> None: + actual = {path.relative_to(root).as_posix() for path in root.rglob("*.py")} + expected = inventory["package_rosters"][package] + if len(expected) != len(set(expected)) or actual != set(expected): + raise ValueError(f"US implementation Python inventory differs: {package}.") + if package == "microunit": + # py.typed is packaging metadata. Everything else must be explicitly + # bound, including an added non-Python rule/data file. + actual_resources = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + and path.suffix != ".py" + and "__pycache__" not in path.parts + and path.name != "py.typed" + } + if actual_resources != set(inventory["package_resources"][package]): + raise ValueError("US implementation resource inventory differs: microunit.") + + +def _projections() -> dict: + from . import operator_boundary + + def columns(value): + if isinstance(value, Mapping): + return {key: columns(item) for key, item in sorted(value.items())} + if not isinstance(value, (tuple, list, frozenset, set)) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "US source registry projection requires string column sets." + ) + return sorted(value) + + return { + name: columns(getattr(operator_boundary, name)) + for name in ( + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "FORMULA_OWNED_SOURCE_COLUMNS", + ) + } + + +def _loader_ref(loader: object, expected: str, roots: Mapping[str, Path]) -> None: + actual = ( + f"{getattr(loader, '__module__', '')}:{getattr(loader, '__qualname__', '')}" + ) + module, _ = expected.split(":") + package = next(name for name in roots if module.startswith(name + ".")) + path = roots[package] / ( + module.removeprefix(package + ".").replace(".", "/") + ".py" + ) + # A correctly named function imported from another checkout is not the + # implementation whose file bytes this manifest promises. + source = inspect.getsourcefile(loader) if inspect.isfunction(loader) else None + if actual != expected or source is None or Path(source).resolve() != path.resolve(): + raise ValueError( + f"US source codec loader differs from declared reference: {expected}." + ) + + +def validate_source_codecs(codecs) -> None: + """Bind US registry names to real loaders; generic graph codec APIs stay unchanged.""" + roots = _package_roots() + # Explicit declared selectors keep this registry inspection source-blind. + try: + loaders = ( + codecs.get("us-asec-raw-stage-v4"), + codecs.get("us-acs-native-2024-v1"), + codecs.get("us-asec-prepared-current-money-v3"), + codecs.get("raw-bytes-v1"), + codecs.get("us-acs-housing-universe-2024-v1"), + ) + except Exception as error: + raise ValueError("US source codec loader missing.") from error + for (_, (_, expected)), loader in zip(_CODECS.items(), loaders, strict=True): + _loader_ref(loader, expected, roots) + + +def implementation_manifest(stage: str) -> dict: + """Return small auditable identities, never source data or absolute paths.""" + if stage not in STAGE_DEPENDENCIES: + raise ValueError(f"Unclassified US implementation stage: {stage}.") + roots = _package_roots() + inventory, inventory_bytes = _inventory(roots) + spec = inventory["stages"][stage] + modules, resources = {}, {} + packages = {name.split("/", 1)[0] for name in spec["modules"]} + if not packages <= roots.keys(): + raise ValueError("US implementation dependency package inventory unavailable.") + for package in packages - {"microcosm.build"}: + _validate_package_roster(package, roots[package], inventory) + if len(spec["modules"]) != len(set(spec["modules"])): + raise ValueError("Duplicate US implementation module inventory.") + for name in spec["modules"]: + package, relative = name.split("/", 1) + path = roots[package] / relative + if not path.is_file(): + raise ValueError(f"US implementation module inventory missing: {name}.") + payload = path.read_bytes() + actual = _dependency_contract(payload, name, _covered_imports(name, inventory)) + if actual != inventory["contracts"][name]: + raise ValueError(f"Unclassified US dependency/resource contract: {name}.") + modules[name] = _digest(payload) + for name in spec["resources"]: + package, relative = name.split("/", 1) + resources[name] = _digest((roots[package] / relative).read_bytes()) + manifest = { + "schema": "microcosm.us.implementation-manifest.v1", + "stage": stage, + "inventory_sha256": _digest(inventory_bytes), + "modules": modules, + "resources": resources, + "dependencies": { + name: importlib_metadata.version(name) for name in STAGE_DEPENDENCIES[stage] + }, + "projections": _projections() if spec["source_boundary_projection"] else {}, + "runtime_options": ( + { + "pandas.mode.string_storage": pd.get_option("mode.string_storage"), + "pandas.future.infer_string": pd.get_option("future.infer_string"), + "acs_reader_resolved_string_storage": pd.StringDtype().storage, + } + if stage + in { + "acs_codec_2024", + "assembly_prepare", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + } + else {} + ), + "codecs": {}, + } + if stage in { + "asec_prepared_v3", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + }: + # These stages parse pinned official CSV members with the stdlib reader. + # csv's field size limit is process state, so it belongs in the identity + # of each stage that reads under it. Whole-module and inventory hashes + # separately record code changes, including additions of new stages. + manifest["runtime_options"]["csv.field_size_limit"] = csv.field_size_limit() + if stage in { + "assembly_prepare", + "composed_population_v1", + "composed_asec_binding_v1", + }: + bound = ( + ("asec_codec_v4", "acs_codec_2024") + if stage == "assembly_prepare" + # The composed CREATE reads the prepared directory, not the + # raw-stage checkpoint, so it binds that codec's identity instead. + # The binding stage reads no source, but every artifact it admits + # was produced under those two codecs, so it binds them too rather + # than claiming an identity narrower than its actual inputs. + else ("asec_prepared_v3", "acs_codec_2024") + ) + for name, (codec_stage, loader) in _CODECS.items(): + if codec_stage in bound: + manifest["codecs"][name] = { + "loader": loader, + "implementation_sha256": implementation_hash(codec_stage), + } + elif stage == "authenticated_survey_population_v1": + # The standalone factory registers one real closure over an operational + # capture directory. Its whole module belongs to this stage. The legacy + # five-codec registry and its strict validator retain their contract. + manifest["codecs"] = { + "us-survey-population-source-v1": { + "loader": ( + "microcosm.build.us_runtime.graph_survey_population:" + "survey_population_source_codecs..load" + ) + } + } + elif stage in ( + "asec_codec_v4", + "acs_codec_2024", + "geography", + "asec_prepared_v3", + "acs_housing_universe_2024", + ): + manifest["codecs"] = { + name: {"loader": loader} + for name, (codec_stage, loader) in _CODECS.items() + if codec_stage == stage + } + return manifest + + +def implementation_hash(stage: str) -> str: + return _digest( + b"microcosm.us.scoped-implementation.v1\0" + + _canonical(implementation_manifest(stage)) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json new file mode 100644 index 000000000..ebd5b5066 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -0,0 +1,2488 @@ +{ + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_checkpoint.py": { + "imports": [ + "h5py", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "6e84799b53beb6b39981b770f65412a60e3808f6416bda934c0017ad36196e13", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_sampling.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/gates.py": { + "imports": [ + "microcosm.build.ledger_targets", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.calibrate.registry", + "microcosm.calibrate.solve", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "06b1202277975ae894fc8ece00dbac4b51129c78ab41d44d45aa86969f8e940e" + }, + "microcosm.build/outer_stage_runtime.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a0d0bd839844e63cf2581413187e20748db921b657051a72a022ea6185176fd0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/serialization_dtypes.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_allocation.py": { + "imports": [ + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_domain_sample.py": { + "imports": [ + "microcosm.build.survey_allocation", + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/table_identity.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/_asec_current_money_codec.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.acs_sources", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a8d2536f4bb75f36e40995185901ce8867ce1c01478efba980a75d0bd778b3f7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_inputs.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_native_coverage_binding.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.frame" + ], + "resource_accesses_sha256": "493c1a38451cc17c161232077fcea53211dc46f91695d6ec2485d755da52b291", + "unbound_uses_sha256": "937ac46755604aef55c18570df2520f92e51dd0486b93e2316a160cdaa67ad6a" + }, + "microcosm.build/us_runtime/acs_person_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27da902db7790fcf13f79f383532fb1eaffea49c118f54f76870c36cad264004", + "unbound_uses_sha256": "dc548c6590471125d90a86717034d6ed275b6e98115676512a623e4b9f7dc323" + }, + "microcosm.build/us_runtime/acs_person_coverage_columns.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "pandas" + ], + "resource_accesses_sha256": "94d488b944fbeaa44d07120dd12f7f1b01e7e20faddf2e3daac77d905c2eca88", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8899ffc02d4808966ab99d6897940046a3d422acb99014c0f3cb44591978c775", + "unbound_uses_sha256": "c175b4f5a567b65b62146020a7b0ae39aa9d32d57bdf0839618b13d1003721a1" + }, + "microcosm.build/us_runtime/acs_pums.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.frame", + "microcosm.frame.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3de7aae8590dfd67cfd50c129c5fca3b3ba9eb8b2444ed59f84f92ff94a735ee", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_sources.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums" + ], + "resource_accesses_sha256": "1f6355e38c0cc1c36391097afe9917115275b0e5585a9429af6007419dbee036", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/alimony.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.source_manifest", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "905273f67cd9edd6939cb6467fd9466c837f90603e4fff9365c09b42326ce7b9", + "unbound_uses_sha256": "c07a32776a6c50eb55aa704fa8671cf2e5a029c88ccd2c86e418ed0310d6fd43" + }, + "microcosm.build/us_runtime/asec_2024_native_population.py": { + "imports": [ + "microcosm.build.cd_benchmark", + "microcosm.build.us_runtime", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "466e46803aecf67189f31031cdb4eb9287af37669e5fa0b36610788d62564e46", + "unbound_uses_sha256": "71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608" + }, + "microcosm.build/us_runtime/asec_checkpoint.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.education_assistance_source", + "pandas" + ], + "resource_accesses_sha256": "f1826cc233390a0a2b3e97a9905d95d51df58edb6bcee521e8893be26d37969d", + "unbound_uses_sha256": "f03f18df090746827bf57705d6bcc5bc28fcaaf07c16832c8450ac3f28c1c492" + }, + "microcosm.build/us_runtime/asec_current_money.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_graph_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "eaea1354548803c511b49d0f3c3d9d88bb50700553056a34e88a2c314de2282e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "2a2ed530efb66ac76ba8757f6bd5bd7492e586fe2f28c6d2eddada8aed722f78", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_selection.py": { + "imports": [ + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.graph", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.asec_person_income_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "10c92df9a22413a1f5bdee648fc1a79edefc13d469c1f915c14b2c3f90eb605d", + "unbound_uses_sha256": "edeb60d6acaab0ae263edf0537a6de3887a07bbd00382f545191e41cb1a1e278" + }, + "microcosm.build/us_runtime/asec_current_money_units.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_source", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.frame", + "microunit", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "5bebc4222c02f5c2f4e165df30b178cea156109dfbff6d6d88e5aed190e34e9c", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_engine_evaluation.py": { + "imports": [ + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "ef73c9051031f964f8066b33572925566d6e8e7757d633429d48f67951d462ec", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_household_coverage_fields.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "57a3c65a298ad7f131570a5873ebb67b964c566b1f6368636b53e4a4c16c39f3", + "unbound_uses_sha256": "7a523e77a972d9ee60ba7ad78d95c13660d89df0920c6e82ecb98f38292c7cc8" + }, + "microcosm.build/us_runtime/asec_household_observations.py": { + "imports": [ + "h5py", + "microcosm.build.outer_stage_runtime", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "c44acab8834834c2e938718ae7f8da07bc2a3ee1e147c4f5d1ca40c7654d1180", + "unbound_uses_sha256": "17ce33d5d0b830d29e75dbee4a5c87d6f5b08415ac1d421f23aa46bb73b1c8f8" + }, + "microcosm.build/us_runtime/asec_housing_status.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "11b2b6becb66ad05e2cc9d4728cb56c348701971f0dcf4cd491d56a8c813b23f", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_status_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bebe07e6f115689ca38ae68c2d7f69df4ab09cfaad3326e622f9f7b8a9dc89a0", + "unbound_uses_sha256": "ec593873f78e53325d93fc7f8cd3422f3935c0bec2a60120738bd3e606e065fc" + }, + "microcosm.build/us_runtime/asec_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_universe_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ac86369b087151b3b538064a5d1dcfaa533bd2adf1948c1132fb6ace4a2365d0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_universe_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.asec_housing_universe", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "0f622ceee1c600d15eb314ecf7b88121f6c707b9cf8703c2965af95240ec73ee", + "unbound_uses_sha256": "a8cb1277b205b614033078bca03e9c055d190134adc1172bcae5294c76d2737e" + }, + "microcosm.build/us_runtime/asec_income_observations.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "92fb0be32eb3167e2c7fdcff8ba929ef0ad3b8a49d50eb5c19a5e7431d84fe23", + "unbound_uses_sha256": "21636adcde1832a321b968f7959d8c75331d08831f9ed093003a26eb450acecd" + }, + "microcosm.build/us_runtime/asec_original_household_weights.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8c98f1d18560b79086a6ce71868f5800b3d6b8a214bb20446cafe3e3204e6180", + "unbound_uses_sha256": "d01a0598bee95a0bc72aa37ad70ec054cc52e17e07f426f9cb3d25ca464d1c9a" + }, + "microcosm.build/us_runtime/asec_person_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "d048b0a585f81a8902dc11ef2fa5da96c8874f77802885fdaa08c7e69764994b", + "unbound_uses_sha256": "5a52b6865fa982890fc246f9fcf2568fb77fa725623e9759dd3342e26fae2ae7" + }, + "microcosm.build/us_runtime/asec_person_income_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ef03627f20e0d35ccaacd16397281e6bbe5b1f765a55f58cc4702a3fe2dfe2a3", + "unbound_uses_sha256": "d3ed60989ff82d6e0c96de69dcd77106689c19ecdb7f96c57a4f6d15bd0d415d" + }, + "microcosm.build/us_runtime/asec_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "2622beb3c86b2a9e1cea2db1c6b34a1ec2d2cf29824c3d3e276ea41baeb4f93b", + "unbound_uses_sha256": "05ea2b2c3d774e57d76e89616228798fa8ef05c1f6c8524b08816b5ad3844a81" + }, + "microcosm.build/us_runtime/asec_prepared_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d53265b60115118b47ecd636a9fb01a878791d7c9fb1deff86ea21e35947b9c6", + "unbound_uses_sha256": "57f904134e8a69655e1b36357fd288a9ab69966fe10f27dd4f7607c80fba93af" + }, + "microcosm.build/us_runtime/asec_student_controls.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d20705bfb2110cfc573f493e988e0dbc47b5441e2bda83ef5425110adab2d4e8", + "unbound_uses_sha256": "983062f97faeff20a1761770968d967cc871520de8decbbd98c8018067475796" + }, + "microcosm.build/us_runtime/congressional_district_geography.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/congressional_district_vintage.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "cba8c434f995a8a867f6b67951909b2038954efc13cf25c04ea958ffac6a8f61", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/cps_carried.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.public_assistance_type_source", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "d2c7c444dbfd99bf73f141320b4a1d4a6eb409a07dbc84d36984f1bc5c95e288" + }, + "microcosm.build/us_runtime/cps_carried_current.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/education_assistance_source.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27781733e25a9a4e98fb2ee8fe937391e7a301cf22f3ed1b2faa9bf8524831a0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/geography_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "539fedb954503afc210fa2e9ad841638d8fc2fc1ab6e5070d21827cb896246fc", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_acs_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.keys", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "fb7b9295aae7f0e4db8fc053fa9fdb192fdb5176f4db5d0aa80bb0c2e61d8a86" + }, + "microcosm.build/us_runtime/graph_asec_income.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "79c5158f94d6ba24554bdfe39ec48a02debebd761b4b67ae40caa22a7c471dd2" + }, + "microcosm.build/us_runtime/graph_asec_prepared.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_graph_resources", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_engine_evaluation", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "1c0575822ea8e45ece5c94da38b4593df28eb7d96df04cece468174f1a38ff2d" + }, + "microcosm.build/us_runtime/graph_composed_asec_binding.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_asec_measures.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_asec_binding", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_contracts.py": { + "imports": [ + "microcosm.graph" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_population.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_context.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_geography.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.congressional_district_vintage", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.puma_ladder", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.kernel", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "457bc0cc87c6883ba9423769b70238b0e4ea060bca16c94d03379de01ac882c6" + }, + "microcosm.build/us_runtime/graph_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "454f9e3c1f7f1b840429f7c14c16dac67ab8219df0c9c32cf5807352b5c338ce", + "unbound_uses_sha256": "b45c8251bea93efe1847308338b21c6f8e74852564ad1a76ba5c384bd6de99a1" + }, + "microcosm.build/us_runtime/graph_sources.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe_source", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "627540b59d564b58991a32dc38aed03b9e2c852a3e9326792f21ae5beef19ebb", + "unbound_uses_sha256": "e9036a4821c20237bd98219383026cf440cdb29ac1b78f7b1bafd9be99b1dd30" + }, + "microcosm.build/us_runtime/graph_survey_population.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_combined_clone", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.survey_catalogue_selection", + "microcosm.build.us_runtime.survey_population_domains", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.codecs", + "microcosm.graph.executor", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "5a6799ab18a561e0d4fdc4e08b11656a6100216df59e268e27e0ad4d38b1c994" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + }, + "microcosm.build/us_runtime/operator_boundary.py": { + "imports": [ + "microcosm.build.us_runtime.adult_care", + "microcosm.build.us_runtime.child_support", + "microcosm.build.us_runtime.childcare", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.cps_carried", + "microcosm.build.us_runtime.disability_benefits", + "microcosm.build.us_runtime.education_inputs", + "microcosm.build.us_runtime.eligibility_inputs", + "microcosm.build.us_runtime.energy_subsidy", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.hours_worked", + "microcosm.build.us_runtime.housing_inputs", + "microcosm.build.us_runtime.immigration", + "microcosm.build.us_runtime.medicare_take_up", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.pregnancy", + "microcosm.build.us_runtime.prior_year_income", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.relationship_inputs", + "microcosm.build.us_runtime.retirement_contributions", + "microcosm.build.us_runtime.retirement_distributions", + "microcosm.build.us_runtime.scf_wealth", + "microcosm.build.us_runtime.weeks_unemployed", + "microcosm.build.us_runtime.wic_claim", + "microcosm.build.us_runtime.workers_compensation", + "microcosm.frame", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "f3e52e09d1c5b005d16f083e716966bf60087b72eba71a149a8128faa7551617" + }, + "microcosm.build/us_runtime/operator_column_contracts.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puf_support.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.puf_e01000_reconciliation", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "851b8041c69b1b721dcdc02d1ce312976a87e45b66fc4aca59d0ec538ab62b44", + "unbound_uses_sha256": "3b7e8d1e071256065b1c8a43c7fea2f9f39378b10d1613f76bfa9cbbf0cff6ad" + }, + "microcosm.build/us_runtime/puma_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.puma_ladder_sources", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "056fbb3baf3f3008aa654612e700957b3ecb795b8da9460be5e9a884930c36a7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puma_ladder_sources.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/reported_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4b87bba3a98f67416ff34353df950e2638c86d56170d03d372f62578149f3e2e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/source_csv_builtin.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/spine_assembly.py": { + "imports": [ + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/stacked_spine.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.gates", + "microcosm.build.serialization_dtypes", + "microcosm.build.source_manifest", + "microcosm.build.table_identity", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.acs_transfer", + "microcosm.build.us_runtime.late_producer_dag", + "microcosm.build.us_runtime.multispine_pool", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.post_transfer_calibration", + "microcosm.build.us_runtime.puf_aggregate_records", + "microcosm.build.us_runtime.puf_capital_gains_tail", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.puf_qrf_chain", + "microcosm.build.us_runtime.puf_support", + "microcosm.build.us_runtime.spine_assembly", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.us_late_overlap_ownership", + "microcosm.build.us_runtime.us_late_producer_registry", + "microcosm.fit", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3839407286d44ddbf3f207c63dcb92e1c0b45e2eec2e88a09c25ebb3a4711430", + "unbound_uses_sha256": "062e621661c109d963f2356bcaa67a3da0906c8e66af1dcd81122d7ce05bd454" + }, + "microcosm.build/us_runtime/support_provenance.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "bfcbcc3d41a567ee0d5b015c6715c3fb659937713db37d3b2d74e5fa2ddeead3" + }, + "microcosm.build/us_runtime/survey_catalogue_selection.py": { + "imports": [ + "microcosm.build.survey_domain_sample", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "00c2ba97f0fb73d788c8f9c6e42ab90a67c5a2f7fe947d2232441aa6b03ae27c" + }, + "microcosm.build/us_runtime/survey_observed_age.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_domains.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_preparation.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ccfed1c1acff5a1c50b538424dc129930f69c374d9841e233b3fb593a440c3aa", + "unbound_uses_sha256": "29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91" + }, + "microcosm.frame/__init__.py": { + "imports": [ + "microcosm.frame.accounting", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "microcosm.frame.weights" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/accounting.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas", + "pandas.api.types" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/__init__.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/_policyengine_us_source_index.py": { + "imports": [ + "microcosm.frame.schema", + "yaml" + ], + "resource_accesses_sha256": "a64f5650fc19608459f4e1c034e1f7e692de8ca77e4e2e111f8dd40559ce481f", + "unbound_uses_sha256": "6da25f88c6cadb5b1522da2e977a71e749dd54c6cb1d58fb1c6224161e9e42be" + }, + "microcosm.frame/adapters/axiom.py": { + "imports": [ + "axiom_rules_engine", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "9f0928dfa308893a0724f2e22e2676993aa7084c83115c4fdaecc341e515bd76" + }, + "microcosm.frame/adapters/policyengine_uk.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "policyengine_uk", + "policyengine_uk.data" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "366f98c5d3507744c7f9667e7323821c41358a47c56bd83f7ca9c677756c7094" + }, + "microcosm.frame/adapters/policyengine_us.py": { + "imports": [ + "microcosm.frame.adapters._policyengine_us_source_index", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "numpy", + "pandas", + "policyengine_us", + "policyengine_us.data" + ], + "resource_accesses_sha256": "b87a963f56b28cef5b93690cec90f1ff2e935075283e553fdf7af693f8bf3204", + "unbound_uses_sha256": "175c1d0a9621cee21f400b0c3e4618a40eb2fdb795adbf42b16488ea2dcf5b23" + }, + "microcosm.frame/bundle.py": { + "imports": [ + "microcosm.frame.schema", + "microcosm.frame.weights", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/kernels.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/materialize.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/rules.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bb3739bfa9c1bdb290ce2e1136dfd73bc4f2c40715a822ea87e3c91ef51a5760", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/schema.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/units.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "microcosm.frame.weights", + "microunit", + "microunit.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "e40d25e078da274aaaa43639b66ac70df5a8b87b6b637f46d00069e73bff3e73" + }, + "microcosm.frame/weights.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/__init__.py": { + "imports": [ + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.executor", + "microcosm.graph.explain", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.randomness", + "microcosm.graph.serialize", + "microcosm.graph.store", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/artifact_edges.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/attachments.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.store" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/availability.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/canonical.py": { + "imports": [ + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/codecs.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4adeae91a0658854557dc4d2bf01d09ad531726de7cec890f00258a2dbe81da2", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/decl.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/errors.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/executor.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.attachments", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/explain.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/kernel.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.decl", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "313aceccf09abe24eace2f6d432f41960afb9ed7e80ebe890a10cc41557d87d3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/keys.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "5e91517c603797fcf9131c3cffd0119c3226dd706f2c219658ffd571c5aa893c", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/manifest.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.artifact_edges", + "microcosm.graph.attachments", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.population", + "microcosm.graph.store", + "pandas" + ], + "resource_accesses_sha256": "13c805cbaa1c2fd026afc6574c4c08dd71143a68f787d616852d5efcbb151c82", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/population.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/randomness.py": { + "imports": [ + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/serialize.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/store.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.errors", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "99afee26ca660c678dcbd7117eb2e64d70a0343ead9a602fa5819222aca02bc3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/view.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.manifest" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/__init__.py": { + "imports": [ + "microunit.core", + "microunit.diagnostics", + "microunit.registry", + "microunit.rule_helpers", + "microunit.tax_unit_construction" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/core.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/diagnostics.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/registry.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/rule_helpers.py": { + "imports": [ + "yaml" + ], + "resource_accesses_sha256": "d880e849f439db697d177523c8c7e92775b109fc42d2f71372e2702dd9c19749", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/tax_unit_construction.py": { + "imports": [ + "microunit.rule_helpers", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/__init__.py": { + "imports": [ + "microunit.units.medicaid", + "microunit.units.passthrough", + "microunit.units.programs", + "microunit.units.snap", + "microunit.units.spm", + "microunit.units.tax" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/_helpers.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/medicaid.py": { + "imports": [ + "microunit.core", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/passthrough.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/programs.py": { + "imports": [ + "microunit.core", + "microunit.units.spm", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/snap.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/spm.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/tax.py": { + "imports": [ + "microunit.core", + "microunit.tax_unit_construction", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + "import_classifications": { + "axiom_rules_engine": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "h5py": "versioned dependency in explicit stages", + "microcosm.build": "package namespace import of explicitly bound frame_checkpoint; other package exports are inactive in this slice", + "microcosm.build.cd_benchmark": "package-relative original source-member identity alias; origin and its pure canonical helper are explicitly whole-module bound in authenticated_survey_population_v1", + "microcosm.build.cd_benchmark.canonical": "whole-module pure canonical JSON and source digest validation in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.cd_benchmark.origin": "whole-module original source-member identity in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.frame_checkpoint": "whole-module in explicit stages", + "microcosm.build.frame_sampling": "whole-module in explicit stages", + "microcosm.build.gates": "whole-module in explicit stages", + "microcosm.build.ledger_targets": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.outer_stage_runtime": "whole-module in explicit stages", + "microcosm.build.serialization_dtypes": "whole-module in explicit stages", + "microcosm.build.source_manifest": "inactive legacy alimony source-stage declaration; scoped-use and resource-access tripwires retained", + "microcosm.build.survey_allocation": "whole-module pure domain share and original-anchor contracts in authenticated_survey_population_v1", + "microcosm.build.survey_domain_sample": "whole-module deterministic per-domain household selection in authenticated_survey_population_v1", + "microcosm.build.table_identity": "whole-module in explicit stages", + "microcosm.build.us_runtime": "package-relative live aliases for original money/schema/checkpoint/household/person-income/housing verifiers, separate student controls and graph context are explicitly module-bound in asec_prepared_v3; other shared-module or optional routes remain inactive with scoped-use tripwires; v2 adds package-relative HU definition/source aliases and non-authoritative transport, each explicitly whole-module bound. The authenticated survey stage additionally binds its exact catalogue, selected-native, original-source, pure-selection and preparation aliases; optional PUF clone validation remains downstream and bound by the clone operator identity.", + "microcosm.build.us_runtime._asec_current_money_codec": "whole-module in asec_prepared_v3; original canonical full-source replay unchanged", + "microcosm.build.us_runtime.acs_housing_universe": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_housing_universe_source": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_income_universe": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.acs_inputs": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_pums": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_sources": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_transfer": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.adult_care": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.alimony": "whole-module in asec_prepared_v3 for pure ASEC split; source manifest and other nominal routes inactive", + "microcosm.build.us_runtime.asec_checkpoint": "whole-module in explicit stages", + "microcosm.build.us_runtime.asec_current_money": "whole-module in asec_prepared_v3; immutable recipe source identity", + "microcosm.build.us_runtime.asec_current_money_graph_resources": "whole-module in asec_prepared_v3; additive status loader leaves original T1/money resource identity unchanged", + "microcosm.build.us_runtime.asec_current_money_resources": "whole-module in asec_prepared_v3; original recipe and installed microunit verification unchanged", + "microcosm.build.us_runtime.asec_current_money_selection": "whole-module in asec_prepared_v3; selected body has provenance but no full-source authority", + "microcosm.build.us_runtime.asec_current_money_source": "whole-module in asec_prepared_v3; authenticated original source verification", + "microcosm.build.us_runtime.asec_current_money_units": "whole-module in asec_prepared_v3; real microunit with separate authenticated student controls", + "microcosm.build.us_runtime.asec_engine_evaluation": "whole-module in asec_prepared_v3; pinned complete installed formula closures and empty defaults allowlist", + "microcosm.build.us_runtime.asec_housing_status": "whole-module in asec_prepared_v3; status definition and sealed authenticated artifact", + "microcosm.build.us_runtime.asec_housing_status_source": "whole-module in asec_prepared_v3; housing source remains original T1", + "microcosm.build.us_runtime.asec_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_housing_universe_source": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_person_income_source": "whole-module in asec_prepared_v3; unchanged T1 restoration verifier closure", + "microcosm.build.us_runtime.asec_prepared_source": "whole-module in asec_prepared_v3; registry-only import in other graph_sources scopes retains scoped-use tripwire", + "microcosm.build.us_runtime.asec_student_controls": "whole-module in asec_prepared_v3; separate S authority before tax units, never part of original T1 verifier", + "microcosm.build.us_runtime.child_support": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.childcare": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.congressional_district_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.congressional_district_vintage": "whole-module in explicit stages", + "microcosm.build.us_runtime.cps_carried": "whole-module in asec_prepared_v3 for split constants; nominal consumer route inactive", + "microcosm.build.us_runtime.cps_carried_current": "whole-module in asec_prepared_v3; typed corrected money amounts and declared raw routing only", + "microcosm.build.us_runtime.disability_benefits": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.education_assistance_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.education_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.eligibility_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.energy_subsidy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.geography_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_asec_income": "Whole-module in asec_prepared_v3; PAW source evidence, bound reported-income accounting or shared typed graph declarations. No source authority is minted from graph transport.", + "microcosm.build.us_runtime.graph_asec_prepared": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_combined_clone": "optional downstream clone declaration/registration; clone kernels retain their own implementation identities; no donor or target resource in authenticated_survey_population_v1", + "microcosm.build.us_runtime.graph_composed_asec_binding": "whole-module in composed_asec_binding_v1; composed-arm resolution, its typed binding/arm-row transport and the declared row mask. Reads source provenance by charter and mints no source authority.", + "microcosm.build.us_runtime.graph_composed_contracts": "Pure composed producer IDs and artifact types; whole module bound by both legacy composed stages and explicit PUF extension hashes. No resources or operation dispatch.", + "microcosm.build.us_runtime.graph_composed_population": "whole-module in composed_population_v1 and composed_asec_binding_v1; the composing CREATE's declarations, node ids and typed origin binder. Adds no source route of its own.", + "microcosm.build.us_runtime.graph_context": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.graph_implementation": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_sources": "whole-module in explicit graph stages; prepared slice uses frame column declarations and validated codec registration only; authenticated survey preparation uses only frame column declarations and lossless storage canonicalization, not legacy source codecs or assembly treatment", + "microcosm.build.us_runtime.hours_worked": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.housing_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.immigration": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.late_producer_dag": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.medicare_take_up": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.multispine_pool": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.operator_boundary": "whole-module in explicit stages", + "microcosm.build.us_runtime.operator_column_contracts": "Pure shared column declarations bound by every source/assembly scope that consumes them; no runtime operator or resource loading.", + "microcosm.build.us_runtime.post_transfer_calibration": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.pregnancy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.prior_year_income": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.public_assistance_type_source": "inactive legacy cps_carried consumer route; split constants used by corrected slice do not call this reader", + "microcosm.build.us_runtime.puf_aggregate_records": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_capital_gains_tail": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.puf_e01000_reconciliation": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_interest_components": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_qrf_chain": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_support": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder_sources": "whole-module in explicit stages", + "microcosm.build.us_runtime.qbi_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.relationship_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.reported_coverage_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.retirement_contributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.retirement_distributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.scf_wealth": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.source_csv_builtin": "Whole-module in acs_housing_universe_2024; stdlib-only checked CSV builtin capture, without source or resource access.", + "microcosm.build.us_runtime.spine_assembly": "whole-module in explicit stages", + "microcosm.build.us_runtime.stacked_spine": "whole-module in explicit stages", + "microcosm.build.us_runtime.support_provenance": "whole-module in explicit stages", + "microcosm.build.us_runtime.survey_catalogue_selection": "whole-module pure complete-catalogue selection plan in authenticated_survey_population_v1", + "microcosm.build.us_runtime.survey_population_domains": "whole-module raw source-record classification and prespecified development allocation declaration in authenticated_survey_population_v1", + "microcosm.build.us_runtime.us_late_overlap_ownership": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.us_late_producer_registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.weeks_unemployed": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.wic_claim": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.workers_compensation": "live registry projection; other shared-module routes inactive", + "microcosm.calibrate.registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.calibrate.solve": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.fit": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.frame": "whole-module in explicit stages", + "microcosm.frame.accounting": "whole-module in explicit stages", + "microcosm.frame.adapters._policyengine_us_source_index": "whole-module in explicit stages", + "microcosm.frame.adapters.policyengine_us": "whole-module in explicit stages", + "microcosm.frame.bundle": "whole-module in explicit stages", + "microcosm.frame.materialize": "whole-module in explicit stages", + "microcosm.frame.rules": "whole-module in explicit stages", + "microcosm.frame.schema": "whole-module in explicit stages", + "microcosm.frame.units": "whole-module in explicit stages", + "microcosm.frame.weights": "whole-module in explicit stages", + "microcosm.graph": "whole-module in explicit stages", + "microcosm.graph.artifact_edges": "whole-module in explicit stages", + "microcosm.graph.attachments": "whole-module in explicit stages", + "microcosm.graph.availability": "whole-module in explicit stages", + "microcosm.graph.canonical": "whole-module in explicit stages", + "microcosm.graph.codecs": "whole-module in explicit stages", + "microcosm.graph.decl": "whole-module in explicit stages", + "microcosm.graph.errors": "whole-module in explicit stages", + "microcosm.graph.executor": "whole-module in explicit stages", + "microcosm.graph.explain": "whole-module in explicit stages", + "microcosm.graph.kernel": "whole-module in explicit stages", + "microcosm.graph.keys": "whole-module in explicit stages", + "microcosm.graph.manifest": "whole-module in explicit stages", + "microcosm.graph.population": "whole-module in explicit stages", + "microcosm.graph.randomness": "whole-module in explicit stages", + "microcosm.graph.serialize": "whole-module in explicit stages", + "microcosm.graph.store": "whole-module in explicit stages", + "microcosm.graph.view": "whole-module in explicit stages", + "microunit": "whole-module in explicit stages", + "microunit.core": "whole-module in explicit stages", + "microunit.diagnostics": "whole-module in explicit stages", + "microunit.registry": "whole-module in explicit stages", + "microunit.rule_helpers": "whole-module in explicit stages", + "microunit.tax_unit_construction": "whole-module in explicit stages", + "microunit.units": "whole-module in explicit stages", + "microunit.units._helpers": "whole-module in explicit stages", + "microunit.units.medicaid": "whole-module in explicit stages", + "microunit.units.passthrough": "whole-module in explicit stages", + "microunit.units.programs": "whole-module in explicit stages", + "microunit.units.snap": "whole-module in explicit stages", + "microunit.units.spm": "whole-module in explicit stages", + "microunit.units.tax": "whole-module in explicit stages", + "numpy": "versioned dependency in explicit stages", + "pandas": "versioned dependency in explicit stages", + "pandas.api.types": "versioned dependency in explicit stages", + "policyengine_uk": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_uk.data": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_us": "live evaluation only, explicitly pinned installed runtime and parameter identity; inactive in other shared adapter routes", + "policyengine_us.data": "live evaluation dataset adapter only, bound by pinned installed PE-US runtime identity; inactive in other stages", + "yaml": "versioned dependency in explicit stages" + }, + "package_resources": { + "microunit": [ + "data/dependent_gross_income_limit.yaml" + ] + }, + "package_rosters": { + "microcosm.frame": [ + "__init__.py", + "accounting.py", + "adapters/__init__.py", + "adapters/_policyengine_us_source_index.py", + "adapters/axiom.py", + "adapters/policyengine_uk.py", + "adapters/policyengine_us.py", + "bundle.py", + "kernels.py", + "materialize.py", + "rules.py", + "schema.py", + "units.py", + "weights.py" + ], + "microcosm.graph": [ + "__init__.py", + "artifact_edges.py", + "attachments.py", + "availability.py", + "canonical.py", + "codecs.py", + "decl.py", + "errors.py", + "executor.py", + "explain.py", + "kernel.py", + "keys.py", + "manifest.py", + "population.py", + "randomness.py", + "schema.py", + "serialize.py", + "store.py", + "view.py" + ], + "microunit": [ + "__init__.py", + "core.py", + "diagnostics.py", + "registry.py", + "rule_helpers.py", + "tax_unit_construction.py", + "units/__init__.py", + "units/_helpers.py", + "units/medicaid.py", + "units/passthrough.py", + "units/programs.py", + "units/snap.py", + "units/spm.py", + "units/tax.py" + ] + }, + "reviewed_base": "c71e31654faf0e3f874e68e8c100b09c7eb9955f", + "schema": "microcosm.us.implementation-inventory.v1", + "scope_notes": { + "asec_prepared_engine": "PE-US/Core are live only in the evaluation kernel. That kernel adds their dependencies and verifies pinned installed implementation/parameter bytes in its implementation identity before cache lookup. The common prepared scope can be hashed without engine extras; its local adapter/source-index modules and empty-defaults resource are bound.", + "asec_prepared_v3": "Full P/H/T1 restoration, unchanged money recipe, separate S authentication/attachment before tax units, housing authentication against original T1, lossless dtype promotion, typed whole-household selection, corrected CPS leaves and four admitted PE outputs. Graph/Frame/microunit rosters are bound conservatively. Only declared source codec registration and frame column declarations run from graph_sources; ACS and stack assembly routes remain inactive. Alimony's pure ASEC split and cps_carried's split constants are bound, while their legacy consumer/source-manifest routes are inactive. V2 additionally authenticates HU from original T1, attaches it after housing status, preserves the pre-HU parent signature and carries native HU bytes plus selected-row ancestry with exact int64 promotion; pure ACS HU remains inactive. V3 additionally issues separate PAW_VAL/PAW_YN observations from the same official members and a derived six-column reported-income branch. The 33-field body, T1/S definitions, selected-money v1 and selection v2 payload contract remain sealed; historical v2 artifacts are preserved, but v3 runtime keys invalidate old caches.", + "authenticated_survey_population_v1": "Complete ACS and ASEC catalogues authenticate original source records before one domain draw. Selected native issuers produce original DESIGN anchors, which are stacked without share normalization; the graph applies share/inclusion probability once. Explicit narrow source closure includes pure origin canonical encoding and the original household/person custody helpers, not the legacy prepared ASEC resource union. Original source JSONs and microunit YAML are the only packaged data. The isolated real source-codec loader is declared separately from the unchanged global five codecs. The runner reconstructs source authority on cold and warm invocations and checks actual populations, design ancestry, owners, weight kinds, mass ledgers and typed artifact descriptors. Optional PUF support cloning follows over both survey arms under existing clone implementation identities. No donor fit/draw, policy engine, calibration or target measurement is part of this stage. Initial source and graph snapshots remain preliminary pending integrated invented execution and independent review.", + "bound_modules": "Whole bytes of explicitly named modules; helper-only parsing never imports adapters.", + "composed_asec_binding_v1": "Binding the carried prepared-ASEC evidence to the composed ASEC rows and deriving the reviewed corrected leaves and reported observations over them. This stage reads no source; every artifact it admits was produced by composed_population_v1, and every verifier, selector and derivation it reuses already lives in that scope, so its module inventory is exactly that scope's plus the two binding modules and its dependency set is identical. Because the inventory is a superset of composed_population_v1 and therefore of each of its parents, no existing module's covered-import set moves. It binds the same two codecs the composing CREATE binds rather than claiming an identity narrower than its actual inputs.", + "composed_population_v1": "The composed prepared-ASEC/native-ACS population. One CREATE runs both reviewed source scopes, so this stage's module inventory is exactly the union of assembly_prepare, asec_prepared_v3 and geography plus the composing module, and its dependency set is theirs. Because the union is a superset of each parent, no existing module's covered-import set and therefore no existing stage identity moves. It binds the prepared codec and the ACS codec it actually reads; the raw-stage codec is not bound because this stage never reads it. Sampling, stacking, provenance and importance allocation remain in the unchanged stacked_spine/spine_assembly modules, and harmonization reuses the assembly_harmonize scope through its own kernel. No engine, clone, transfer or calibration route is active in this scope.", + "current_money_resources": "Original source recipe and consumers declaration bytes remain unchanged to preserve authenticated T1/money identities. The additive graph consumer resource is separately pinned and supersedes only implementation status for the graph slice and microunit; all old nominal consumer routes remain pending.", + "occupied_housing_review_base": "HU v1 source definition and attachment are exact approved 4b26e59b additions on d4bddcb6ac2aba52903cd8dace90330cee4a21e1; only the v2 prepared graph/transport wiring is new. Original source-verifier modules and resources are unchanged.", + "operator_boundary": "Resolved output/formula registries projected; provider bodies are not called by source construction.", + "other_imports": "Canonical non-stdlib import modules are explicitly classified. Ordinary symbols from a module/dependency bound in every using scope need no inventory edit. Only unbound-helper uses retain a lexical-scope fingerprint; resource-access expressions retain their own fingerprint. Expanded AST details are audit output, not packaged data. Neither fingerprint is a proof of dynamic closure.", + "resources": "Microunit rule YAML is bound. Default packaged geography/source-stage resources belong to inactive legacy entrypoints; graph lookups are explicit SourceRefs." + }, + "stages": { + "acs_codec_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "acs_housing_universe_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "asec_codec_v4": { + "dependencies": [ + "numpy", + "pandas", + "h5py" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": true + }, + "asec_prepared_v3": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "assembly_harmonize": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + }, + "assembly_prepare": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "authenticated_survey_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/survey_allocation.py", + "microcosm.build/survey_domain_sample.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_native_coverage_binding.py", + "microcosm.build/us_runtime/acs_person_coverage_authentication.py", + "microcosm.build/us_runtime/acs_person_coverage_columns.py", + "microcosm.build/us_runtime/acs_population_catalogue.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/asec_2024_native_population.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_coverage_authentication.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_household_coverage_fields.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_original_household_weights.py", + "microcosm.build/us_runtime/asec_person_coverage_source.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_population_catalogue.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/graph_survey_population.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.build/us_runtime/survey_catalogue_selection.py", + "microcosm.build/us_runtime/survey_observed_age.py", + "microcosm.build/us_runtime/survey_population_domains.py", + "microcosm.build/us_runtime/survey_population_preparation.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_asec_binding_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_asec_binding.py", + "microcosm.build/us_runtime/graph_composed_asec_measures.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "geography": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/attachments.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py new file mode 100644 index 000000000..c30472fa5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py @@ -0,0 +1,451 @@ +"""Disjoint household age counts, and the composition that calibrates them. + +This is an ordinary source-blind population operator. It declares exactly what +it reads — each person's age and household membership, and the household ids — +and owns one int64 count column per declared age band. It opens no file, reads +no source-channel or spine column, imports no rules engine, calculates nothing +from a model, and never touches weights: these are unweighted head counts, and +the graph executor owns every weight in the build. + +The graph contract splits that read surface in two, and the declaration follows +it rather than working around it. ``age`` is an owned cell, so it travels in an +input :class:`~microcosm.graph.Slice`. Entity ids and person-to-group +memberships are structural: the executor puts them in every context table and +no node owns them, so a ``Slice`` cannot name them. They are declared instead +in the node's normative parameters, where they still enter the node key, and +the kernel refuses a context whose household view carries anything beyond the +declared id column. + +Every included person is counted exactly once. Group-quarters people are +included because the activated target universe is the whole resident +population, and their household membership is preserved rather than dropped. +The bands are declared, contiguous from zero, disjoint, and closed by a single +open top band, so an age outside every band is impossible by construction and a +person can never fall into two. + +Refusals are explicit; nothing is silently dropped. An unknown, non-integer, +negative or non-finite age refuses the node, as does a membership that names no +household row, a duplicated household id, or a household row that no person +belongs to. The last one matches the population contract itself, which requires +each group row to be referenced by at least one person — a memberless household +is a broken population, not a zero-count household. + +:func:`national_age_calibration_nodes` composes this operator with the existing +`demographic_calibration_node`, unchanged. It emits no US frame-context +artifact: a context written before reweighting must not be relabelled after it, +so consumers read the executor-owned calibrated population and the diagnostics +edge instead. +""" + +from __future__ import annotations + +import json +import sys + +import numpy as np +import pandas as pd +from pandas.api.types import ( + is_bool_dtype, + is_complex_dtype, + is_integer_dtype, + is_numeric_dtype, +) + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) + +from .demographic_calibration_graph import demographic_calibration_node +from .national_age_activation import NATIONAL_AGE_ACTIVATION, AgeBand + +__all__ = [ + "AGE_COUNT_SCHEMA_VERSION", + "SUPPORTED_AGE_CONVENTION", + "NationalAgeCountKernel", + "national_age_count_node", + "national_age_calibration_nodes", +] + +#: Revision of the band-schema parameter's canonical form. +AGE_COUNT_SCHEMA_VERSION = 1 + +#: The one convention this operator implements: the person's ``age`` column is +#: read verbatim as completed years observed at that person's own source +#: interview. No birth date is consulted, no one is aged forward, and no claim +#: is made that the person was observed in the target period. +SUPPORTED_AGE_CONVENTION = "observed_interview_age_completed_years" +if SUPPORTED_AGE_CONVENTION != NATIONAL_AGE_ACTIVATION.age_convention_id: + raise ValueError("The age-count operator and target activation must agree on age.") + +# Operational input envelope for this development connection, not an assertion +# that every source age within it is observed or valid. Outliers refuse rather +# than silently entering the open top band. +MAX_SUPPORTED_AGE = 120 + +_AGE_COLUMN = "age" +_PERSON = US_SCHEMA.person_entity +_HOUSEHOLD = "household" +_HOUSEHOLD_ID = US_SCHEMA.entity_id_column(_HOUSEHOLD) +_PERSON_HOUSEHOLD_ID = US_SCHEMA.membership_column(_HOUSEHOLD) + +#: Every person column the kernel reads: the owned age cell plus the structural +#: household membership. Declared normatively so the read surface is in the key. +_PERSON_COLUMNS = (_AGE_COLUMN, _PERSON_HOUSEHOLD_ID) +#: Every household column the kernel reads: the structural id, and nothing else. +_HOUSEHOLD_COLUMNS = (_HOUSEHOLD_ID,) + +_PARAMS = frozenset( + { + "bands", + "person_columns", + "household_columns", + "age_convention", + "schema_version", + "maximum_age", + } +) + + +def _canonical(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _bands_param(bands: tuple[AgeBand, ...]) -> str: + return _canonical([[band.column, band.low, band.high] for band in bands]) + + +def _bands_from_param(text: object) -> tuple[tuple[str, int, int | None], ...]: + """Parse and fully re-validate the declared band schema. + + The kernel trusts nothing it did not check here: the schema must be the + canonical form of a contiguous, disjoint, zero-based partition closed by + one open top band, with distinct column names. + """ + + if not isinstance(text, str) or len(text.encode()) > 65_536: + raise ValueError("A bounded canonical age-band declaration is required.") + rows = json.loads(text) + if not isinstance(rows, list) or not rows: + raise ValueError("The age-band declaration must be a nonempty list.") + bands: list[tuple[str, int, int | None]] = [] + expected_low = 0 + for index, row in enumerate(rows): + if ( + not isinstance(row, list) + or len(row) != 3 + or not isinstance(row[0], str) + or not row[0] + or type(row[1]) is not int + ): + raise ValueError("Each age band declares a column, a low and a high.") + column, low, high = row + last = index == len(rows) - 1 + if (high is None) is not last: + raise ValueError("Exactly the final age band is an open interval.") + if high is not None and (type(high) is not int or high < low): + raise ValueError("A closed age band needs an integer high at or above low.") + if low != expected_low: + raise ValueError("Age bands must be contiguous and start at zero.") + expected_low = None if high is None else high + 1 + bands.append((column, low, high)) + if len({column for column, _, _ in bands}) != len(bands): + raise ValueError("Age bands must own distinct count columns.") + if _canonical([[c, lo, hi] for c, lo, hi in bands]) != text: + raise ValueError("The age-band declaration must be in canonical form.") + return tuple(bands) + + +def _node_from_schema(schema: str, *, population: str, node_id: str) -> Node: + """The single declaration site, shared by the factory and the kernel.""" + + bands = _bands_from_param(schema) + return Node( + id=node_id, + kernel=NationalAgeCountKernel.ref, + population=population, + inputs=(Slice(_PERSON, (_AGE_COLUMN,)),), + outputs=tuple(Owned(_HOUSEHOLD, column, "int64") for column, _, _ in bands), + params={ + "bands": schema, + "person_columns": _PERSON_COLUMNS, + "household_columns": _HOUSEHOLD_COLUMNS, + "age_convention": SUPPORTED_AGE_CONVENTION, + "schema_version": AGE_COUNT_SCHEMA_VERSION, + "maximum_age": MAX_SUPPORTED_AGE, + }, + ) + + +def national_age_count_node( + *, + population: str, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + node_id: str = "national.age_counts", +) -> Node: + """Declare the household age-count operator over one population version. + + Args: + population: The population version whose rows the counts live in. + bands: The activated bands, in order. Their canonical form enters the + node key, so an altered schema is a different node. + node_id: The node's id. + + Returns: + The node owning one int64 count column per band. + """ + + return _node_from_schema( + _bands_param(bands), population=population, node_id=node_id + ) + + +class NationalAgeCountKernel(KernelBase): + """Count each person once into their household's declared age band.""" + + ref = "us.national_age_counts@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.NONE, + consumes_se=False, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + return source_hash( + sys.modules[__name__], dependencies=self.capabilities.dependencies + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + if set(params) != _PARAMS: + raise ValueError("The age-count node declares exactly its band schema.") + if params["age_convention"] != SUPPORTED_AGE_CONVENTION: + raise ValueError( + "This operator implements only " + f"{SUPPORTED_AGE_CONVENTION!r}; it cannot honour " + f"{params['age_convention']!r}." + ) + if params["schema_version"] != AGE_COUNT_SCHEMA_VERSION: + raise ValueError("Unsupported age-band schema version.") + if ( + params["person_columns"] != _PERSON_COLUMNS + or params["household_columns"] != _HOUSEHOLD_COLUMNS + ): + raise ValueError( + "This operator reads exactly " + f"{_PERSON_COLUMNS} on {_PERSON!r} and {_HOUSEHOLD_COLUMNS} on " + f"{_HOUSEHOLD!r}." + ) + bands = _bands_from_param(params["bands"]) + expected = _node_from_schema( + params["bands"], + population=context.node.population or "", + node_id=context.node.id, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Age-count node differs from its complete declaration.") + + household = context.tables[_HOUSEHOLD] + person = context.tables[_PERSON] + # The household view is exactly its id column: no weight, no source + # channel, no other attribute is visible to this operator. + if tuple(household.columns) != _HOUSEHOLD_COLUMNS: + raise ValueError( + f"The household view must carry exactly {_HOUSEHOLD_COLUMNS}, " + f"got {tuple(household.columns)}." + ) + missing = [column for column in _PERSON_COLUMNS if column not in person] + if missing: + raise ValueError(f"The person view is missing {missing}.") + household_ids = _ids(household, _HOUSEHOLD_ID, what="household id") + index = pd.Index(household_ids, name=_HOUSEHOLD_ID) + if index.has_duplicates: + raise ValueError("Household ids must be unique.") + # Align by id, never by row order: neither axis is assumed sequential, + # sorted, or shared between the two tables. + positions = index.get_indexer( + _ids(person, _PERSON_HOUSEHOLD_ID, what="household membership") + ) + if np.any(positions < 0): + missing = np.unique(person[_PERSON_HOUSEHOLD_ID].to_numpy()[positions < 0])[ + :5 + ] + raise ValueError( + "Every person's household membership must resolve to a household " + f"row; unresolved ids include {missing.tolist()}." + ) + + age = _ages(person) + band_of_person = np.full(len(person), -1, dtype=np.int64) + for ordinal, (_, low, high) in enumerate(bands): + inside = age >= low if high is None else (age >= low) & (age <= high) + if np.any(band_of_person[inside] >= 0): + raise ValueError("Declared age bands overlap.") + band_of_person[inside] = ordinal + if np.any(band_of_person < 0): + raise ValueError("Declared age bands do not cover every observed age.") + + counts = np.bincount( + band_of_person * len(index) + positions, + minlength=len(bands) * len(index), + ).astype(np.int64, copy=False) + counts = counts.reshape(len(bands), len(index)) + if int(counts.sum()) != len(person): + raise ValueError("Every person must be counted into exactly one band.") + people = counts.sum(axis=0) + if np.any(people == 0): + # The population contract requires every group row to be referenced + # by a person, so this is a broken population, not an empty house. + raise ValueError( + "Every household row must contain at least one person; " + f"{int((people == 0).sum())} household(s) contain none." + ) + return KernelResult( + columns={ + (_HOUSEHOLD, column): pd.Series( + counts[ordinal], index=index, dtype="int64" + ) + for ordinal, (column, _, _) in enumerate(bands) + }, + receipt={ + "scope": "national_age_counts_development", + "age_convention": SUPPORTED_AGE_CONVENTION, + "bands": params["bands"], + "people_counted": int(counts.sum()), + "households": int(len(index)), + "maximum_observed_age": int(age.max()) if len(age) else None, + "minimum_observed_age": int(age.min()) if len(age) else None, + "maximum_supported_age": MAX_SUPPORTED_AGE, + "consumes_weights": False, + "universe": "every person on the population, group quarters included", + "release_eligible": False, + }, + ) + + +def _ids(table: pd.DataFrame, column: str, *, what: str) -> np.ndarray: + series = table[column] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError(f"The {what} column must be an integer column.") + if series.isna().any(): + raise ValueError(f"The {what} column contains missing values.") + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values) & (values == np.floor(values))): + raise ValueError(f"The {what} column must hold whole numbers.") + # Preserve exact integer ids without a float round trip, and refuse casts + # that would wrap or saturate into another household's identity. The open + # float upper bound matters: float64 cannot represent int64's maximum. + if is_integer_dtype(series.dtype): + limits = np.iinfo(np.int64) + outside = ((series < limits.min) | (series > limits.max)).any() + else: + outside = np.any(values < -(2**63)) or np.any(values >= 2**63) + if outside: + raise ValueError(f"The {what} column must fit signed int64 exactly.") + return series.to_numpy(dtype="int64") + + +def _ages(person: pd.DataFrame) -> np.ndarray: + """Ages as nonnegative whole years; anything else refuses the node.""" + + series = person[_AGE_COLUMN] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError("Person age must be a real numeric column.") + if series.isna().any(): + raise ValueError( + f"{int(series.isna().sum())} person age(s) are unknown; an age-count " + "node refuses rather than dropping people." + ) + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values)): + raise ValueError("Person age must be finite.") + if np.any(values < 0): + raise ValueError("Person age must not be negative.") + if np.any(values > MAX_SUPPORTED_AGE): + raise ValueError( + f"Person age exceeds the declared maximum of {MAX_SUPPORTED_AGE}." + ) + if not np.all(values == np.floor(values)): + raise ValueError("Person age must be a whole number of completed years.") + return values + + +def national_age_calibration_nodes( + *, + base: str, + registry, + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + count_node_id: str = "national.age_counts", + calibration_node_id: str = "national.demographic_calibration", +) -> tuple[Node, Node]: + """Compose the age-count operator with the existing calibration node. + + The count node lives in ``base``'s population version and owns the columns; + the unchanged `demographic_calibration_node` reweights over ``base`` and + reads exactly those columns, so the count node is its predecessor. + + Args: + base: The population version carrying household importance weights. + registry: The activated :class:`TargetRegistry`. Its measures must be + exactly the declared band columns, in order — a registry and a band + schema that disagree are refused here rather than half-wired. + epochs: Solver epochs. + learning_rate: Solver learning rate. + max_weight_ratio: The executor's cap against original design weights. + max_initial_weight_ratio: The solver's cap against incoming weights. + bands: The activated bands, in order. + count_node_id: Id of the count node. + calibration_node_id: Id of the calibration node. + + Returns: + ``(count_node, calibration_node)`` in dependency order. + + Raises: + ValueError: If the registry's measures are not the band columns. + """ + + measures = tuple(spec.measure for spec in registry) + columns = tuple(band.column for band in bands) + if measures != columns: + raise ValueError( + "The registry's measures must be exactly the declared band columns " + f"in order; got {measures} against {columns}." + ) + return ( + national_age_count_node(population=base, bands=bands, node_id=count_node_id), + demographic_calibration_node( + registry, + base=base, + node_id=calibration_node_id, + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + ), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py new file mode 100644 index 000000000..3d52c78a7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py @@ -0,0 +1,415 @@ +"""Graph source producers and population-grain native household lineage. + +Only source kernels open original members. Consumers preserve the executor's +typed producer edges when decoding transport; a nominal artifact type by itself +is not source authentication. These nodes issue no benchmark approval or score. +""" + +from __future__ import annotations + +import json +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, +) +from microcosm.graph.artifact_edges import value_from_descriptor +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import native_household_origin as native +from .graph_native_origin_implementation import ( + DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +SOURCE_TYPE = ArtifactType("microcosm.us.native_household_origins", 1) +BINDING_TYPE = ArtifactType("microcosm.us.population_household_origins", 1) +SOURCE_OUTPUT = (ArtifactOutput("origins", SOURCE_TYPE),) +BINDING_OUTPUT = (ArtifactOutput("binding", BINDING_TYPE),) +PREFIX = "native_household_origins" +ACS_NODE = PREFIX + ".acs" +ASEC_NODE = PREFIX + ".asec" +ACS_SOURCE = "native_origin_acs" +ASEC_SOURCES = tuple(f"native_origin_asec_{year}" for year in (2022, 2023, 2024)) +SOURCES = ( + SourceRef(ACS_SOURCE, native.acs.ACS_HU_CODEC), + *(SourceRef(name, "raw-bytes-v1") for name in ASEC_SOURCES), +) + + +class _Kernel(KernelBase): + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=DEPENDENCIES, + ) + + def implementation_hash(self): + return implementation_hash() + + +class NativeACSOriginKernel(_Kernel): + """Authenticate original ACS archive members through the accepted codec.""" + + ref = "us.native_household_origin.acs@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == (ACS_SOURCE,) + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ACS_ORIGIN_DECLARATION", + ) + with tempfile.TemporaryDirectory(prefix="graph-native-acs-") as tmp: + root = Path(tmp).resolve() + snapshots = root / "snapshots" + snapshots.mkdir() + source = native.produce_acs_native_origins( + context.sources[ACS_SOURCE], + snapshot_root=snapshots, + output_dir=root / "projection", + ) + return _source_result(source) + + +class NativeASECOriginKernel(_Kernel): + """Authenticate the registered original HOUSEHOLD CSV projections.""" + + ref = "us.native_household_origin.asec@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == ASEC_SOURCES + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ASEC_ORIGIN_DECLARATION", + ) + source = native.produce_asec_native_origins( + { + year: context.sources[name] + for year, name in zip((2022, 2023, 2024), ASEC_SOURCES, strict=True) + } + ) + return _source_result(source) + + +def _source_result(source): + document = source.document + return KernelResult( + artifacts={"origins": source.payload}, + receipt={ + "phase": "native_household_origin_source", + "arm": document["arm"], + "households": len(document["records"]), + "source_projection_sha256": native._sha(source.payload), + "source_receipt_sha256": document["source_receipt_sha256"], + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def _artifact(context, name, expected_type, output): + value = context.artifacts[name] + native._require( + value.type == expected_type + and value.key == opaque_artifact_key(value.producer_key, output), + "ORIGIN_ARTIFACT_EDGE", + ) + return value + + +class PopulationOriginBindingKernel(_Kernel): + """Bind carried source evidence to every current row and clone membership.""" + + ref = "us.native_household_origin.bind@1" + + def run(self, context): + node = context.node + native._require( + not node.sources + and not node.outputs + and node.artifact_outputs == BINDING_OUTPUT + and set(context.params) == {"columns", "parent_binding"}, + "ORIGIN_BIND_DECLARATION", + ) + expected = { + entity: [US_SCHEMA.entity_id_column(entity)] + for entity in US_SCHEMA.entities + } + expected[US_SCHEMA.person_entity].extend( + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + ) + for entity, column, _dtype in context.params["columns"]: + expected[entity].append(column) + native._require( + set(context.tables) == set(expected) + and all( + set(context.tables[e]) == set(columns) + for e, columns in expected.items() + ), + "ORIGIN_BIND_COLUMN_INVENTORY", + ) + # Graph context supplies IDs and memberships implicitly. All remaining + # source cells must be declared by the integrating graph. The returned + # content digest must also be verified against the complete materialized + # population: this projected context cannot discover omitted columns. + tables = { + e: context.tables[e].loc[:, columns] for e, columns in expected.items() + } + frame = Frame(tables, US_SCHEMA, dict(context.weights), context.strata) + aliases = {"acs", "asec"} + if context.params["parent_binding"]: + aliases.add("parent_binding") + native._require( + set(context.artifacts) == aliases, "ORIGIN_BIND_ARTIFACT_ROSTER" + ) + values = [ + _artifact(context, arm, SOURCE_TYPE, "origins") for arm in ("acs", "asec") + ] + # Typed immutable transport was content-checked by the executor. This + # does not reopen sources or mint an independent source-authentication + # claim; the concrete producer keys are carried into the new artifact. + sources = [] + for arm, value in zip(("acs", "asec"), values, strict=True): + native._require( + native._source_document(value.payload)["arm"] == arm, + "ORIGIN_SOURCE_ARM", + ) + sources.append( + native.AuthenticatedNativeOriginSource( + value.payload, _token=native._TOKEN + ) + ) + parent = None + if "parent_binding" in aliases: + value = _artifact(context, "parent_binding", BINDING_TYPE, "binding") + doc = json.loads(value.payload) + native._require( + canonical_json(doc) == value.payload + and doc["schema"] == native.BINDING_SCHEMA, + "ORIGIN_PARENT_PAYLOAD", + ) + parent = native.PopulationOriginBinding( + value.payload, _token=native._BOUND_TOKEN + ) + result = native.bind_population_origins(frame, sources=sources, parent=parent) + document = result.document + document["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in context.artifacts.items() + } + document["population_node"] = node.population + payload = canonical_json(document) + return KernelResult( + artifacts={"binding": payload}, + receipt={ + "phase": "native_household_origin_binding", + "summary": result.summary, + "binding_sha256": native._sha(payload), + "parent_artifact_keys": { + alias: value.key for alias, value in context.artifacts.items() + }, + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def native_origin_source_nodes(*, population: str) -> tuple[Node, ...]: + """Declare source-only producers at an explicit graph population version. + + The graph requires the version to disambiguate structural branches. These + kernels declare no population columns and change no rows. + """ + return ( + Node( + ACS_NODE, + NativeACSOriginKernel.ref, + population=population, + sources=(ACS_SOURCE,), + artifact_outputs=SOURCE_OUTPUT, + ), + Node( + ASEC_NODE, + NativeASECOriginKernel.ref, + population=population, + sources=ASEC_SOURCES, + artifact_outputs=SOURCE_OUTPUT, + ), + ) + + +def native_origin_binding_node( + columns: Sequence[Owned], + *, + population: str, + node_id: str, + parent_binding_node: str | None = None, +) -> Node: + """Bind the exact declared full population; optionally extend native ancestry.""" + grouped = {entity: [] for entity in US_SCHEMA.entities} + for cell in columns: + grouped[cell.entity].append(cell.column) + native._require( + all(grouped.values()) and all(len(v) == len(set(v)) for v in grouped.values()), + "ORIGIN_COLUMN_DECLARATION", + ) + artifacts = [ + ArtifactInput("acs", ACS_NODE, "origins", SOURCE_TYPE), + ArtifactInput("asec", ASEC_NODE, "origins", SOURCE_TYPE), + ] + if parent_binding_node is not None: + artifacts.append( + ArtifactInput( + "parent_binding", parent_binding_node, "binding", BINDING_TYPE + ) + ) + return Node( + node_id, + PopulationOriginBindingKernel.ref, + population=population, + inputs=tuple( + Slice(entity, tuple(values)) for entity, values in grouped.items() + ), + artifact_inputs=tuple(artifacts), + artifact_outputs=BINDING_OUTPUT, + params={ + "columns": tuple((c.entity, c.column, c.dtype) for c in columns), + "parent_binding": parent_binding_node is not None, + }, + ) + + +def register_native_origin_kernels(registry: KernelRegistry) -> None: + """Register only the three new kernels, without editing upstream identities.""" + for kernel in ( + NativeACSOriginKernel(), + NativeASECOriginKernel(), + PopulationOriginBindingKernel(), + ): + registry.register(kernel) + + +def verify_materialized_population_origins( + manifest, + store, + *, + population_node: str, + binding_node: str, + parent_binding_node: str | None = None, +) -> native.PopulationOriginBinding: + """Verify graph evidence against its complete materialized population. + + This is a mandatory integration boundary: a kernel cannot know whether its + declared slices omitted another population column. The manifest must come + from the reviewed graph and store; matching local receipts do not constitute + an external producer authorization or approve any benchmark evaluation. + No original source is reopened, and no kernel is executed here. + """ + expected_implementation = implementation_hash() + + def read(node_id, output, expected_type, expected_kernel): + receipt = manifest.nodes[node_id] + descriptor = receipt.typed_artifacts["outputs"][output] + payload = store.load_bytes(receipt.opaque_artifacts[output]) + value = value_from_descriptor(payload, descriptor) + native._require( + receipt.kernel_ref == expected_kernel + and receipt.kernel_impl_hash == expected_implementation + and descriptor["producer"] == node_id + and descriptor["artifact"] == output + and value.producer_key == receipt.key + and value.key == receipt.opaque_artifacts[output] + and value.type == expected_type, + "ORIGIN_MATERIALIZED_PRODUCER", + ) + return value + + try: + values = { + arm: read(node_id, "origins", SOURCE_TYPE, kernel.ref) + for arm, node_id, kernel in ( + ("acs", ACS_NODE, NativeACSOriginKernel), + ("asec", ASEC_NODE, NativeASECOriginKernel), + ) + } + sources = [ + native.AuthenticatedNativeOriginSource(value.payload, _token=native._TOKEN) + for value in values.values() + ] + parent = None + if parent_binding_node is not None: + values["parent_binding"] = read( + parent_binding_node, + "binding", + BINDING_TYPE, + PopulationOriginBindingKernel.ref, + ) + parent = native.PopulationOriginBinding( + values["parent_binding"].payload, _token=native._BOUND_TOKEN + ) + # The native parent's own projection must also describe its actual + # complete population. This boundary supports the first full clone + # (or one native household selection), not an implicit ancestry DAG. + verify_materialized_population_origins( + manifest, + store, + population_node=parent.document["population_node"], + binding_node=parent_binding_node, + ) + artifact = read( + binding_node, "binding", BINDING_TYPE, PopulationOriginBindingKernel.ref + ) + actual = native.PopulationOriginBinding( + artifact.payload, _token=native._BOUND_TOKEN + ) + document = actual.document + frame = manifest.population(population_node) + native.verify_population_origin_binding(frame, actual) + expected = native.bind_population_origins( + frame, sources=sources, parent=parent + ).document + expected["population_node"] = population_node + expected["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in values.items() + } + native._require( + document == expected and canonical_json(document) == artifact.payload, + "ORIGIN_MATERIALIZED_LINEAGE", + ) + return actual + except native.NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise native.NativeOriginError("ORIGIN_MATERIALIZED_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py new file mode 100644 index 000000000..26977b24c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py @@ -0,0 +1,111 @@ +"""Closed additional identity for native-origin nodes; upstream bytes stay fixed. + +The shared v1 service and inventory are unchanged. Their complete composition +and ACS-source scopes are validated before adding this independently classified +inventory. A new origin helper or registry changes only these extension keys. +""" + +from __future__ import annotations + +import csv +import hashlib +import json + +from . import native_household_origin as native +from .graph_implementation import ( + STAGE_DEPENDENCIES, + _canonical, + _covered_imports, + _dependency_contract, + _package_roots, +) +from .graph_implementation import implementation_manifest as upstream_manifest + +STAGE = "native_household_origin_v1" +BASE_STAGES = ("composed_asec_binding_v1", "acs_housing_universe_2024") +DEPENDENCIES = STAGE_DEPENDENCIES[BASE_STAGES[0]] +INVENTORY_FILE = "us_runtime/native_origin_graph_inventory.json" +INVENTORY_SCHEMA = "microcosm.us.native-origin-extension-inventory.v1" +EXTRA_MODULES = ( + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py", +) + + +def implementation_manifest(): + """Verify all live classified dependencies without opening any survey file.""" + bases = {stage: upstream_manifest(stage) for stage in BASE_STAGES} + roots = _package_roots() + payload = (roots["microcosm.build"] / INVENTORY_FILE).read_bytes() + inventory = json.loads(payload) + native._require( + set(inventory) + == { + "schema", + "stage", + "base_stages", + "dependencies", + "extra_modules", + "contracts", + "import_classifications", + } + and inventory["schema"] == INVENTORY_SCHEMA + and inventory["stage"] == STAGE + and inventory["base_stages"] == list(BASE_STAGES) + and inventory["dependencies"] == list(DEPENDENCIES) + and inventory["extra_modules"] == list(EXTRA_MODULES) + and set(inventory["contracts"]) == set(EXTRA_MODULES), + "ORIGIN_IMPLEMENTATION_INVENTORY", + ) + imports = { + name + for contract in inventory["contracts"].values() + for name in contract["imports"] + } + native._require( + set(inventory["import_classifications"]) == imports + and all( + isinstance(label, str) and bool(label.strip()) + for label in inventory["import_classifications"].values() + ), + "ORIGIN_IMPORT_CLASSIFICATION", + ) + modules = set(EXTRA_MODULES) + for base in bases.values(): + modules.update(base["modules"]) + scope = { + "stages": { + STAGE: {"modules": sorted(modules), "dependencies": list(DEPENDENCIES)} + } + } + hashes = {} + for name in EXTRA_MODULES: + package, relative = name.split("/", 1) + code = (roots[package] / relative).read_bytes() + actual = _dependency_contract(code, name, _covered_imports(name, scope)) + native._require( + actual == inventory["contracts"][name], "ORIGIN_UNCLASSIFIED_DEPENDENCY" + ) + hashes[name] = hashlib.sha256(code).hexdigest() + return { + "schema": "microcosm.us.native-origin-extension-implementation.v1", + "stage": STAGE, + "inventory_sha256": hashlib.sha256(payload).hexdigest(), + "modules": hashes, + "upstream_implementations": bases, + "dependencies": dict(bases[BASE_STAGES[0]]["dependencies"]), + "acs_source_implementation": native.acs._implementation(), + "asec_household_member_registry": native.asec_member_registry(), + "csv_field_size_limit": csv.field_size_limit(), + } + + +def implementation_hash(): + """Identity of the additional implementation, not a replacement upstream key.""" + return hashlib.sha256( + b"microcosm.us.native-origin-extension-implementation.v1\0" + + _canonical(implementation_manifest()) + ).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py new file mode 100644 index 000000000..dfb87f1f1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_detail_transfer.py @@ -0,0 +1,780 @@ +"""A fixture-only price → donor → matrix → raw draw → masked placement graph. + +These kernels do not grant genuine source or fit admission. Existing source, +growth and targetwise QRF kernels perform all parsing, growth, fit and draw math. +""" + +from __future__ import annotations + +import sys +from collections.abc import Mapping +from importlib import import_module + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import graph_legacy_apply_matrix, model_input, qrf +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.keys import opaque_artifact_key + +from . import puf_detail_transfer as detail +from . import puf_growth as growth +from . import puf_growth_graph as growth_graph +from . import puf_monetary_agi_projection as agi +from . import puf_monetary_source as monetary +from . import puf_price_baseline as price +from . import puf_raw_source as raw +from . import support_provenance as provenance + +require = detail.require +PRICE_TYPE = ArtifactType("microcosm.us.fixture_puf_price_arrays", 1) +PRICE_BINDING_TYPE = ArtifactType("microcosm.us.fixture_puf_price_binding", 1) +HOST_TYPE = ArtifactType("microcosm.us.fixture_combined_host", 1) +PLACEMENT_TYPE = ArtifactType("microcosm.us.fixture_puf_detail_placement", 1) +SOURCE_EDGES = ( + ArtifactInput( + "status", raw.PUF_RAW_SOURCE_NODE, "return_status", raw.RETURN_STATUS_TYPE + ), + ArtifactInput( + "projection", + agi.AGI_PROJECTION_NODE, + "monetary_agi_projection", + agi.AGI_PROJECTION_TYPE, + ), +) + + +def plain(value): + if isinstance(value, Mapping): + return {k: plain(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [plain(v) for v in value] + return value + + +def artifact(context, alias, expected_type): + edges = [e for e in context.node.artifact_inputs if e.name == alias] + require(len(edges) == 1, "DETAIL_ARTIFACT_DECLARATION") + edge, value = edges[0], context.artifacts[alias] + require( + edge.type == value.type == expected_type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact), + "DETAIL_ARTIFACT_IDENTITY", + ) + return value + + +def siblings(context, names): + edges = {e.name: e for e in context.node.artifact_inputs} + require( + len({edges[name].producer for name in names}) == 1 + and len({context.artifacts[name].producer_key for name in names}) == 1, + "DETAIL_SIBLING_PRODUCER", + ) + + +class _Kernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self): + # Separate extension identity. Never edit the shared upstream inventory. + base = source_hash( + sys.modules[__name__], + detail, + raw, + agi, + monetary, + price, + growth, + growth_graph, + codec, + model_input, + graph_legacy_apply_matrix, + qrf, + provenance, + detail.origin, + detail.origin.acs, + *(import_module(name) for name in growth_graph._IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + return codec.sha( + codec.encode_json( + { + "modules": base, + "csv_profile": raw.csv_acceptance_profile(), + "source_definition": raw.packaged_definition().sha256, + "monetary_definition": monetary.packaged_projection().sha256, + "agi_definition": agi.packaged_agi_projection().sha256, + } + ) + ) + + +def fixture_inputs(context): + """Consume authenticated fixture source artifacts and the actual CPI codec bytes.""" + params = context.params + require(params["scope"] == detail.SCOPE, "GENUINE_TRANSFER_NOT_ADMITTED") + definition, projection = detail.fixture_sources( + params["definition"], params["projection"] + ) + status_value = artifact(context, "status", raw.RETURN_STATUS_TYPE) + projection_value = artifact(context, "projection", agi.AGI_PROJECTION_TYPE) + status = raw.decode_return_status(status_value.payload, definition) + decoded = agi.decode_agi_projection( + projection_value.payload, + projection, + expected_status_artifact_sha256=codec.sha(status_value.payload), + ) + cpi = context.sources["fixture_cpi"].read_bytes() + resource = codec.decode_json(params["resource"].encode()) + compiled = price.compile_price_baseline( + detail.MONEY_FIELDS, resource_bytes=cpi, **resource + ) + adaptation = price.adapt_projection_to_money_view( + decoded, status, compiled, resource_bytes=cpi + ) + require( + compiled.release_eligible is False + and adaptation.excluded["rows"] == 4 + and len(set(adaptation.excluded["recid"])) == 4 + and set(adaptation.excluded["recid"]) == set(raw.PUF_AGGREGATE_RECIDS), + "FIXTURE_PRICE_UNIVERSE", + ) + expected = { + "definition": definition.canonical, + "projection_definition": projection.canonical, + "status": status_value.payload, + "projection": projection_value.payload, + } + inputs = codec.decode_json(params["inputs"].encode()) + require(set(inputs) == set(expected), "FIXTURE_INPUT_ROSTER") + for name, body in expected.items(): + require( + inputs[name]["sha256"] == codec.sha(body) + and inputs[name]["bytes"] == len(body), + "FIXTURE_SOURCE_BINDING", + ) + return definition, projection, status, decoded, compiled, adaptation + + +class FixturePriceAdaptKernel(_Kernel): + ref = "us.fixture_puf_detail.price_adapt@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.CREATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context): + _, projection, _, _, compiled, adapted = fixture_inputs(context) + table = adapted.money_view.copy() + for name in compiled.money_fields: + require( + np.all(np.abs(table[name].to_numpy(dtype=object)) <= 2**53), + "FIXTURE_EXACT_MONEY_FLOAT", + ) + table[name] = table[name].astype("float64") + require(np.all(adapted.design_weight <= 2**53), "FIXTURE_EXACT_WEIGHT_FLOAT") + table["S006"] = adapted.design_weight.astype("float64") + table["FLPDYR"] = adapted.flpdyr.astype("int32") + table["demographic_status"] = adapted.demographic_status.astype("int32") + ids = np.arange(1, len(table) + 1, dtype="int64") + table.insert(0, "tax_unit_id", ids) + return KernelResult( + frame=Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": ids, + "person_tax_unit_id": ids, + "is_primary_filer": np.ones(len(ids), dtype=bool), + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + { + "tax_unit": Weights( + adapted.design_weight.astype("float64"), WeightKind.DESIGN + ) + }, + ), + receipt={ + "scope": detail.SCOPE, + "source_projection": projection.canonical.decode("utf-8"), + "source_admission": detail.SOURCE_ADMISSION, + "money_view": plain(adapted.receipt), + }, + ) + + +class FixturePriceExportKernel(_Kernel): + ref = "us.fixture_puf_detail.price_export@1" + + def run(self, context): + definition, projection, status, decoded, compiled, adapted = fixture_inputs( + context + ) + table = context.tables["tax_unit"] + arrays = { + name: table[name].to_numpy(copy=True) + for name in detail.ARRAY_DTYPES + if "_delivered_" not in name + } + arrays.update( + S006_delivered_int64=adapted.design_weight.copy(), + FLPDYR_delivered_int16=adapted.flpdyr.copy(), + demographic_status_delivered_int8=adapted.demographic_status.copy(), + ) + # Exact dea helper envelope; this fixture producer states its own actual + # code identities. It never impersonates a completed genuine price run. + header = { + "schema": "local.puf.price_restatement.arrays.v1", + "input_binding_sha256": codec.sha(context.params["inputs"].encode()), + "contract_sha256": compiled.sha256, + "source_head": context.params["code_head"], + "helper_head": context.params["code_head"], + "release_eligible": False, + } + identities = { + n: {"dtype": v.dtype.str, "rows": len(v), "sha256": codec.sha(v.tobytes())} + for n, v in sorted(arrays.items()) + } + encoded = codec.encode_json({**header, "columns": identities}) + payload = ( + detail.MAGIC + + len(encoded).to_bytes(8, "big") + + encoded + + b"".join( + np.ascontiguousarray(arrays[n]).tobytes() for n in sorted(arrays) + ) + ) + binding = { + "scope": detail.SCOPE, + "payload_sha256": codec.sha(payload), + "header": header, + "definition_sha256": definition.sha256, + "projection_sha256": projection.sha256, + "status_sha256": codec.sha(context.artifacts["status"].payload), + "source_projection_document": projection.canonical.decode("utf-8"), + "factor_document": compiled.factors.document.decode("utf-8"), + "recipe": compiled.recipe.identity.decode("utf-8"), + "resource": codec.decode_json(context.params["resource"].encode()), + } + # Exercise the import at production time too, without doing growth twice. + checked = detail.decode_price_arrays( + payload, expected=header, expected_sha256=codec.sha(payload) + ) + detail.donor_frame(checked, status, decoded, projection) + return KernelResult( + artifacts={"price": payload, "price_binding": codec.encode_json(binding)}, + receipt={ + "scope": detail.SCOPE, + "price_payload_sha256": codec.sha(payload), + "source_admission": detail.SOURCE_ADMISSION, + "release_eligible": False, + }, + ) + + +class FixtureDonorKernel(_Kernel): + ref = "us.fixture_puf_detail.donor@1" + capabilities = FixturePriceAdaptKernel.capabilities + + def run(self, context): + definition, projection, status, decoded, compiled, _ = fixture_inputs(context) + value = artifact(context, "price", PRICE_TYPE) + bound = artifact(context, "price_binding", PRICE_BINDING_TYPE) + siblings(context, ("price", "price_binding")) + binding = codec.decode_json(bound.payload) + require( + binding["scope"] == detail.SCOPE + and binding["definition_sha256"] == definition.sha256 + and binding["projection_sha256"] == projection.sha256 + and binding["status_sha256"] + == codec.sha(context.artifacts["status"].payload) + and binding["source_projection_document"] + == projection.canonical.decode("utf-8") + and binding["factor_document"] == compiled.factors.document.decode("utf-8") + and binding["recipe"] == compiled.recipe.identity.decode("utf-8") + and binding["resource"] + == codec.decode_json(context.params["resource"].encode()), + "DONOR_PRICE_BINDING", + ) + header = binding["header"] + require( + header["source_head"] + == header["helper_head"] + == context.params["code_head"] + and header["contract_sha256"] == compiled.sha256 + and header["input_binding_sha256"] + == codec.sha(context.params["inputs"].encode()), + "DONOR_PRICE_AUTHORITY", + ) + arrays = detail.decode_price_arrays( + value.payload, expected=header, expected_sha256=binding["payload_sha256"] + ) + frame = detail.donor_frame(arrays, status, decoded, projection) + return KernelResult( + frame=frame, + receipt={ + "scope": detail.SCOPE, + "source_admission": detail.SOURCE_ADMISSION, + "source_projection_document": projection.canonical.decode("utf-8"), + "donor_design_weight_units": "delivered_S006_integer_hundredths", + "release_eligible": False, + }, + ) + + +class DetailBoundaryKernel(_Kernel): + ref = "us.fixture_puf_detail.boundary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context): + return KernelResult( + keep=pd.Series(True, index=context.tables["person"].person_id, dtype=bool) + ) + + +def context_frame(context): + tables = { + e: context.tables[e].drop(columns=[detail.MASK, detail.OUTPUT], errors="ignore") + if e == "tax_unit" + else context.tables[e] + for e in US_SCHEMA.entities + } + return Frame(tables, US_SCHEMA, dict(context.weights), context.strata) + + +def verify_host_scope(context, frame): + value = artifact(context, "fixture_host", HOST_TYPE) + scope = codec.decode_json(value.payload) + require( + set(scope) == {"scope", "native_content_sha256", "source_fixture_sha256"} + and scope["scope"] == detail.SCOPE + and scope["source_fixture_sha256"] == context.params["source_fixture_sha256"], + "HOST_FIXTURE_AUTHORITY", + ) + native_tables = { + e: frame.table(e) + .loc[frame.table(e)[provenance.support_clone_index_column(e)].eq(0)] + .reset_index(drop=True) + for e in frame.entities + } + native_person = ( + frame.person[provenance.support_clone_index_column("person")].eq(0).to_numpy() + ) + native_hh = ( + frame.table("household")[provenance.support_clone_index_column("household")] + .eq(0) + .to_numpy() + ) + require( + frame.weights_for("household").kind is WeightKind.IMPORTANCE, "HOST_WEIGHT_KIND" + ) + native = Frame( + native_tables, + US_SCHEMA, + { + "household": Weights( + frame.weights_for("household").values[native_hh] * 2, + WeightKind.IMPORTANCE, + ) + }, + frame.strata.loc[native_person].reset_index(drop=True), + ) + require( + detail.population_content(native) == scope["native_content_sha256"], + "HOST_NATIVE_SOURCE_BINDING", + ) + return scope + + +class DetailMatrixKernel(_Kernel): + ref = "us.fixture_puf_detail.matrix@1" + scope = detail.SCOPE + host_binding_name = "fixture_host" + verify_host = staticmethod(verify_host_scope) + make_matrix = staticmethod(detail.recipient_matrix) + + def run(self, context): + require(context.params["scope"] == self.scope, "GENUINE_TRANSFER_NOT_ADMITTED") + frame = context_frame(context) + scope = self.verify_host(context, frame) + payload, mask = self.make_matrix(frame) + binding = { + "scope": self.scope, + "population": context.node.population, + "host_content_sha256": detail.population_content(frame), + self.host_binding_name: scope, + "matrix_sha256": codec.sha(payload), + "features": list(detail.FEATURES), + "targets": [detail.TARGET], + "output": detail.OUTPUT, + "selected_ids": frame.table("tax_unit") + .tax_unit_id.to_numpy()[mask] + .tolist(), + } + return KernelResult( + columns={ + ("tax_unit", detail.MASK): pd.Series( + mask, index=frame.table("tax_unit").tax_unit_id, dtype=bool + ) + }, + artifacts={"matrix": payload, "placement": codec.encode_json(binding)}, + receipt={ + "scope": self.scope, + "recipient_rows": int(mask.sum()), + "zero_weight_rows_retained": True, + "source_admission": detail.SOURCE_ADMISSION, + "release_eligible": False, + }, + ) + + +class DetailAttachKernel(_Kernel): + ref = "us.fixture_puf_detail.attach@1" + scope = detail.SCOPE + host_binding_name = "fixture_host" + placement_type = PLACEMENT_TYPE + verify_host = staticmethod(verify_host_scope) + make_matrix = staticmethod(detail.recipient_matrix) + + def run(self, context): + require( + context.params["scope"] == self.scope + and context.node.outputs + == (Owned("tax_unit", detail.OUTPUT, "float64", rows=detail.MASK),), + "DETAIL_OWNERSHIP", + ) + frame = context_frame(context) + scope = self.verify_host(context, frame) + matrix = artifact(context, "matrix", model_input.RECIPIENT_MATRIX_TYPE) + placement = artifact(context, "placement", self.placement_type) + raw_draw = artifact(context, "raw_draw", codec.RAW_TARGET_TYPE) + state = artifact(context, "apply_state", MATRIX_APPLY_STATE_TYPE) + siblings(context, ("matrix", "placement")) + siblings(context, ("raw_draw", "apply_state")) + binding = codec.decode_json(placement.payload) + expected, mask = self.make_matrix(frame) + require( + np.array_equal(context.tables["tax_unit"][detail.MASK], mask), + "DETAIL_MASK_CHANGED", + ) + require( + matrix.payload == expected + and binding + == { + "scope": self.scope, + "population": context.node.population, + "host_content_sha256": detail.population_content(frame), + self.host_binding_name: scope, + "matrix_sha256": codec.sha(expected), + "features": list(detail.FEATURES), + "targets": [detail.TARGET], + "output": detail.OUTPUT, + "selected_ids": frame.table("tax_unit") + .tax_unit_id.to_numpy()[mask] + .tolist(), + }, + "DETAIL_PLACEMENT_BINDING", + ) + packet = decode_matrix_apply_state(state.payload) + require( + packet["matrix_sha256"] == codec.sha(matrix.payload) + and packet["matrix_producer_key"] == matrix.producer_key, + "DETAIL_MATRIX_PRODUCER", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + tuple(chain.completed_targets) == (detail.TARGET,) + and tuple(chain.predictors) == detail.FEATURES + and application["seed"] == 578 + and application["raw_targets"] + == [{"target": detail.TARGET, "sha256": codec.sha(raw_draw.payload)}], + "DETAIL_RAW_STATE_BINDING", + ) + prepared = model_input.decode_recipient_matrix(matrix.payload) + require( + chain.entity == prepared.entity == "tax_unit" + and tuple(chain.targets) == (detail.TARGET,) + and chain.recipient_index == qrf._index_identity(prepared.features.index), + "DETAIL_STATE_ENTITY_INDEX", + ) + values = codec.read_raw_target( + raw_draw.payload, target=detail.TARGET, index=prepared.features.index + ) + return KernelResult( + columns={ + ("tax_unit", detail.OUTPUT): pd.Series( + values, index=prepared.entity_ids, dtype="float64" + ) + }, + receipt={ + "scope": self.scope, + "raw_sha256": codec.sha(raw_draw.payload), + "matrix_producer_key": matrix.producer_key, + "application_producer_key": state.producer_key, + "native_output_cells_written": 0, + "host_weights_changed": False, + "release_eligible": False, + }, + ) + + +def host_nodes( + columns, + *, + base, + host_producer, + source_fixture_sha256, + fit_nodes, + prefix="fixture_puf_detail", +): + """Declare the clone-only matrix and exact raw attachment after whole-household cloning.""" + from microcosm.fit.graph_legacy_qrf import legacy_qrf_apply_matrix_nodes + + grouped = { + e: tuple(o.column for o in columns if o.entity == e) for e in US_SCHEMA.entities + } + inputs = tuple(Slice(e, values) for e, values in grouped.items()) + boundary = prefix + ".population" + matrix_id = prefix + ".matrix" + host_edge = ArtifactInput("fixture_host", host_producer, "fixture_host", HOST_TYPE) + params = {"scope": detail.SCOPE, "source_fixture_sha256": source_fixture_sha256} + matrix = Node( + matrix_id, + DetailMatrixKernel.ref, + population=boundary, + inputs=inputs, + outputs=(Owned("tax_unit", detail.MASK, "bool"),), + params=params, + artifact_inputs=(host_edge,), + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ArtifactOutput("placement", PLACEMENT_TYPE), + ), + ) + applies = legacy_qrf_apply_matrix_nodes( + prefix + ".apply", + population=boundary, + fit_nodes=fit_nodes, + matrix_producer=matrix_id, + seed=578, + phase=detail.SCOPE, + ) + attach_inputs = tuple( + Slice(e, (*values, detail.MASK) if e == "tax_unit" else values) + for e, values in grouped.items() + ) + attach = Node( + prefix + ".attach", + DetailAttachKernel.ref, + population=boundary, + inputs=attach_inputs, + params=params, + outputs=(Owned("tax_unit", detail.OUTPUT, "float64", rows=detail.MASK),), + artifact_inputs=( + host_edge, + ArtifactInput( + "matrix", matrix_id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ArtifactInput("placement", matrix_id, "placement", PLACEMENT_TYPE), + ArtifactInput( + "raw_draw", applies[-1].id, "raw_draw", codec.RAW_TARGET_TYPE + ), + ArtifactInput( + "apply_state", applies[-1].id, "apply_state", MATRIX_APPLY_STATE_TYPE + ), + ), + ) + return ( + Node( + boundary, + DetailBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", (provenance.support_source_id_column("person"),)),), + ), + matrix, + *applies, + attach, + ) + + +def verify_materialized_transfer( + before, + after, + binding_payload, + *, + before_design, + after_design, + before_ledger, + after_ledger, + boundary_node, +): + """Detect omitted projected columns and any non-owned materialized change.""" + binding = codec.decode_json(binding_payload) + require( + detail.population_content(before) == binding["host_content_sha256"], + "MATERIALIZED_HOST_BINDING", + ) + tables = { + e: after.table(e).drop(columns=[detail.MASK, detail.OUTPUT], errors="ignore") + if e == "tax_unit" + else after.table(e) + for e in after.entities + } + restored = Frame( + tables, + after.schema, + {e: after.weights_for(e) for e in after.weighted_entities}, + after.strata, + ) + require( + detail.population_content(restored) == detail.population_content(before), + "MATERIALIZED_NONOWNED_CHANGED", + ) + require(before.mass_log == after.mass_log, "MATERIALIZED_FRAME_LEDGER_CHANGED") + require(before_design.keys() == after_design.keys(), "MATERIALIZED_DESIGN_ROSTER") + for entity in before_design: + require( + np.array_equal( + before_design[entity].view("uint64"), + after_design[entity].view("uint64"), + ), + "MATERIALIZED_DESIGN_CHANGED", + ) + require( + bool(before_ledger) + and tuple(after_ledger) == tuple(before_ledger) + and after_ledger[-1].node_id == boundary_node, + "MATERIALIZED_GRAPH_LEDGER_CHANGED", + ) + native = after.table("tax_unit")[ + provenance.support_clone_index_column("tax_unit") + ].eq(0) + require( + after.table("tax_unit").loc[native, detail.OUTPUT].isna().all(), + "MATERIALIZED_NATIVE_OUTPUT", + ) + return { + "all_nonowned_preserved": True, + "native_output_null": True, + "design_anchors_preserved": True, + "design_anchor_entities": len(before_design), + "graph_ledger_preserved": True, + "scope": binding["scope"], + } + + +def price_nodes(params, compiled): + """Real growth and strict donor import, independent of any survey host.""" + inputs = ("fixture_cpi",) + adapted = "fixture_puf_price.adapted" + export = "fixture_puf_price.export" + grown = growth_graph.us_puf_growth_nodes( + compiled, + base=adapted, + entity="tax_unit", + person_boundary_columns=("is_primary_filer",), + ) + money_columns = ( + "RECID", + *detail.MONEY_FIELDS, + "S006", + "FLPDYR", + "demographic_status", + ) + adapted_outputs = ( + Owned("person", "is_primary_filer", "bool"), + *( + Owned( + "tax_unit", + name, + "int64" + if name == "RECID" + else "int32" + if name in ("FLPDYR", "demographic_status") + else "float64", + ) + for name in money_columns + ), + ) + export_columns = tuple(n for n in detail.ARRAY_DTYPES if "_delivered_" not in n) + donor_outputs = tuple( + Owned("tax_unit", name, "float64") for name in (*detail.FEATURES, detail.TARGET) + ) + return ( + Node( + adapted, + FixturePriceAdaptKernel.ref, + structural=StructuralDelta.CREATE, + sources=inputs, + params=params, + artifact_inputs=SOURCE_EDGES, + outputs=adapted_outputs, + ), + *grown, + Node( + export, + FixturePriceExportKernel.ref, + population=growth_graph.PUF_GROWTH_BOUNDARY_NODE, + inputs=(Slice("tax_unit", export_columns),), + sources=inputs, + params=params, + artifact_inputs=SOURCE_EDGES, + artifact_outputs=( + ArtifactOutput("price", PRICE_TYPE), + ArtifactOutput("price_binding", PRICE_BINDING_TYPE), + ), + ), + Node( + "fixture_puf_donor", + FixtureDonorKernel.ref, + structural=StructuralDelta.CREATE, + outputs=donor_outputs, + sources=inputs, + params=params, + artifact_inputs=( + *SOURCE_EDGES, + ArtifactInput("price", export, "price", PRICE_TYPE), + ArtifactInput( + "price_binding", export, "price_binding", PRICE_BINDING_TYPE + ), + ), + ), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py new file mode 100644 index 000000000..d3c62c6ae --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf_diagnostic_consumer.py @@ -0,0 +1,763 @@ +"""Source-qualified development diagnostic, preserving the fixture-only path. + +The runner must first perform the bounded reader/feature check before allowing +fit. Graph fit keys deliberately have no host ancestry: changing recipient +features can reuse the same donor model. No source metadata is promoted. +""" + +from __future__ import annotations + +import json +import sys + +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, + StructuralDelta, + source_hash, +) + +from . import graph_asec_income as income +from . import graph_combined_clone as survey_clone +from . import graph_composed_contracts as composed_contracts +from . import graph_context +from . import graph_native_household_origin as origins +from . import graph_puf_detail_transfer as shared +from . import graph_survey_population as survey_graph +from . import puf_diagnostic_consumer as consumer +from . import survey_origin_budget as survey_budget +from . import survey_population_preparation as survey_source +from . import survey_population_replay as survey_replay +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, +) + +detail, codec, require = shared.detail, shared.codec, shared.require +TRANSPORT_TYPE = ArtifactType("microcosm.us.puf_diagnostic_transport", 1) +PLACEMENT_TYPE = ArtifactType("microcosm.us.puf_diagnostic_placement", 1) +PREFIX = "puf_diagnostic" +SOURCE_NAMES = {n: PREFIX + "_" + n for n in consumer.ROLES} +SOURCES = tuple(SourceRef(n, "raw-bytes-v1") for n in SOURCE_NAMES.values()) + + +class _Kernel(shared._Kernel): + def implementation_hash(self): + # Only the legacy diagnostic attests the composed measurement owner. + from . import graph_composed_asec_measures as measures + + return codec.sha( + codec.encode_json( + { + "consumer": source_hash( + sys.modules[__name__], + consumer, + composed_contracts, + dependencies=self.capabilities.dependencies, + ), + "shared": shared._Kernel().implementation_hash(), + "asec": measures.USComposedAsecReportedIncomeKernel().implementation_hash(), + "native_origins": origins.PopulationOriginBindingKernel().implementation_hash(), + } + ) + ) + + +class DiagnosticDonorKernel(_Kernel): + ref = "us.puf_diagnostic.donor@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.CREATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context): + require( + set(context.params) == {"expected_pins", "route", "recipe"} + and not context.node.inputs + and not context.artifacts + and set(context.sources) == set(SOURCE_NAMES.values()), + "DIAGNOSTIC_DONOR_DECLARATION", + ) + pins = codec.decode_json(context.params["expected_pins"].encode()) + require(set(pins) == set(consumer.ROLES), "PRICE_TRANSPORT_ROSTER") + payloads = {} + for role, name in SOURCE_NAMES.items(): + size = pins[role]["bytes"] + require( + type(size) is int and 0 < size <= detail.MAX_BYTES, + "PRICE_TRANSPORT_SIZE", + ) + with context.sources[name].open("rb") as handle: + payloads[role] = handle.read(size + 1) + consumer.check_pin( + payloads[role], pins[role], "PRICE_TRANSPORT_PIN:" + role + ) + frame, payload = consumer.qualify_price_transport( + payloads, + pins, + route=context.params["route"], + recipe=codec.decode_json(context.params["recipe"].encode()), + ) + return KernelResult( + frame=frame, + artifacts={"transport": payload}, + receipt={ + "scope": consumer.SCOPE, + "route": context.params["route"], + "recipe": consumer.recipe_document(), + "transport_sha256": codec.sha(payload), + "original_source_fit_admission": detail.SOURCE_ADMISSION, + "release_eligible": False, + }, + ) + + +def host_edges(*, native_binding, clone_binding): + return ( + ArtifactInput("acs", origins.ACS_NODE, "origins", origins.SOURCE_TYPE), + ArtifactInput("asec", origins.ASEC_NODE, "origins", origins.SOURCE_TYPE), + ArtifactInput( + "parent_binding", native_binding, "binding", origins.BINDING_TYPE + ), + ArtifactInput("clone_binding", clone_binding, "binding", origins.BINDING_TYPE), + ArtifactInput( + "asec_binding", + composed_contracts.BIND_NODE, + "asec_binding", + composed_contracts.US_COMPOSED_ASEC_BINDING_TYPE, + ), + ArtifactInput( + "arm_rows", + composed_contracts.BIND_NODE, + "arm_rows", + composed_contracts.US_COMPOSED_ASEC_ARM_ROWS_TYPE, + ), + ArtifactInput( + "selected_current_money", + composed_contracts.BIND_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + ArtifactInput( + "prepared_receipt", + composed_contracts.CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ), + ArtifactInput( + "income_observations", + composed_contracts.CREATE_NODE, + "income_observations", + income.US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ArtifactInput( + "frame_context", + composed_contracts.LEAVES_NODE, + "frame_context", + graph_context.US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput( + "reported_income", + composed_contracts.REPORTED_INCOME_NODE, + "reported_income", + income.US_ASEC_REPORTED_INCOME_TYPE, + ), + ) + + +def verify_host(context, frame): + """Typed actual producer edges plus externally pinned accepted host evidence.""" + require( + context.params["scope"] == consumer.SCOPE + and context.params["recipe"].encode() + == codec.encode_json(consumer.recipe_document()) + and context.params["route"] in ("packaged", "test_fixture"), + "DIAGNOSTIC_HOST_RECIPE", + ) + expected = codec.decode_json(context.params["host_edges"].encode()) + types = { + e.name: e.type + for e in host_edges(native_binding="native", clone_binding="clones") + } + edges = {e.name: e for e in context.node.artifact_inputs if e.name in expected} + require(set(expected) == set(edges) == set(types), "DIAGNOSTIC_HOST_EDGE_ROSTER") + values = {} + for name in edges: + value = shared.artifact(context, name, types[name]) + require( + expected[name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + }, + "DIAGNOSTIC_HOST_EDGE_PIN:" + name, + ) + values[name] = value + shared.siblings(context, ("asec_binding", "arm_rows", "selected_current_money")) + shared.siblings(context, ("prepared_receipt", "income_observations")) + evidence = consumer.qualify_host_population(frame, values) + evidence["route"] = context.params["route"] + evidence["actual_producer_edges"] = expected + return evidence + + +class DiagnosticMatrixKernel(shared.DetailMatrixKernel, _Kernel): + ref = "us.puf_diagnostic.matrix@1" + scope = consumer.SCOPE + host_binding_name = "source_qualified_host" + verify_host = staticmethod(verify_host) + make_matrix = staticmethod(consumer.recipient_matrix) + implementation_hash = _Kernel.implementation_hash + + +class DiagnosticAttachKernel(shared.DetailAttachKernel, _Kernel): + ref = "us.puf_diagnostic.attach@1" + scope = consumer.SCOPE + host_binding_name = "source_qualified_host" + placement_type = PLACEMENT_TYPE + verify_host = staticmethod(verify_host) + make_matrix = staticmethod(consumer.recipient_matrix) + implementation_hash = _Kernel.implementation_hash + + +def diagnostic_nodes( + columns, *, base, native_binding, clone_binding, price_pins, host_pins, route +): + """Extend the complete bound clone pool with one raw diagnostic target.""" + require(route in ("packaged", "test_fixture"), "DIAGNOSTIC_SOURCE_ROUTE") + donor = PREFIX + ".donor" + boundary = PREFIX + ".population" + matrix = PREFIX + ".matrix" + recipe = consumer.recipe_document() + fits = legacy_qrf_train_nodes( + PREFIX + ".fit", + population=donor, + entity="tax_unit", + predictors=detail.FEATURES, + targets=(detail.TARGET,), + seed=578, + n_estimators=2, + zero_atol=0.0, + phase=consumer.SCOPE, + ) + applies = legacy_qrf_apply_matrix_nodes( + PREFIX + ".apply", + population=boundary, + fit_nodes=fits, + matrix_producer=matrix, + seed=578, + phase=consumer.SCOPE, + ) + grouped = { + e: tuple(o.column for o in columns if o.entity == e) for e in US_SCHEMA.entities + } + inputs = tuple(Slice(e, names) for e, names in grouped.items()) + edges = host_edges(native_binding=native_binding, clone_binding=clone_binding) + params = { + "scope": consumer.SCOPE, + "route": route, + "recipe": codec.encode_json(recipe).decode(), + "host_edges": codec.encode_json(host_pins).decode(), + } + return ( + Node( + donor, + DiagnosticDonorKernel.ref, + structural=StructuralDelta.CREATE, + sources=tuple(SOURCE_NAMES.values()), + outputs=tuple( + Owned("tax_unit", n, "float64") + for n in (*detail.FEATURES, detail.TARGET) + ), + params={ + "route": route, + "recipe": codec.encode_json(recipe).decode(), + "expected_pins": codec.encode_json(price_pins).decode(), + }, + artifact_outputs=(ArtifactOutput("transport", TRANSPORT_TYPE),), + ), + Node( + boundary, + shared.DetailBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=( + Slice( + "person", (shared.provenance.support_source_id_column("person"),) + ), + ), + ), + Node( + matrix, + DiagnosticMatrixKernel.ref, + population=boundary, + inputs=inputs, + outputs=(Owned("tax_unit", detail.MASK, "bool"),), + params=params, + artifact_inputs=edges, + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ArtifactOutput("placement", PLACEMENT_TYPE), + ), + ), + *fits, + *applies, + Node( + PREFIX + ".attach", + DiagnosticAttachKernel.ref, + population=boundary, + inputs=tuple( + Slice(e, (*names, detail.MASK) if e == "tax_unit" else names) + for e, names in grouped.items() + ), + outputs=(Owned("tax_unit", detail.OUTPUT, "float64", rows=detail.MASK),), + params=params, + artifact_inputs=( + *edges, + ArtifactInput( + "matrix", matrix, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ArtifactInput("placement", matrix, "placement", PLACEMENT_TYPE), + ArtifactInput( + "raw_draw", applies[-1].id, "raw_draw", codec.RAW_TARGET_TYPE + ), + ArtifactInput( + "apply_state", + applies[-1].id, + "apply_state", + shared.MATRIX_APPLY_STATE_TYPE, + ), + ), + ), + ) + + +def register_diagnostic_kernels(registry): + for kernel in ( + DiagnosticDonorKernel(), + DiagnosticMatrixKernel(), + DiagnosticAttachKernel(), + ): + registry.register(kernel) + + +# This host-only route neither declares nor admits a donor, fit, draw or attach. +# It extends the maintained four-node current survey prefix, not the legacy host. +CURRENT_PROJECTION_TYPE = ArtifactType("microcosm.us.survey_puf_host_projection", 1) +CURRENT_PROJECTION_NODE = "puf_diagnostic.current_survey_host" +CURRENT_MATRIX_NODE = "puf_diagnostic.current_survey_matrix" + + +def qualify_current_survey_host(preparation, allocated_population, clone_population): + """Check one live current prefix and project its wages once. + + Returning bytes is transport only. Call verify_materialized_current_survey_host + with actual executor/store outputs on cold and replay boundaries. No budget, + successor, source capsule or legacy artifact is issued here. + """ + require( + type(preparation) is survey_source.AuthenticatedSurveyPopulationPreparation, + "SURVEY_HOST_PREPARATION_TYPE", + ) + entry = preparation._checked() # One full source preparation check. + state = entry[2] + view = survey_source.CheckedSurveyPopulationView( + entry[1], state.context, state.frame, state.plan, json.loads(entry[1]) + ) + # The actual maintained owner checks allocation, all clone cells, membership, + # pair weights, full receiving state, design anchors and mass ledgers. + _, allocation = survey_budget._initial(view, allocated_population, clone_population) + identities = tuple( + survey_budget._population_identity(p) + for p in (allocated_population, clone_population) + ) + asec = survey_source._current_survey_wage_projection(preparation, entry) + document = { + "protocol": consumer.CURRENT_SURVEY_PROJECTION_PROTOCOL, + "route": consumer.CURRENT_SURVEY_ROUTE, + "preparation_sha256": codec.sha(entry[1]), + "allocation_sha256": codec.sha(allocation), + "clone_sha256": survey_source._frame_identity(clone_population.frame), + "asec": json.loads(asec), + "release_eligible": False, + } + projection = survey_graph._bounded_json(document, consumer.CURRENT_SURVEY_MAX_BYTES) + matrix, mask, evidence = consumer.current_survey_recipient_matrix( + clone_population.frame, projection + ) + # No I/O after the projection; compare complete retained receiving state and + # actual source issuance after readiness and feature calculation. + require( + identities + == tuple( + survey_budget._population_identity(p) + for p in (allocated_population, clone_population) + ), + "SURVEY_HOST_FINAL_POPULATION", + ) + survey_budget._preparation_entry(preparation, entry[1], entry) + return projection, matrix, mask, evidence + + +def current_survey_host_edges(): + return ( + ArtifactInput( + "preparation", + survey_graph.CREATE_NODE, + "preparation", + survey_graph.PREPARATION_TYPE, + ), + ArtifactInput( + "allocation", + survey_graph.ALLOCATION_NODE, + "allocation", + survey_graph.ALLOCATION_TYPE, + ), + ArtifactInput( + "frame_context", + survey_graph.ALLOCATION_NODE, + "frame_context", + graph_context.US_FRAME_CONTEXT_TYPE, + ), + ) + + +def _current_params(document): + return { + "host_route": consumer.CURRENT_SURVEY_ROUTE, + **{ + key: document[key] + for key in ("preparation_sha256", "allocation_sha256", "clone_sha256") + }, + } + + +def _current_context_frame(context, expected): + # The executor puts structural columns before the declared data columns. + # Validate the full unmasked declaration from the retained Frame first; + # actual context columns must never choose their own comparison projection. + ordered = {} + inputs = [] + for entity in US_SCHEMA.entities: + structural = [US_SCHEMA.entity_id_column(entity)] + if entity == US_SCHEMA.person_entity: + structural.extend( + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + ) + columns = tuple( + c for c in expected.table(entity).columns if c not in structural + ) + inputs.append(Slice(entity, columns)) + ordered[entity] = [*structural, *columns] + require(context.node.inputs == tuple(inputs), "SURVEY_HOST_INPUTS") + require(set(context.tables) == set(US_SCHEMA.entities), "SURVEY_HOST_TABLES") + + # Match the executor's effective-weight roster, including inherited weights. + # Ambiguous inherited group weights are omitted by the executor; an invalid + # explicitly stored weight is an error rather than an absent context entry. + weights = {} + for entity in US_SCHEMA.entities: + try: + weights[entity] = expected.resolve_weights(entity) + except ValueError: + if entity in expected.weighted_entities: + raise + require(set(context.weights) == set(weights), "SURVEY_HOST_WEIGHTS") + require( + type(context.strata) is type(expected.strata) + and type(context.strata.name) is type(expected.strata.name) + and context.strata.name == expected.strata.name, + "SURVEY_HOST_STRATA_NAME", + ) + # Use the same all-row boolean selection so index class/name/dtype survive + # exactly as they do in a KernelContext. Frame normalizes the strata name, + # which was checked above before either comparison Frame is constructed. + tables = { + entity: expected.table(entity).loc[ + [True] * len(expected.table(entity)), ordered[entity] + ] + for entity in US_SCHEMA.entities + } + strata = expected.strata.loc[[True] * len(expected.strata)] + actual = Frame( + dict(context.tables), US_SCHEMA, dict(context.weights), context.strata + ) + projected = Frame(tables, US_SCHEMA, weights, strata) + survey_replay.same_replayed_frame(projected, actual) + # KernelContext has no Population metadata. The mandatory materialized + # verifier separately checks metadata, ledgers, owners and design anchors. + return actual + + +class _CurrentSurveyKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=( + *survey_graph.STAGE_DEPENDENCIES, + "scikit-learn", + "quantile-forest", + ), + ) + + def implementation_hash(self): + # No PUF raw/price definitions or acceptance locators are consulted. + # The reviewed survey inventory remains an independent fail-closed gate. + return codec.sha( + codec.encode_json( + { + "survey": survey_graph._Kernel().implementation_hash(), + "host": source_hash( + sys.modules[__name__], + consumer, + composed_contracts, + shared, + shared.qrf, + detail, + codec, + model_input, + survey_budget, + survey_replay, + consumer.current_money, + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + +class CurrentSurveyHostProjectionKernel(_CurrentSurveyKernel): + ref = "us.puf_diagnostic.current_survey_host@1" + + def __init__(self, preparation, allocated_population, clone_population): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + + def run(self, context): + require( + context.node.id == CURRENT_PROJECTION_NODE + and context.node.population == survey_clone.COMBINED_CLONE_NODE + and context.node.artifact_inputs == current_survey_host_edges() + and context.node.artifact_outputs + == (ArtifactOutput("projection", CURRENT_PROJECTION_TYPE),) + and not context.node.outputs + and not context.sources + and set(context.artifacts) + == {"preparation", "allocation", "frame_context"}, + "SURVEY_HOST_DECLARATION", + ) + projection, _, _, evidence = qualify_current_survey_host( + self.preparation, self.allocated_population, self.clone_population + ) + document = codec.decode_json(projection) + require( + set(context.params) == set(_current_params(document)) | {"host_edges"}, + "SURVEY_HOST_PARAMS", + ) + require( + {k: context.params[k] for k in _current_params(document)} + == _current_params(document), + "SURVEY_HOST_PARAMS", + ) + pins_bytes = context.params["host_edges"].encode() + require(len(pins_bytes) <= 4096, "SURVEY_HOST_EDGE_PIN_BOUND") + pins = codec.decode_json(pins_bytes) + require( + set(pins) == {e.name for e in current_survey_host_edges()}, + "SURVEY_HOST_EDGE_PIN_ROSTER", + ) + _current_context_frame(context, self.clone_population.frame) + prepared = shared.artifact( + context, "preparation", survey_graph.PREPARATION_TYPE + ) + allocated = shared.artifact(context, "allocation", survey_graph.ALLOCATION_TYPE) + frame_context = shared.artifact( + context, "frame_context", graph_context.US_FRAME_CONTEXT_TYPE + ) + shared.siblings(context, ("allocation", "frame_context")) + for edge, value in zip( + current_survey_host_edges(), + (prepared, allocated, frame_context), + strict=True, + ): + require( + pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + }, + "SURVEY_HOST_EDGE_PIN", + ) + require( + prepared.payload == self.preparation.payload + and codec.sha(allocated.payload) == document["allocation_sha256"] + and codec.sha(frame_context.payload) + == codec.decode_json(allocated.payload)["output_context_sha256"], + "SURVEY_HOST_EDGE_BYTES", + ) + return KernelResult(artifacts={"projection": projection}, receipt=evidence) + + +class CurrentSurveyMatrixKernel(_CurrentSurveyKernel): + ref = "us.puf_diagnostic.current_survey_matrix@1" + + def run(self, context): + require( + context.node.id == CURRENT_MATRIX_NODE + and context.node.population == survey_clone.COMBINED_CLONE_NODE + and context.node.artifact_inputs + == ( + ArtifactInput( + "projection", + CURRENT_PROJECTION_NODE, + "projection", + CURRENT_PROJECTION_TYPE, + ), + ) + and context.node.artifact_outputs + == (ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),) + and not context.node.outputs + and not context.sources + and set(context.artifacts) == {"projection"}, + "SURVEY_MATRIX_DECLARATION", + ) + projection = shared.artifact( + context, "projection", CURRENT_PROJECTION_TYPE + ).payload + require( + type(projection) is bytes + and len(projection) <= consumer.CURRENT_SURVEY_MAX_BYTES, + "SURVEY_HOST_PROJECTION_BOUND", + ) + require( + dict(context.params) == _current_params(codec.decode_json(projection)), + "SURVEY_MATRIX_PARAMS", + ) + # Do not strip any carried cells; attachment/mask columns are not produced. + frame = Frame( + dict(context.tables), US_SCHEMA, dict(context.weights), context.strata + ) + matrix, _, evidence = consumer.current_survey_recipient_matrix( + frame, projection + ) + return KernelResult(artifacts={"matrix": matrix}, receipt=evidence) + + +def current_survey_host_nodes( + columns, *, preparation_sha256, allocation_sha256, clone_sha256, host_pins +): + """Two artifact-only nodes over the existing pre-calibration clone version. + + Digest arguments name values; they do not authenticate them. This deliberately + cannot be substituted into the eleven-edge legacy donor/attachment factory. + """ + document = dict( + preparation_sha256=preparation_sha256, + allocation_sha256=allocation_sha256, + clone_sha256=clone_sha256, + ) + require(all(codec._hash(v) for v in document.values()), "SURVEY_HOST_DIGEST") + require(columns and all(type(o) is Owned for o in columns), "SURVEY_HOST_COLUMNS") + require( + len({(o.entity, o.column) for o in columns}) == len(columns), + "SURVEY_HOST_COLUMNS", + ) + inputs = tuple( + Slice(e, tuple(o.column for o in columns if o.entity == e)) + for e in US_SCHEMA.entities + ) + require(all(item.columns for item in inputs), "SURVEY_HOST_COLUMNS") + params = _current_params(document) + require( + set(host_pins) == {e.name for e in current_survey_host_edges()}, + "SURVEY_HOST_EDGE_PIN_ROSTER", + ) + for pin in host_pins.values(): + require( + set(pin) == {"producer_key", "artifact_key", "payload_sha256"} + and all(codec._hash(value) for value in pin.values()), + "SURVEY_HOST_EDGE_PIN", + ) + encoded_pins = codec.encode_json(host_pins) + require(len(encoded_pins) <= 4096, "SURVEY_HOST_EDGE_PIN_BOUND") + return ( + Node( + CURRENT_PROJECTION_NODE, + CurrentSurveyHostProjectionKernel.ref, + population=survey_clone.COMBINED_CLONE_NODE, + inputs=inputs, + params={**params, "host_edges": encoded_pins.decode()}, + artifact_inputs=current_survey_host_edges(), + artifact_outputs=(ArtifactOutput("projection", CURRENT_PROJECTION_TYPE),), + ), + Node( + CURRENT_MATRIX_NODE, + CurrentSurveyMatrixKernel.ref, + population=survey_clone.COMBINED_CLONE_NODE, + inputs=inputs, + params=params, + artifact_inputs=( + ArtifactInput( + "projection", + CURRENT_PROJECTION_NODE, + "projection", + CURRENT_PROJECTION_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ), + ) + + +def verify_materialized_current_survey_host( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, +): + """Mandatory cold/replay check of actual outputs, not manifest receipt labels. + + The integrating runner first checks typed descriptors and actual content-store + keys, then supplies their loaded bytes and the executor-observed Population. + This verifier reconstructs live source projection once and compares the full + materialized Population after that last source I/O. + """ + expected_projection, expected_matrix, _, evidence = qualify_current_survey_host( + preparation, allocated_population, clone_population + ) + require( + type(projection) is bytes and projection == expected_projection, + "SURVEY_HOST_MATERIALIZED_PROJECTION", + ) + require( + type(matrix) is bytes and matrix == expected_matrix, + "SURVEY_HOST_MATERIALIZED_MATRIX", + ) + survey_replay.same_replayed_population(clone_population, population) + return evidence diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py new file mode 100644 index 000000000..dcacf9dcb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_sources.py @@ -0,0 +1,555 @@ +"""Real, declared US source construction for the population graph. + +The ACS source is a directory containing exactly the two Census archives. Its +content key binds both archives without absolute paths entering the node key. +The ASEC source is the authenticated operator-free raw-stage checkpoint. A +candidate pool or an enriched legacy checkpoint cannot substitute for it. The +prepared current-money source is a directory of the reviewed restoration +inputs; its loader is declared here and executed only by its CREATE kernel. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from importlib import import_module +from pathlib import Path +from threading import RLock + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, EntitySchema, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_raw_bytes +from microcosm.graph.population import dtype_for_token, token_for_dtype + +from .acs_housing_universe_source import ACS_HU_CODEC, load_graph_acs_housing_universe +from .acs_inputs import map_acs_native_inputs +from .acs_pums import AcsPumsSource, build_acs_pums_unit_frame +from .asec_checkpoint import load_asec_raw_stage_checkpoint_v4 +from .asec_prepared_source import load_graph_asec_prepared +from .graph_context import ( + US_FRAME_CONTEXT_TYPE, + _decode, + _json_data, + _normative_metadata, + _row_identity, + encode_us_frame_context, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, + validate_source_codecs, +) +from .operator_boundary import assert_operator_free_source_frame +from .support_provenance import support_channel_column + +# Preserve every historical star export, including public imported bindings. +# Star import explicitly requests all three lazy operations. Ordinary source +# imports leave them unresolved. +__all__ = [ + "annotations", + "Sequence", + "asdict", + "dataclass", + "Path", + "np", + "pd", + "canonicalize_frame_string_dtypes", + "US_SCHEMA", + "EntitySchema", + "Frame", + "ArtifactInput", + "ArtifactOutput", + "ArtifactType", + "Capabilities", + "Determinism", + "Graph", + "KernelBase", + "KernelContext", + "KernelRegistry", + "KernelResult", + "Node", + "Numeric", + "Owned", + "SeedSource", + "Slice", + "SourceRef", + "StructuralDelta", + "WeightTransition", + "canonical_json", + "SourceCodecRegistry", + "load_raw_bytes", + "dtype_for_token", + "token_for_dtype", + "ACS_HU_CODEC", + "load_graph_acs_housing_universe", + "map_acs_native_inputs", + "AcsPumsSource", + "build_acs_pums_unit_frame", + "load_asec_raw_stage_checkpoint_v4", + "load_graph_asec_prepared", + "US_FRAME_CONTEXT_TYPE", + "encode_us_frame_context", + "STAGE_DEPENDENCIES", + "implementation_hash", + "implementation_manifest", + "validate_source_codecs", + "assert_operator_free_source_frame", + "assemble_stacked_spine", # noqa: F822 - lazy __getattr__ export + "harmonize_stacked_spine_weights", # noqa: F822 - lazy __getattr__ export + "prepare_stacked_spine", # noqa: F822 - lazy __getattr__ export + "support_channel_column", + "ASEC_CODEC", + "ACS_CODEC", + "ASEC_PREPARED_CODEC", + "ASSEMBLY_PHASE", + "US_STACK_PREPARATION_TYPE", + "US_SOURCE_DEPENDENCIES", + "GraphAssemblyResult", + "load_graph_asec", + "load_graph_acs", + "us_source_codecs", + "assembly_from_sources", + "frame_column_declarations", + "USAssemblyCreateKernel", + "USSpineHarmonizeKernel", + "us_assembly_graph", + "us_assembly_registry", + "import_module", + "RLock", +] + +_STACKED_ALIASES = frozenset( + { + "assemble_stacked_spine", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + } +) +_STACKED_LOCK = RLock() + + +def _stacked_alias(name: str): + if name in globals(): + return globals()[name] + value = getattr(import_module("microcosm.build.us_runtime.stacked_spine"), name) + with _STACKED_LOCK: + return globals().setdefault(name, value) + + +def __getattr__(name: str): + if name in _STACKED_ALIASES: + return _stacked_alias(name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(set(globals()) | _STACKED_ALIASES) + + +ASEC_CODEC = "us-asec-raw-stage-v4" +ACS_CODEC = "us-acs-native-2024-v1" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASSEMBLY_PHASE = "assemble_stacked_spine" +US_STACK_PREPARATION_TYPE = ArtifactType("microcosm.us.stacked_spine_preparation", 1) +US_SOURCE_DEPENDENCIES = STAGE_DEPENDENCIES["assembly_prepare"] + + +@dataclass(frozen=True) +class GraphAssemblyResult: + frame: Frame + receipt: object + dtype_transitions: tuple[dict[str, str], ...] + + +def _canonical_assembly(frame: Frame, sources: tuple[Frame, ...]): + """Represent source-declared integer/bool/string absence with nullable storage. + + Legacy union assembly fills absent non-float columns with object None. + The graph has no arbitrary-object dtype. Promotion is determined by the + actual source dtype, never by guessing from the assembled observations. + Values and the missing mask must be identical after the explicit boundary. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if not pd.api.types.is_object_dtype(original.dtype): + continue + dtypes = [ + source.table(entity)[column].dtype + for source in sources + if column in source.table(entity) + ] + if not dtypes: + continue # graph-added provenance strings are handled below + if all(pd.api.types.is_bool_dtype(dtype) for dtype in dtypes): + target = "boolean" + elif all(pd.api.types.is_integer_dtype(dtype) for dtype in dtypes): + target = "Int64" + elif all(isinstance(dtype, pd.StringDtype) for dtype in dtypes): + target = "string" + else: + continue + promoted = original.astype(dtype_for_token(target)) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph dtype promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph dtype promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + {"entity": entity, "column": column, "from": "object", "to": target} + ) + canonicalize_frame_string_dtypes(frame, boundary="US graph assembly", in_place=True) + # Build checkpoint strings use NumPy NaN; the graph column codec requires + # nullable pandas strings. This is an explicit physical storage boundary. + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph string promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph string promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": "string[python,nan]", + "to": "string[python,pd.NA]", + } + ) + return tuple(transitions) + + +def load_graph_asec(path: Path, *, store=None) -> Frame: + return _decode_graph_asec(path, store=store) + + +def _decode_graph_asec(path: Path, *, store=None) -> Frame: + del store + frame, _binding = load_asec_raw_stage_checkpoint_v4(path) + return frame + + +def load_graph_acs(path: Path, *, store=None) -> Frame: + return _decode_graph_acs(path, store=store) + + +def _decode_graph_acs(path: Path, *, store=None) -> Frame: + del store + expected = {"csv_hus.zip", "csv_pus.zip"} + if not path.is_dir() or {entry.name for entry in path.iterdir()} != expected: + raise ValueError("US ACS graph source must contain exactly the two archives.") + if any(not (path / name).is_file() for name in expected): + raise ValueError("US ACS graph source archives must be regular files.") + source = AcsPumsSource(path / "csv_hus.zip", path / "csv_pus.zip", vintage=2024) + raw, _receipt = build_acs_pums_unit_frame(source) + mapped = map_acs_native_inputs(raw) + assert_operator_free_source_frame( + mapped.frame, label="US graph ACS source", native_inputs=mapped.native_inputs + ) + return mapped.frame + + +def us_source_codecs() -> SourceCodecRegistry: + codecs = SourceCodecRegistry() + codecs.register(ASEC_CODEC, load_graph_asec) + codecs.register(ACS_CODEC, load_graph_acs) + # The prepared current-money directory source. Its loader lives with the + # preparation it names, so registering it here adds no import cycle and no + # second preparation path. + codecs.register(ASEC_PREPARED_CODEC, load_graph_asec_prepared) + codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + codecs.register(ACS_HU_CODEC, load_graph_acs_housing_universe) + validate_source_codecs(codecs) + return codecs + + +def assembly_from_sources( + asec_path: Path, acs_path: Path, *, sample_fraction: float, sample_seed: int +): + """The direct-call parity oracle and the CREATE kernel share this operation.""" + asec = load_graph_asec(asec_path) + acs = load_graph_acs(acs_path) + result = _stacked_alias("assemble_stacked_spine")( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + return GraphAssemblyResult(result.frame, result.receipt, transitions) + + +def frame_column_declarations(frame: Frame) -> tuple[Owned, ...]: + """Inventory data cells; the graph owns IDs and membership implicitly.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph columns require the US schema.") + return tuple( + Owned(entity, column, token_for_dtype(table[column].dtype)) + for entity in US_SCHEMA.entities + for table in (frame.table(entity),) + for column in table.columns + if column != US_SCHEMA.entity_id_column(entity) + and not ( + entity == US_SCHEMA.person_entity + and column + in { + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + } + ) + ) + + +def _source_implementation_hash() -> str: + """Compatibility spelling for the new preparation identity only.""" + # This runs before cache lookup, including resume=require. A renamed or + # substituted public loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash("assembly_prepare") + + +class USAssemblyCreateKernel(KernelBase): + ref = "us.production.spine_prepare@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=US_SOURCE_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return _source_implementation_hash() + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"phase", "sample_fraction", "sample_seed"}: + raise ValueError("US assembly parameters have an unsupported contract.") + if context.params["phase"] != ASSEMBLY_PHASE: + raise ValueError("US assembly has an incorrect outer phase.") + asec = load_graph_asec(context.sources["asec_raw_stage"]) + acs = load_graph_acs(context.sources["acs_native"]) + result = _stacked_alias("prepare_stacked_spine")( + asec, + acs, + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + if frame_column_declarations(result.frame) != context.node.outputs: + raise ValueError( + "US assembly source columns differ from the declared inventory." + ) + return KernelResult( + frame=result.frame, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_prepare"), + "preparation": result.receipt, + "dtype_transitions": transitions, + }, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.receipt)), + }, + ) + + +class USSpineHarmonizeKernel(KernelBase): + ref = "us.production.spine_harmonize@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES["assembly_harmonize"], + ) + + def implementation_hash(self) -> str: + return implementation_hash("assembly_harmonize") + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": ASSEMBLY_PHASE}: + raise ValueError( + "US spine harmonization has an unsupported phase contract." + ) + bound = context.artifacts["frame_context"] + prepared = context.artifacts["preparation"] + if ( + bound.type != US_FRAME_CONTEXT_TYPE + or prepared.type != US_STACK_PREPARATION_TYPE + or bound.producer_key != prepared.producer_key + ): + raise ValueError( + "US spine preparation/context must share a typed producer." + ) + document = _decode(bound.payload) + if document["weight_sources"] != {"household": "design"}: + raise ValueError( + "US spine harmonization requires its explicit DESIGN anchor." + ) + for entity in ("person", "household"): + if any( + document["entities"][entity][key] != value + for key, value in _row_identity(context.tables[entity], entity).items() + ): + raise ValueError(f"US spine {entity} context identity differs.") + # This is canonical, producer-owned JSON, not a self-authored declaration. + import json + + preparation = json.loads(prepared.payload) + if canonical_json(preparation) != prepared.payload: + raise ValueError("US spine preparation must be canonical JSON.") + if preparation["assembly_metadata"] != document["metadata"]: + raise ValueError("US spine preparation and context metadata differ.") + result = _stacked_alias("harmonize_stacked_spine_weights")( + household=context.tables["household"], + weights=context.weights["household"], + preparation=preparation, + ) + # The graph's declared mass is person mass by source stratum. Compute + # it with the same Frame reducer from minimal declared membership views. + tables = { + "person": context.tables["person"][["person_id", "person_household_id"]], + "household": context.tables["household"][["household_id"]], + } + schema = EntitySchema(group_entities=("household",)) + before = Frame( + tables, schema, {"household": context.weights["household"]}, context.strata + ) + after = Frame(tables, schema, {"household": result.weights}, context.strata) + before_mass, after_mass = before.stratum_mass(), after.stratum_mass() + document["metadata"] = _normative_metadata(result.metadata) + document["mass_log"] = [ + _json_data(asdict(item)) for item in result.legacy_mass_log + ] + document["weight_sources"] = {"household": "importance"} + return KernelResult( + weights=result.weights, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_harmonize"), + "assembly": result.receipt, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": before_mass.to_dict(), + "stratum_after": after_mass.to_dict(), + }, + }, + ) + + +def us_assembly_graph( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> Graph: + """Construct only the assembly development graph, never a release pool.""" + return Graph( + country="us", + sources=( + SourceRef( + "asec_raw_stage", ASEC_CODEC, "Operator-free ASEC raw-stage checkpoint." + ), + SourceRef( + "acs_native", ACS_CODEC, "2024 ACS one-year national PUMS archives." + ), + ), + nodes=( + Node( + id=f"{ASSEMBLY_PHASE}.prepare", + kernel=USAssemblyCreateKernel.ref, + outputs=tuple(columns), + structural=StructuralDelta.CREATE, + sources=("asec_raw_stage", "acs_native"), + params={ + "phase": ASSEMBLY_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ), + ), + Node( + id=ASSEMBLY_PHASE, + kernel=USSpineHarmonizeKernel.ref, + base=f"{ASSEMBLY_PHASE}.prepare", + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", + f"{ASSEMBLY_PHASE}.prepare", + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput( + "preparation", + f"{ASSEMBLY_PHASE}.prepare", + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ), + ), + ) + + +def us_assembly_registry() -> KernelRegistry: + kernels = KernelRegistry() + kernels.register(USAssemblyCreateKernel()) + kernels.register(USSpineHarmonizeKernel()) + return kernels diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py new file mode 100644 index 000000000..b59f3f4df --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py @@ -0,0 +1,221 @@ +"""Unweighted age counts as an ordered artifact, without population columns. + +The actual NationalAgeCountKernel performs the measurement. Its household +roster is the sorted unique membership of the declared person projection; +Frame requires every household to be referenced. Downstream admission must +also compare the artifact's ordered IDs with the complete receiving population. +This artifact is a measurement, not source authority or target activation. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + KernelBase, + KernelResult, + source_hash, +) + +from . import graph_national_age_counts as ages +from . import national_age_activation as activation + +COUNTS_TYPE = ArtifactType("microcosm.us.survey_household_age_counts", 1) +MAGIC = b"MCUSAGE1\n" +MAX_BYTES = 64 * 1024**2 +MAX_HEADER_BYTES = 65_536 +MAX_PEOPLE = 10_000_000 +_COLUMNS = tuple(b.column for b in activation.NATIONAL_AGE_ACTIVATION.bands) +_WIDTH = 1 + len(_COLUMNS) +_FIELDS = frozenset( + {"protocol", "population", "columns", "rows", "people", "age_convention"} +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_ARTIFACT_" + reason) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _shape(rows, people): + _require(type(rows) is int and type(people) is int, "INTEGER_SHAPE") + _require(0 < rows <= people <= MAX_PEOPLE, "SHAPE") + _require( + len(MAGIC) + 4 + MAX_HEADER_BYTES + rows * _WIDTH * 8 <= MAX_BYTES, "LIMIT" + ) + + +def _validate(header: dict, values: np.ndarray): + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require( + header["protocol"] == "microcosm.us.survey-household-age-counts.v1", "PROTOCOL" + ) + _require( + type(header["population"]) is str and 0 < len(header["population"]) <= 256, + "POPULATION", + ) + _require(header["columns"] == list(_COLUMNS), "COLUMNS") + _require(header["age_convention"] == ages.SUPPORTED_AGE_CONVENTION, "CONVENTION") + _shape(header["rows"], header["people"]) + _require( + type(values) is np.ndarray + and values.dtype == np.dtype(" ids[:-1]), "ORDERED_IDS") + _require(np.all(counts >= 0) and np.all(counts <= header["people"]), "COUNTS") + _require(np.all(np.any(counts > 0, axis=1)), "EMPTY_HOUSEHOLD") + # The shape and per-cell limits above bound this sum well below int64 max. + _require(int(counts.sum(dtype=np.int64)) == header["people"], "PEOPLE_TOTAL") + + +@dataclass(frozen=True) +class SurveyAgeCountValues: + """Decoded values only. Typed ancestry and current-population checks are separate.""" + + population: str + household_ids: np.ndarray + counts: pd.DataFrame + people: int + + +def decode_survey_age_counts(payload: bytes) -> SurveyAgeCountValues: + _require( + type(payload) is bytes and len(MAGIC) + 4 < len(payload) <= MAX_BYTES, "PAYLOAD" + ) + _require(payload.startswith(MAGIC), "MAGIC") + offset = len(MAGIC) + length = int.from_bytes(payload[offset : offset + 4], "big") + _require( + 0 < length <= MAX_HEADER_BYTES and offset + 4 + length <= len(payload), + "HEADER_LIMIT", + ) + offset += 4 + raw = payload[offset : offset + length] + try: + header = json.loads(raw) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_AGE_ARTIFACT_HEADER_JSON") from None + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require(_json(header) == raw, "CANONICAL_HEADER") + _shape(header["rows"], header["people"]) + offset += length + _require(len(payload) - offset == header["rows"] * _WIDTH * 8, "BYTE_COUNT") + values = np.frombuffer(payload, dtype=" bytes: + """Project numbers from bounded bytes; this pure operation authenticates nothing.""" + document = _document(payload) + ids = document.get("household_ids") + groups = document.get("group_indices") + records = document.get("origins") + _require( + type(ids) is list + and 0 < len(ids) <= numeric.MAX_ROWS + and type(groups) is list + and len(groups) == len(ids) + and type(records) is list + and 0 < len(records) <= budgets.MAX_GROUPS + and type(document.get("group_count")) is int + and document["group_count"] == len(records) + and len(ids) == 2 * len(records), + "COMPLETE_CLONE_SHAPE", + ) + _require( + all(type(i) is int and -(2**63) <= i < 2**63 for i in ids) + and len(set(ids)) == len(ids), + "HOUSEHOLD_IDS", + ) + positions = {i: position for position, i in enumerate(ids)} + incoming, row_upper = [None] * len(ids), [None] * len(ids) + upper, seen = [], set() + for group, record in enumerate(records): + _require(type(record) is dict, "ORIGIN_RECORD") + # Validate before duplicating per-origin tokens into per-row output. + numeric._vector([record.get("design_bound_float64_hex")], 1) + numeric._vector([record.get("upper_float64_hex")], 1) + members, weights = ( + record.get("members"), + record.get("incoming_clone_float64_bytes"), + ) + _require( + type(members) is list + and len(members) == 2 + and type(weights) is list + and len(weights) == 2, + "COMPLETE_ROLES", + ) + roles = set() + for member, weight_bytes in zip(members, weights, strict=True): + _require( + type(member) is list + and len(member) == 2 + and type(member[0]) is int + and member[0] in positions + and member[0] not in seen + and type(member[1]) is int + and member[1] in (0, 1), + "MEMBERSHIP", + ) + seen.add(member[0]) + roles.add(member[1]) + position = positions[member[0]] + _require( + type(groups[position]) is int and groups[position] == group, + "GROUP_ORDER", + ) + _require( + type(weight_bytes) is str + and len(weight_bytes) == 16 + and all(c in "0123456789abcdef" for c in weight_bytes), + "WEIGHT_BYTES", + ) + value = np.frombuffer(bytes.fromhex(weight_bytes), dtype=np.float64)[0] + incoming[position] = float(value).hex() + row_upper[position] = record.get("design_bound_float64_hex") + _require(roles == {0, 1}, "COMPLETE_ROLES") + upper.append(record.get("upper_float64_hex")) + _require(seen == set(ids), "COMPLETE_MEMBERSHIP") + output = graph._bounded_json( + { + "protocol": numeric.BOUNDS_PROTOCOL, + "budget_sha256": _sha(payload), + "population": clone.COMBINED_CLONE_NODE, + "household_ids": ids, + "group_indices": groups, + "group_upper_hex": upper, + "row_upper_hex": row_upper, + "incoming_hex": incoming, + }, + numeric.MAX_BYTES, + ) + _require(len(output) <= numeric.MAX_BYTES, "NUMERIC_LIMIT") + numeric.decode_numeric_survey_bounds(output) + return output + + +def survey_sampling_budget_node(*, budget_sha256): + _require(numeric._digest(budget_sha256), "BUDGET_DIGEST") + return Node( + BUDGET_NODE, + SurveySamplingBudgetKernel.ref, + population=clone.COMBINED_CLONE_NODE, + # This provenance input forces the actual ownership claim to precede + # the budget transport. It is not an imputation feature. + inputs=(Slice("household", (support_clone_index_column("household"),)),), + params={"budget_sha256": budget_sha256, "authority": "country_runner_required"}, + artifact_inputs=( + ArtifactInput( + "preparation", graph.CREATE_NODE, "preparation", graph.PREPARATION_TYPE + ), + ArtifactInput( + "allocation", graph.ALLOCATION_NODE, "allocation", graph.ALLOCATION_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("budget", budgets.BUDGET_TYPE), + ArtifactOutput("numeric_bounds", numeric.BOUNDS_TYPE), + ), + ) + + +class SurveySamplingBudgetKernel(KernelBase): + """Carry immutable caller values with checked source and clone projections.""" + + ref = "us.survey_sampling_budget@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, payload): + _document(payload) + self._payload = payload + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + budgets, + numeric, + numeric.group_bounds, + support_clone_index_column, + graph, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + payload = self._payload + document = _document(payload) + expected = survey_sampling_budget_node(budget_sha256=_sha(payload)) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + set(context.tables) == set(context.weights) == {"household"} + and not context.sources + and set(context.artifacts) == {"preparation", "allocation"}, + "CONTEXT", + ) + for edge in expected.artifact_inputs: + value = context.artifacts[edge.name] + _require( + value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact) + and _sha(value.payload) == document[edge.name + "_sha256"], + "SOURCE_EDGE", + ) + numbers_raw = numeric_survey_budget_payload(payload) + numbers = numeric.decode_numeric_survey_bounds(numbers_raw) + household = context.tables["household"] + role_column = support_clone_index_column("household") + _require(set(household.columns) == {"household_id", role_column}, "PROJECTION") + _require( + tuple(household.household_id) == numbers.grouped.household_ids, + "ORDERED_IDS", + ) + roles = { + member[0]: member[1] + for record in document["origins"] + for member in record["members"] + } + _require( + tuple(household[role_column]) + == tuple(roles[i] for i in household.household_id), + "CLONE_ROLES", + ) + weights = context.weights["household"] + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.dtype == numbers.incoming.dtype + and weights.values.tobytes() == numbers.incoming.tobytes(), + "ACTUAL_INCOMING_WEIGHTS", + ) + result = KernelResult( + artifacts={"budget": payload, "numeric_bounds": numbers_raw}, + receipt={ + "budget_sha256": _sha(payload), + "numeric_bounds_sha256": _sha(numbers_raw), + "constraint_digest": numbers.grouped.digest, + "group_count": numbers.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + _require( + type(self._payload) is bytes and self._payload == payload, "FINAL_PAYLOAD" + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py new file mode 100644 index 000000000..b3d839f8c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py @@ -0,0 +1,417 @@ +"""Source-blind age calibration over typed numeric budget and count artifacts. + +These kernels do not authenticate a survey. The country runner must reconstruct +the source-qualified budget and independently admit the complete population on +cold execution, cached restoration and final return. Frozen incoming weights +are numeric artifact values, never replacement original DESIGN anchors. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np + +from microcosm.calibrate import calibrate, diagnostics_payload, group_bounds +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import demographic_calibration_graph as demographic +from . import graph_survey_age_artifact as ages +from . import survey_age_activation as age_activation + +BOUNDS_TYPE = ArtifactType("microcosm.us.survey_calibration_numeric_bounds", 1) +BOUNDS_PROTOCOL = "microcosm.us.survey-calibration-numeric-bounds.v1" +MAX_BYTES = 64 * 1024**2 +MAX_ROWS = MAX_BYTES // 128 +CALIBRATION_NODE = "survey.age_calibration" +_FIELDS = frozenset( + { + "protocol", + "budget_sha256", + "population", + "household_ids", + "group_indices", + "group_upper_hex", + "row_upper_hex", + "incoming_hex", + } +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_CALIBRATION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def _vector(raw, length): + _require(type(raw) is list and len(raw) == length, "VECTOR_LENGTH") + _require(all(type(v) is str and 0 < len(v) <= 32 for v in raw), "FLOAT_TOKEN") + try: + values = np.array([float.fromhex(v) for v in raw], dtype=np.float64) + except (ValueError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_FLOAT_TOKEN") from None + _require( + np.all(np.isfinite(values) & (values >= 0)) + and all(float(v).hex() == text for v, text in zip(values, raw, strict=True)), + "FLOAT_VALUES", + ) + return values + + +@dataclass(frozen=True) +class NumericSurveyBounds: + """Decoded numbers only; neither a source certificate nor a live budget.""" + + budget_sha256: str + population: str + grouped: group_bounds.GroupedUpperBounds + incoming: np.ndarray + row_upper: np.ndarray + + +def decode_numeric_survey_bounds(payload: bytes) -> NumericSurveyBounds: + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD") + try: + document = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_CALIBRATION_JSON") from None + _require(type(document) is dict and set(document) == _FIELDS, "FIELDS") + _require(canonical_json(document) == payload, "CANONICAL") + _require(document["protocol"] == BOUNDS_PROTOCOL, "PROTOCOL") + _require(_digest(document["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(document["population"]) is str and 0 < len(document["population"]) <= 256, + "POPULATION", + ) + ids, indices = document["household_ids"], document["group_indices"] + _require(type(ids) is list and 0 < len(ids) <= MAX_ROWS, "ROW_COUNT") + _require( + all(type(v) is int and -(2**63) <= v < 2**63 for v in ids) + and all(a < b for a, b in zip(ids, ids[1:], strict=False)), + "ORDERED_IDS", + ) + _require(type(indices) is list and len(indices) == len(ids), "GROUP_INDICES") + raw_upper = document["group_upper_hex"] + _require(type(raw_upper) is list and 0 < len(raw_upper) <= len(ids), "GROUP_COUNT") + _require( + all(type(v) is int and 0 <= v < len(raw_upper) for v in indices), + "GROUP_INDICES", + ) + upper = _vector(raw_upper, len(raw_upper)) + incoming = _vector(document["incoming_hex"], len(ids)) + row_upper = _vector(document["row_upper_hex"], len(ids)) + grouped = group_bounds.GroupedUpperBounds(ids, indices, upper) + grouped.check(incoming, positive=False) + _require(np.all(incoming <= row_upper), "INITIAL_ROW_CAP") + _require(np.any(incoming > 0), "EMPTY_POSITIVE_SUPPORT") + return NumericSurveyBounds( + document["budget_sha256"], document["population"], grouped, incoming, row_upper + ) + + +def check_numeric_survey_weights(bounds, values): + """Independent numerical checks, including strict retained zero support.""" + _require(type(bounds) is NumericSurveyBounds, "BOUNDS_TYPE") + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("float64") + and values.shape == bounds.incoming.shape, + "WEIGHT_ARRAY", + ) + bounds.grouped.check(values, positive=False) + _require(np.all(values <= bounds.row_upper), "ROW_REFERENCE_CAP") + _require(np.array_equal(values == 0, bounds.incoming == 0), "FIXED_ZERO_SUPPORT") + + +def survey_age_calibration_node( + registry, *, base, budget_node, count_node, epochs, learning_rate +): + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + _require( + tuple(s.measure for s in frozen) == ages._COLUMNS + and all(s.metadata["evidence_scope"] == "invented" for s in frozen), + "INVENTED_AGE_REGISTRY_REQUIRED", + ) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + ) + + +def survey_age_development_node( + registry, + *, + activation_binding, + base, + budget_node, + count_node, + epochs, + learning_rate, +): + """Declare source-documented numbers; the country runner activates sources.""" + frozen = age_activation.validate_survey_age_registry(registry, activation_binding) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + activation_binding=activation_binding, + ) + + +def _survey_age_calibration_node( + frozen, + *, + base, + budget_node, + count_node, + epochs, + learning_rate, + activation_binding=None, +): + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + activation_params = {} + if activation_binding is not None: + declaration = age_activation.declaration_from_binding(activation_binding) + activation_params["activation"] = canonical_json( + age_activation.activation_binding(declaration) + ).decode() + return Node( + CALIBRATION_NODE, + SurveyAgeCalibrationKernel.ref, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + **activation_params, + "registry": demographic._registry_json(frozen), + "epochs": epochs, + "learning_rate": learning_rate, + "weight_anchor": "source_qualified_sampling_reference", + "cap_enforcement": "mandatory_country_runner_cold_cache_and_final", + "grouped_preserve_zeros": True, + "method": "adam", + "seed": 0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "initial_anchor_meaning": "frozen_incoming_clone_weights", + }, + artifact_inputs=( + ArtifactInput("bounds", budget_node, "numeric_bounds", BOUNDS_TYPE), + ArtifactInput("counts", count_node, "counts", ages.COUNTS_TYPE), + ), + artifact_outputs=(ArtifactOutput("diagnostics", demographic.DIAGNOSTICS_TYPE),), + ) + + +def calibration_receipt_scope(params): + """Identify numerical evidence scope without claiming source admission.""" + return ( + "source_documented_survey_age_calibration_numbers_only" + if "activation" in params + else "invented_survey_age_calibration_numbers_only" + ) + + +def calibration_activation_binding(params): + """Decode the bounded canonical-text profile without acquiring its sources.""" + if "activation" not in params: + return None + value = params["activation"] + _require(type(value) is str and 0 < len(value.encode()) <= 65536, "ACTIVATION_TEXT") + try: + binding = json.loads(value) + encoded = canonical_json(binding).decode() + except (ValueError, TypeError, RecursionError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_ACTIVATION_JSON") from None + _require(encoded == value, "ACTIVATION_CANONICAL") + age_activation.declaration_from_binding(binding) + return binding + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output) + and type(value.payload) is bytes, + "ARTIFACT_EDGE", + ) + return value.payload + + +class SurveyAgeCalibrationKernel(KernelBase): + """Real fixed-zero grouped Adam on a minimal numeric work Frame.""" + + ref = "us.survey_age_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + group_bounds, + dependencies=self.capabilities.dependencies, + ), + "age_artifact": ages.SurveyAgeCountArtifactKernel().implementation_hash(), + "activation_declaration": source_hash( + age_activation, + age_activation.age, + age_activation.survey_age_requests, + ), + "solver": demographic.DemographicCalibrationKernel().implementation_hash(), + } + ) + ) + + def run(self, context): + aliases = context.node.artifact_inputs + _require(tuple(a.name for a in aliases) == ("bounds", "counts"), "ALIASES") + registry = demographic._registry_from_json(context.params["registry"]) + factory = survey_age_calibration_node + profile_options = {} + if "activation" in context.params: + factory = survey_age_development_node + profile_options["activation_binding"] = calibration_activation_binding( + context.params + ) + expected = factory( + registry, + **profile_options, + base=context.node.base, + budget_node=aliases[0].producer, + count_node=aliases[1].producer, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + ) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + not context.tables and not context.weights and not context.sources, + "CONTEXT", + ) + bounds_raw = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + counts_raw = _artifact(context, "counts", ages.COUNTS_TYPE, "counts") + bounds = decode_numeric_survey_bounds(bounds_raw) + counts = ages.decode_survey_age_counts(counts_raw) + _require( + bounds.population == counts.population == context.node.base + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "COMPLETE_ORDERED_POPULATION", + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + incoming = Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + work = demographic.kernels_module._frame_from_context( + replace( + context, tables={"household": table}, weights={"household": incoming} + ), + "household", + ) + result = calibrate( + work, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + mass="free", + max_weight_ratio=None, + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + l2_lambda=context.params["l2_lambda"], + l2_anchor=context.params["l2_anchor"], + ) + _require(not result.skipped, "SKIPPED_TARGETS") + weights = result.frame.weights_for("household") + accepted = weights.values.tobytes() + check_numeric_survey_weights(bounds, weights.values) + anchors = { + "budget_sha256": bounds.budget_sha256, + "numeric_bounds_sha256": _sha(bounds_raw), + "counts_sha256": _sha(counts_raw), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "weight_anchor": expected.params["weight_anchor"], + "cap_enforcement": expected.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming == 0)), + } + payload = canonical_json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": payload}, + receipt={ + **anchors, + "diagnostics_sha256": _sha(payload), + "scope": calibration_receipt_scope(expected.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + # Validation follows diagnostics and return-object materialization. + fresh = decode_numeric_survey_bounds(bounds_raw) + check_numeric_survey_weights(fresh, weights.values) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + tuple(result.frame.table("household").household_id) + == fresh.grouped.household_ids, + "FINAL_HOUSEHOLD_IDS", + ) + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py new file mode 100644 index 000000000..624a088d8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py @@ -0,0 +1,1259 @@ +"""Graph declarations for a source-authenticated, selected survey population. + +The catalogue draw precedes native construction. CREATE therefore materializes +the selected combined population, and a separate REWEIGHT applies source shares +and inverse inclusion. Neither decoded evidence nor these declarations grants +source, population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, MassChange, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Ownership, + Slice, + StructuralDelta, + WeightTransition, +) + +from . import graph_context +from .graph_context import US_FRAME_CONTEXT_TYPE +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) +from .survey_catalogue_selection import CatalogueSelectionPlan, SelectedHousehold +from .survey_population_domains import HouseholdKey, Source + +PHASE = "us.authenticated_survey_population.v1" +SOURCE_NAME = "survey_population_source" +SOURCE_CODEC = "us-survey-population-source-v1" +CREATE_NODE = "survey_population.create" +ALLOCATION_NODE = "survey_population.allocate" +CREATE_REF = "us.survey_population.create@1" +ALLOCATION_REF = "us.survey_population.allocate@1" +PREPARATION_TYPE = ArtifactType("microcosm.us.survey_population_preparation", 2) +ALLOCATION_TYPE = ArtifactType("microcosm.us.survey_population_allocation", 1) +PREPARATION_MAX_BYTES = 64 * 1024**2 +ALLOCATION_MAX_BYTES = 64 * 1024**2 +STAGE = "authenticated_survey_population_v1" +STAGE_DEPENDENCIES = ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", +) +_HOUSEHOLD_ORIGIN_FIELDS = frozenset( + { + "household_id", + "source", + "source_year", + "survey_year", + "raw_native_id", + "selected_receiving_household_id", + "original_anchor", + } +) + + +@dataclass(frozen=True, slots=True) +class AllocationInstruction: + """Exact arithmetic values, without source or population authority.""" + + household_id: int + key: HouseholdKey + selected_receiving_household_id: int + original_anchor: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_weight(self): + return self.original_anchor * self.share / self.inclusion_probability + + +class SurveyPopulationGraphError(ValueError): + """Static graph refusal without source values or local paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationGraphError(code) + + +def _digest(value): + _require( + type(value) is str + and len(value) == 64 + and not set(value) - set("0123456789abcdef"), + "DIGEST", + ) + return value + + +def _draw_parameters(fraction, seed): + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + return (fraction.numerator, fraction.denominator), seed + + +def _fraction_pair(value): + _require( + type(value) in (list, tuple) + and len(value) == 2 + and all(type(v) is int for v in value) + and value[1] > 0, + "FRACTION_PAIR", + ) + number = Fraction(*value) + _require((number.numerator, number.denominator) == tuple(value), "FRACTION_PAIR") + return number + + +def allocation_instructions(plan, household_origins): + """Join checked preparation values; this pure join authenticates nothing. + + The runner obtains both arguments from a freshly issued preparation. Keeping + this operation separate lets the allocation kernel hold immutable values + instead of a source Frame or a mutable decoded preparation document. + """ + _require(type(plan) is CatalogueSelectionPlan, "SELECTION_PLAN") + _require(type(plan.selected) is tuple and plan.selected, "EMPTY_SELECTION") + _require(type(household_origins) in (list, tuple), "HOUSEHOLD_ORIGINS") + _require(len(household_origins) == len(plan.selected), "ORIGIN_COUNT") + # A prospective record bound precedes the two maps and output tuple. The + # streaming transport encoder enforces its exact bound separately. + _require(len(plan.selected) <= ALLOCATION_MAX_BYTES // 128, "ALLOCATION_LIMIT") + selected = {} + for row in plan.selected: + _require( + type(row) is SelectedHousehold and type(row.key) is HouseholdKey, + "SELECTED_TYPE", + ) + _require( + type(row.key.source) is Source + and type(row.key.source_year) is int + and row.key.source_year == 2024 + and type(row.key.survey_year) is int + and row.key.survey_year == (2024 if row.key.source is Source.ACS else 2025) + and type(row.key.native_id) is str + and 0 < len(row.key.native_id) <= 128, + "SELECTED_KEY", + ) + _require(row.key not in selected, "SELECTED_DUPLICATE") + _require( + type(row.original_design_weight) is Fraction + and row.original_design_weight >= 0 + and type(row.share) is Fraction + and 0 < row.share <= 1 + and type(row.inclusion_probability) is Fraction + and 0 < row.inclusion_probability <= 1, + "SELECTED_ARITHMETIC", + ) + selected[row.key] = row + output, seen_ids, seen_native, seen_receiving = [], set(), set(), set() + for origin in household_origins: + _require( + type(origin) is dict and set(origin) == _HOUSEHOLD_ORIGIN_FIELDS, + "ORIGIN_FIELDS", + ) + _require( + type(origin["source"]) is str and origin["source"] in {"acs", "asec"}, + "ORIGIN_SOURCE", + ) + _require( + all( + type(origin[field]) is int + for field in ( + "household_id", + "source_year", + "survey_year", + "selected_receiving_household_id", + ) + ), + "ORIGIN_INTEGER", + ) + _require( + all( + 0 <= origin[field] < 2**63 + for field in ("household_id", "selected_receiving_household_id") + ), + "ORIGIN_INTEGER", + ) + _require(type(origin["raw_native_id"]) is str, "ORIGIN_NATIVE_KEY") + key = HouseholdKey( + Source(origin["source"]), + origin["source_year"], + origin["survey_year"], + origin["raw_native_id"], + ) + row = selected.get(key) + _require(row is not None, "ORIGIN_NATIVE_KEY") + anchor = _fraction_pair(origin["original_anchor"]) + _require(anchor == row.original_design_weight, "ORIGIN_ANCHOR") + hh_id = origin["household_id"] + receiving = (key.source, origin["selected_receiving_household_id"]) + _require( + hh_id not in seen_ids + and key not in seen_native + and receiving not in seen_receiving, + "ORIGIN_DUPLICATE", + ) + seen_ids.add(hh_id) + seen_native.add(key) + seen_receiving.add(receiving) + instruction = AllocationInstruction( + hh_id, key, receiving[1], anchor, row.share, row.inclusion_probability + ) + _finite_weight(anchor) + _finite_weight(instruction.importance_weight) + output.append(instruction) + return tuple(output) + + +def _finite_weight(value): + try: + number = float(value) + except (OverflowError, ValueError): + raise SurveyPopulationGraphError("FLOAT_WEIGHT") from None + _require( + math.isfinite(number) and number >= 0 and (value == 0 or number > 0), + "FLOAT_WEIGHT", + ) + return number + + +def _bounded_json(value, limit): + """Canonical UTF-8 encoding with a bound checked before every append.""" + _require(type(limit) is int and 0 < limit <= 64 * 1024**2, "TRANSPORT_LIMIT") + result = bytearray() + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + try: + for piece in encoder.iterencode(value): + # All variable source tokens admitted to allocation are <=128 chars; + # this pre-encoding bound also protects against unexpected strings. + _require(len(piece) <= limit - len(result), "TRANSPORT_LIMIT") + encoded = piece.encode("utf-8") + _require(len(encoded) <= limit - len(result), "TRANSPORT_LIMIT") + result.extend(encoded) + except SurveyPopulationGraphError: + raise + except (TypeError, ValueError, UnicodeError): + raise SurveyPopulationGraphError("TRANSPORT_ENCODING") from None + return bytes(result) + + +def _source_owner(): + # The owner remains independent of this graph adapter. Declaration-only + # imports do not load source capture owners or their runtime resources. + from . import survey_population_preparation + + return survey_population_preparation + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_preparation(preparation): + owner = _source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + view = owner.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) + payload, context = view.payload, view.context + _require( + type(payload) is bytes and 0 < len(payload) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context) is bytes and 0 < len(context) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + return owner, view + + +def _artifact(context, name, expected_type, expected_payload): + from microcosm.graph.keys import opaque_artifact_key + + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and value.type == expected_type + and len(edges) == 1 + and edges[0].producer == CREATE_NODE + and edges[0].artifact == name + and value.key == opaque_artifact_key(value.producer_key, name) + and type(value.payload) is bytes + and value.payload == expected_payload, + "ARTIFACT_BINDING", + ) + return value + + +def _instruction_document(row): + def pair(value): + return [value.numerator, value.denominator] + + return { + "household_id": row.household_id, + "source": row.key.source.value, + "source_year": row.key.source_year, + "survey_year": row.key.survey_year, + "raw_native_id": row.key.native_id, + "selected_receiving_household_id": row.selected_receiving_household_id, + "original_anchor": pair(row.original_anchor), + "share": pair(row.share), + "inclusion_probability": pair(row.inclusion_probability), + "importance_multiplier": pair(row.share / row.inclusion_probability), + "importance_weight_float64_hex": _finite_weight(row.importance_weight).hex(), + } + + +def _allocation_payload( + instructions, *, preparation_sha256, input_context, output_context +): + """Stream rows individually; never construct a selected-population JSON tree.""" + metadata = { + "protocol": "microcosm.us.survey-population-allocation.v1", + "preparation_sha256": preparation_sha256, + "input_context_sha256": _sha(input_context), + "output_context_sha256": _sha(output_context), + "weight_transition": ["design", "importance"], + "release_eligible": False, + } + tail = b"]," + _bounded_json(metadata, 4096)[1:] + output = bytearray(b'{"households":[') + for index, row in enumerate(instructions): + raw = _bounded_json(_instruction_document(row), 4096) + separator = b"," if index else b"" + _require( + len(output) + len(separator) + len(raw) + len(tail) <= ALLOCATION_MAX_BYTES, + "ALLOCATION_LIMIT", + ) + output.extend(separator) + output.extend(raw) + _require(len(output) + len(tail) <= ALLOCATION_MAX_BYTES, "ALLOCATION_LIMIT") + output.extend(tail) + return bytes(output) + + +def _verify_allocation_view(frame, instructions): + _require(frame.weighted_entities == ("household",), "WEIGHT_SOURCES") + weights = frame.weights_for("household") + _require(weights.kind is WeightKind.DESIGN, "DESIGN_REQUIRED") + household = frame.table("household") + _require(len(household) == len(instructions), "HOUSEHOLD_COUNT") + _require(household.household_id.dtype == np.dtype("int64"), "HOUSEHOLD_ID_DTYPE") + _require( + tuple(household.household_id) == tuple(r.household_id for r in instructions), + "HOUSEHOLD_ORDER", + ) + _require( + tuple(household[support_channel_column("household")]) + == tuple(r.key.source.value for r in instructions), + "HOUSEHOLD_CHANNEL", + ) + _require( + tuple(household[spine_source_id_column("household")]) + == tuple(r.selected_receiving_household_id for r in instructions), + "RECEIVING_ID", + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + _require( + table[support_clone_index_column(entity)].dtype == np.dtype("int64") + and bool(table[support_clone_index_column(entity)].eq(0).all()), + "PRECLONE_REQUIRED", + ) + expected = np.asarray( + [_finite_weight(r.original_anchor) for r in instructions], dtype="float64" + ) + _require( + weights.values.dtype == expected.dtype + and weights.values.tobytes() == expected.tobytes(), + "ORIGINAL_ANCHORS", + ) + + +def _allocation_output(frame, context_bytes, instructions, preparation_sha256): + _verify_allocation_view(frame, instructions) + values = np.asarray( + [_finite_weight(r.importance_weight) for r in instructions], dtype="float64" + ) + weights = Weights(values, WeightKind.IMPORTANCE) + allocated = frame.with_weights( + "household", + weights, + mass=MassChange( + factor=None, reason=f"{PHASE}: declared source shares and inverse inclusion" + ), + ) + context = graph_context._decode(context_bytes) + _require( + context["weight_sources"] == {"household": "design"}, "CONTEXT_WEIGHT_KIND" + ) + context["weight_sources"] = {"household": "importance"} + context["mass_log"] = [ + graph_context._json_data(asdict(r)) for r in allocated.mass_log + ] + new_context = _bounded_json(context, PREPARATION_MAX_BYTES) + payload = _allocation_payload( + instructions, + preparation_sha256=preparation_sha256, + input_context=context_bytes, + output_context=new_context, + ) + before, after = frame.stratum_mass(), allocated.stratum_mass() + receipt = { + "phase": PHASE, + "preparation_sha256": preparation_sha256, + "allocation_sha256": _sha(payload), + "release_eligible": False, + "frame_mass_log_append": [ + graph_context._json_data(asdict(r)) + for r in allocated.mass_log[len(frame.mass_log) :] + ], + "mass": { + "policy": "declared", + "before": float(before.sum()), + "after": float(after.sum()), + "stratum_before": before.to_dict(), + "stratum_after": after.to_dict(), + }, + } + return weights, new_context, payload, receipt, allocated + + +class _Kernel(KernelBase): + def implementation_hash(self): + from .graph_implementation import ( + STAGE_DEPENDENCIES as STAGES, + ) + from .graph_implementation import ( + implementation_hash, + ) + + _require(STAGES[STAGE] == STAGE_DEPENDENCIES, "STAGE_DEPENDENCIES") + return implementation_hash(STAGE) + + +class SurveyPopulationCreateKernel(_Kernel): + ref = CREATE_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation, *, source_dir): + self._preparation = preparation + self._source_dir = Path(source_dir).absolute() + + def run(self, context): + from .graph_sources import frame_column_declarations + + preparation = self._preparation + owner, view = _checked_preparation(preparation) + payload, context_bytes, plan = view.payload, view.context, view.selection_plan + expected, _ = survey_population_nodes( + frame_column_declarations(view.frame), + preparation_sha256=_sha(payload), + fraction=plan.fraction, + seed=plan.seed, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "CREATE_DECLARATION", + ) + _require( + set(context.sources) == {SOURCE_NAME} + and Path(context.sources[SOURCE_NAME]) + == self._source_dir.resolve(strict=True), + "SOURCE_PATH", + ) + owner.verify_survey_population_preparation(preparation) + return KernelResult( + frame=view.frame, + artifacts={"frame_context": context_bytes, "preparation": payload}, + receipt={ + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + }, + ) + + +class SurveyPopulationAllocationKernel(_Kernel): + ref = ALLOCATION_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, instructions, *, preparation_bytes, context_bytes): + _require( + type(instructions) is tuple + and instructions + and all(type(row) is AllocationInstruction for row in instructions), + "INSTRUCTIONS", + ) + _require( + type(preparation_bytes) is bytes + and 0 < len(preparation_bytes) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context_bytes) is bytes + and 0 < len(context_bytes) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + self._instructions = instructions + self._preparation_bytes = preparation_bytes + self._context_bytes = context_bytes + + def run(self, context): + node = context.node + digest = _sha(self._preparation_bytes) + _require( + node.id == ALLOCATION_NODE + and node.kernel == self.ref + and node.structural is StructuralDelta.REWEIGHT + and node.base == CREATE_NODE + and node.mass == "declared" + and not node.outputs + and not node.sources + and node.weights + == WeightTransition("household", "importance", mass="declared") + and node.inputs + == tuple(Slice(e, _provenance_columns(e)) for e in US_SCHEMA.entities) + and node.artifact_inputs + == ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", CREATE_NODE, "preparation", PREPARATION_TYPE + ), + ) + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ) + and dict(context.params) == {"phase": PHASE, "preparation_sha256": digest}, + "ALLOCATION_DECLARATION", + ) + _require( + set(context.artifacts) == {"frame_context", "preparation"}, "ARTIFACTS" + ) + bound = _artifact( + context, "frame_context", US_FRAME_CONTEXT_TYPE, self._context_bytes + ) + prepared = _artifact( + context, "preparation", PREPARATION_TYPE, self._preparation_bytes + ) + _require(bound.producer_key == prepared.producer_key, "ARTIFACT_PRODUCER") + original = graph_context.us_frame_from_context(context) + weights, new_context, payload, receipt, _allocated = _allocation_output( + original, bound.payload, self._instructions, digest + ) + return KernelResult( + weights=weights, + artifacts={"frame_context": new_context, "allocation": payload}, + receipt=receipt, + ) + + +def survey_population_source_codecs(*, snapshot_root): + """One real source codec, isolated from the legacy five-codec registry.""" + from microcosm.graph.codecs import SourceCodecRegistry + + snapshot_root = Path(snapshot_root) + + def load(path, *, store=None): + del store + owner = _source_owner() + fraction, seed = owner.read_survey_population_request(path) + preparation = owner.prepare_authenticated_survey_population( + path, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + owner.verify_survey_population_preparation(preparation) + return view.frame + + codecs = SourceCodecRegistry() + codecs.register(SOURCE_CODEC, load) + return codecs + + +def _same_frame(expected, actual): + """Full semantic identity; mutable-storage source seals stay in their owner.""" + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links + and expected.metadata == actual.metadata + and expected.mass_log == actual.mass_log + and expected.weighted_entities == actual.weighted_entities, + "MATERIALIZED_FRAME_CONTEXT", + ) + try: + for entity in expected.entities: + pd.testing.assert_frame_equal( + expected.table(entity), + actual.table(entity), + check_exact=True, + check_flags=True, + ) + for link in expected.links: + pd.testing.assert_frame_equal( + expected.link(link), + actual.link(link), + check_exact=True, + check_flags=True, + ) + pd.testing.assert_series_equal(expected.strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("MATERIALIZED_FRAME_VALUES") from None + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "MATERIALIZED_FRAME_WEIGHTS", + ) + + +def _verify_cloned_frame(before, actual, design_weights): + from . import puf_support + + _require( + before.schema == actual.schema + and before.entities == actual.entities + and before.links == actual.links == () + and before.metadata == actual.metadata + and before.mass_log == actual.mass_log + and actual.weighted_entities == ("household",), + "CLONE_CONTEXT", + ) + multiplier = puf_support._id_multiplier_for_frame(before) + try: + # Execute the real operator's row transform one entity at a time. This + # checks every carried cell, missing value, dtype, native key and remapped + # membership without constructing another full cloned Frame. + for entity in before.entities: + expected = puf_support._clone_preassembled_entity_table( + before.table(entity), + entity=entity, + schema=before.schema, + id_multiplier=multiplier, + ) + pd.testing.assert_frame_equal( + expected, actual.table(entity), check_exact=True, check_flags=True + ) + expected_strata = pd.concat([before.strata, before.strata], ignore_index=True) + pd.testing.assert_series_equal(expected_strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("CLONE_FRAME_VALUES") from None + weights = actual.weights_for("household") + expected_weights = np.tile(before.weights_for("household").values / 2, 2) + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.tobytes() == expected_weights.tobytes(), + "CLONE_WEIGHTS", + ) + # The graph duplicates original design anchors along lineage. It halves + # current IMPORTANCE weights, never the original design evidence. + expected_design = np.tile(design_weights, 2) + puf_support.validate_puf_clone_attachment( + actual, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + return expected_design + + +def _check_design_anchors(population, expected): + _require(set(population.design_weights) == {"household"}, "DESIGN_ANCHOR_SOURCES") + actual = population.design_weights["household"] + _require( + type(actual) is np.ndarray + and actual.dtype == expected.dtype + and actual.shape == expected.shape + and actual.tobytes() == expected.tobytes(), + "DESIGN_ANCHOR_LINEAGE", + ) + + +def _check_population_state(population, *, version, owners, kind, ledger): + from microcosm.graph.population import MassRecord + + _require( + type(population.version) is str and population.version == version, + "POPULATION_VERSION", + ) + _require( + dict(population.owners) == owners + and all(type(value) is str for value in population.owners.values()), + "POPULATION_OWNERS", + ) + _require( + dict(population.weight_kind) == {"household": kind} + and type(population.weight_kind["household"]) is WeightKind, + "POPULATION_WEIGHT_KIND", + ) + _require( + type(population.mass_ledger) is tuple + and all(type(record) is MassRecord for record in population.mass_ledger) + and population.mass_ledger == ledger, + "POPULATION_MASS_LEDGER", + ) + + +def _final_artifact(manifest, store, *, node_id, name, type_, payload, capabilities): + from microcosm.graph.artifact_edges import descriptor + from microcosm.graph.keys import opaque_artifact_key + + node = manifest.node(node_id) + expected = descriptor( + producer=node_id, + artifact=name, + type_=type_, + producer_key=node.key, + capabilities=capabilities, + ) + _require( + node.typed_artifacts["outputs"].get(name) == expected, + "FINAL_ARTIFACT_DESCRIPTOR", + ) + key = opaque_artifact_key(node.key, name) + _require(node.opaque_artifacts.get(name) == key, "FINAL_ARTIFACT_KEY") + actual = store.load_bytes(key) + _require(type(actual) is bytes and actual == payload, "FINAL_ARTIFACT_BYTES") + + +def _check_node_states(manifest, expected): + from microcosm.graph.keys import _capabilities_projection + + _require(set(manifest.nodes) == set(expected), "MANIFEST_NODE_SET") + for node_id, state in expected.items(): + node = manifest.node(node_id) + _require( + node.key == state["key"] + and node.kernel_ref == state["ref"] + and type(node.capabilities) is Capabilities + and _capabilities_projection(node.capabilities) == state["capabilities"] + and node.kernel_impl_hash == state["implementation"] + and node.typed_artifacts == state["typed_artifacts"] + and type(node.seed) is int + and node.seed == state["seed"] + and node.frame_key == state["frame_key"] + and node.weight_key == state["weight_key"] + and node.artifacts == state["artifacts"] + and node.opaque_artifacts == state["opaque_artifacts"] + and node.receipt == state["receipt"] + and node.legacy_capabilities is False, + "MANIFEST_NODE_STATE", + ) + + +@dataclass(frozen=True) +class SurveyPopulationRunValues: + """Values retained from one completed run, without additional authority. + + These are the actual objects used by the runner, not decoded substitutes. + Downstream source-qualified boundaries must check the issued preparation + and complete populations themselves. Freezing this container does not seal + its contents, authenticate a copy, or admit a later graph descendant. + """ + + manifest: object + preparation: object + allocated_population: object + clone_population: object | None + compiled: object + store: object + kernels: object + sources: dict + + +def run_authenticated_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + resume="auto", + clones=True, + return_values=False, +): + """Reconstruct sources, then execute and verify the real selected graph. + + Warm execution can skip graph kernels; it still reconstructs both complete + catalogues and the selected native populations. Returned manifests are + development build records. Decoding their bytes grants no source authority. + ``return_values=True`` returns the live composition values with the manifest; + it does not bypass any final checks or issue a downstream admission. + """ + from microcosm.graph import ( + ContentStore, + Graph, + KernelRegistry, + SourceRef, + compile_graph, + run_graph, + ) + from microcosm.graph.artifact_edges import typed_contracts + from microcosm.graph.executor import ( + _all_node_keys, + _expand_declared_payload, + _expand_rewrite_coordinates, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, + ) + from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, + ) + from microcosm.graph.keys import seed as node_seed + from microcosm.graph.manifest import _freeze_json + from microcosm.graph.population import ( + _mass_record, + expand_lineage_receipt, + expand_writes_receipt, + mass_record_receipt, + weight_cap_receipt, + ) + + from .graph_sources import frame_column_declarations + + _draw_parameters(fraction, seed) + _require(type(clones) is bool, "CLONES_FLAG") + _require(type(return_values) is bool, "RETURN_VALUES_FLAG") + # Do not resolve source symlinks before their authenticated owner checks them. + source_dir, snapshot_root, store_root = ( + Path(path).absolute() for path in (source_dir, snapshot_root, store_root) + ) + owner = _source_owner() + preparation = owner.prepare_authenticated_survey_population( + source_dir, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + # Resolve '..' only after the source owner has checked the raw path and + # refused symlinks. The executor uses this same canonical path identity. + source_dir = source_dir.resolve(strict=True) + payload, context_bytes, prepared_frame = view.payload, view.context, view.frame + instructions = allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + columns = frame_column_declarations(prepared_frame) + nodes = survey_population_nodes( + columns, preparation_sha256=_sha(payload), fraction=fraction, seed=seed + ) + _weights, allocated_context, allocation_payload, allocation_receipt, allocated = ( + _allocation_output(prepared_frame, context_bytes, instructions, _sha(payload)) + ) + design = np.array(prepared_frame.weights_for("household").values, copy=True) + kernels = KernelRegistry() + kernels.register(SurveyPopulationCreateKernel(preparation, source_dir=source_dir)) + kernels.register( + SurveyPopulationAllocationKernel( + instructions, preparation_bytes=payload, context_bytes=context_bytes + ) + ) + clone_nodes = () + if clones: + from . import graph_combined_clone as clone + from . import puf_support + from .graph_combined_clone import ( + register_us_combined_survey_clone_kernels, + us_combined_survey_clone_nodes, + ) + + clone_nodes = us_combined_survey_clone_nodes( + columns, base=ALLOCATION_NODE, source_channels=("acs", "asec") + ) + register_us_combined_survey_clone_kernels(kernels) + compiled = compile_graph( + Graph("us", (SourceRef(SOURCE_NAME, SOURCE_CODEC),), (*nodes, *clone_nodes)) + ) + store = ContentStore( + store_root, codecs=survey_population_source_codecs(snapshot_root=snapshot_root) + ) + observed = {} + expected_states = {} + expected_writer_receipts = {} + allocation_ledger = ( + _mass_record( + prepared_frame, + allocated, + nodes[1], + KernelResult(receipt=allocation_receipt), + "declared", + ), + ) + all_cells = tuple( + (entity, str(column)) + for entity in prepared_frame.entities + for column in prepared_frame.table(entity) + ) + + def observe(node_id, population): + owner.verify_survey_population_preparation(preparation) + if node_id == CREATE_NODE: + _same_frame(prepared_frame, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, CREATE_NODE) + kind, ledger = WeightKind.DESIGN, () + elif node_id == ALLOCATION_NODE: + _same_frame(allocated, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, ALLOCATION_NODE) + kind, ledger = WeightKind.IMPORTANCE, allocation_ledger + elif node_id in {node.id for node in clone_nodes}: + expected_design = _verify_cloned_frame(allocated, population.frame, design) + owners = dict.fromkeys(all_cells, clone_nodes[0].id) + if node_id == clone_nodes[1].id: + owners.update( + {(o.entity, o.column): node_id for o in clone_nodes[1].outputs} + ) + # Full cloned cells have been checked against the real operator row + # transform. Derive mass from these values, never the cached ledger. + kind = WeightKind.IMPORTANCE + ledger = ( + *allocation_ledger, + _mass_record( + allocated, + population.frame, + clone_nodes[0], + KernelResult(), + "conserve", + ), + ) + else: + raise SurveyPopulationGraphError("UNEXPECTED_POPULATION") + _check_design_anchors(population, expected_design) + state = dict( + version=compiled.versions[node_id], owners=owners, kind=kind, ledger=ledger + ) + _check_population_state(population, **state) + node = compiled.graph.node(node_id) + if node_id == CREATE_NODE: + receipt = { + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + } + elif node_id == ALLOCATION_NODE: + receipt = dict(allocation_receipt) + elif node_id == clone_nodes[0].id: + lineage, entity_facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = clone._entity_lineage( + allocated, population.frame, entity + ) + authority = puf_support.validate_puf_clone_attachment( + population.frame, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + allocated, population.frame, ("acs", "asec"), authority, entity_facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(node) + receipt["expand_writes"] = expand_writes_receipt( + allocated, + population.frame, + node, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, node), + ) + else: + receipt = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{output.entity}.{output.column}" for output in node.outputs + ), + } + receipt["capabilities"] = dict(expected_nodes[node_id]["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=expected_writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + expected_nodes[node_id]["receipt"] = _freeze_json(receipt) + # Value-only input for the generic writer analysis. This is neither a + # returned NodeReceipt nor a source/Frame authority object. + expected_writer_receipts[node_id] = SimpleNamespace( + receipt=expected_nodes[node_id]["receipt"] + ) + _require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + observed[node_id] = population + expected_states[node_id] = state + + # Derive expectations independently of returned/cache receipts. This is one + # additional streaming source-key pass, not another draw or native build. + _paths, source_keys = _source_paths_and_keys( + compiled, {SOURCE_NAME: source_dir}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = {} + for node in compiled.graph.nodes: + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + all_cells + if structural + else tuple((output.entity, output.column) for output in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + expected_nodes[node.id] = { + "key": key, + "ref": node.kernel, + "capabilities": _capabilities_projection( + kernels.get(node.kernel).capabilities + ), + "implementation": implementations[node.id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + output.name: opaque_artifact_key(key, output.name) + for output in node.artifact_outputs + }, + } + manifest = run_graph( + compiled, + sources={SOURCE_NAME: source_dir}, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + _check_node_states(manifest, expected_nodes) + # Explicit store reads are necessary: the observer sees real populations, + # while typed byte artifacts pass through a different materialization path. + for node_id, name, type_, raw, capabilities in ( + ( + CREATE_NODE, + "preparation", + PREPARATION_TYPE, + payload, + SurveyPopulationCreateKernel.capabilities, + ), + ( + CREATE_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + context_bytes, + SurveyPopulationCreateKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "allocation", + ALLOCATION_TYPE, + allocation_payload, + SurveyPopulationAllocationKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + allocated_context, + SurveyPopulationAllocationKernel.capabilities, + ), + ): + _final_artifact( + manifest, + store, + node_id=node_id, + name=name, + type_=type_, + payload=raw, + capabilities=capabilities, + ) + # Construct the optional value container before terminal checks. A caller + # still needs the downstream issuer's independent reconstruction; this is + # deliberately not another source certificate or a manifest decoder. + result = ( + SurveyPopulationRunValues( + manifest=manifest, + preparation=preparation, + allocated_population=observed[ALLOCATION_NODE], + clone_population=observed[clone_nodes[-1].id] if clone_nodes else None, + compiled=compiled, + store=store, + kernels=kernels, + sources={SOURCE_NAME: source_dir}, + ) + if return_values + else manifest + ) + # Finish all potentially expensive owner I/O before the final pure checks + # of graph-owned Frames. A callback during the last producer/source check + # must not mutate an already-observed derivative and escape detection. + _owner, final_view = _checked_preparation(preparation) + _require( + final_view.payload == payload + and final_view.context == context_bytes + and final_view.frame is prepared_frame, + "FINAL_PREPARATION_SEAL", + ) + _same_frame(prepared_frame, manifest.population(CREATE_NODE)) + _same_frame(allocated, manifest.population(ALLOCATION_NODE)) + for node_id in (CREATE_NODE, ALLOCATION_NODE): + _check_design_anchors(observed[node_id], design) + for node in clone_nodes: + cloned_design = _verify_cloned_frame( + allocated, manifest.population(compiled.versions[node.id]), design + ) + _check_design_anchors(observed[node.id], cloned_design) + for node_id, state in expected_states.items(): + _check_population_state(observed[node_id], **state) + _require( + manifest.mass_ledger(compiled.versions[node_id]) == state["ledger"], + "FINAL_MASS_LEDGER", + ) + _check_node_states(manifest, expected_nodes) + return result + + +def _provenance_columns(entity): + return ( + support_channel_column(entity), + support_source_id_column(entity), + spine_source_id_column(entity), + support_clone_index_column(entity), + ) + + +def survey_population_nodes( + columns: Sequence[Owned], *, preparation_sha256: str, fraction: Fraction, seed: int +) -> tuple[Node, Node]: + """Declare selected CREATE then allocation, without an invented FILTER. + + The digest names checked preparation bytes; accepting it here authenticates + nothing. The country runner must reconstruct the preparation and verify the + materialized populations and artifacts on both cold and cached execution. + """ + _require(type(columns) in (tuple, list), "COLUMNS") + columns = tuple(columns) + _require(columns and all(type(column) is Owned for column in columns), "COLUMNS") + _require( + all( + column.entity in US_SCHEMA.entities + and column.rows == "all" + and column.ownership is Ownership.PRODUCED + and column.rewrite is False + for column in columns + ), + "CREATE_COLUMN_CONTRACT", + ) + inventory = {(column.entity, column.column): column for column in columns} + _require(len(inventory) == len(columns), "DUPLICATE_COLUMNS") + for entity in US_SCHEMA.entities: + for name in _provenance_columns(entity): + column = inventory.get((entity, name)) + _require(column is not None, "MISSING_PROVENANCE_COLUMN") + _require( + column.dtype + == ("string" if name == support_channel_column(entity) else "int64"), + "PROVENANCE_DTYPE", + ) + fraction_pair, seed = _draw_parameters(fraction, seed) + digest = _digest(preparation_sha256) + create = Node( + id=CREATE_NODE, + kernel=CREATE_REF, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={ + "phase": PHASE, + "preparation_sha256": digest, + "fraction": fraction_pair, + "sample_seed": seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", PREPARATION_TYPE), + ), + ) + allocate = Node( + id=ALLOCATION_NODE, + kernel=ALLOCATION_REF, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=tuple( + Slice(entity, _provenance_columns(entity)) for entity in US_SCHEMA.entities + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": PHASE, "preparation_sha256": digest}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput("preparation", CREATE_NODE, "preparation", PREPARATION_TYPE), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ), + ) + return create, allocate diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py new file mode 100644 index 000000000..b0ba5fac6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py @@ -0,0 +1,648 @@ +"""Explicit activation of the direct-national ACS 2024 published age counts. + +A captured reference bundle is evidence, not permission. The accepted +`national-derived-inventory.json` says so about itself: `target_activation` is +false, `calibration_target_registry` is `not_created`, and every table carries +`targets_allowed: false`. This module never reads that flag as authority and +never edits the bundle. Authority comes from +:data:`NATIONAL_AGE_ACTIVATION` — an immutable declaration, checked into this +repository and reviewed as code, that names the source digest, the exact +publisher cells, the target role, the population universe, the target period +and the age convention. :func:`activate_national_age_targets` refuses unless +the pinned bytes reproduce that declaration, and it then builds a **new** +:class:`~microcosm.calibrate.registry.TargetRegistry` artifact. + +What is authenticated here: the inventory bytes against the declared digest; +each selected raw Census response against the digest recorded for it *and* +against its request descriptor; the publisher's own labels, group, concept and +integer predicate for every selected cell; the single national row's +`GEO_ID`/`us`/`NAME` identity; and a from-bytes re-derivation of every selected +cell through the same :func:`classify_acs_value` precedence the district +inventory uses. Cells are parsed from exactly the bytes that were hashed. + +What is *not* established here: that the resulting calibration is nationally or +congressional-district valid, that any modelled person was observed in 2024, or +that this registry may be released. B19001 and B25003 are reserved same-source +holdouts and are refused outright — this module cannot activate them, and it +cannot mark them fresh or unseen. + +Published margins of error are converted to standard errors with the Census +90 percent factor and travel on the specs. The current Adam loss does not +consume them. A cell whose margin is a controlled-estimate sentinel carries a +**null** standard error, never zero: "no published variance" is not "no +variance". +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from microcosm.calibrate.registry import TargetRegistry, TargetSpec + +from ..cd_benchmark.protocol import RESERVED_FAMILIES +from .cd_reference import AUTHORITIES, classify_acs_value +from .cd_reference_sources import strict_json + +__all__ = [ + "ACTIVATION_SCHEMA_VERSION", + "ActivationError", + "AgeBand", + "NationalAgeActivation", + "NATIONAL_AGE_ACTIVATION", + "activate_national_age_targets", + "activation_digest", + "band_columns", +] + +#: Revision of the activation declaration's canonical form. Bump it when the +#: meaning of a field changes; the digest already moves when a value changes. +ACTIVATION_SCHEMA_VERSION = 1 + +#: Census publishes ACS margins of error at 90 percent confidence; this is the +#: documented divisor that turns one into a standard error. The authority is +#: the same sample-size-and-data-quality note the district inventory cites. +_MOE_90_TO_SE = 1.645 +_MOE_AUTHORITY = AUTHORITIES["acs_confidence"] + +#: The published label prefix of the disjoint AGE distribution in S0101. Cells +#: under `SELECTED AGE CATEGORIES` overlap it and are refused by construction. +_AGE_LABEL_PREFIX = "Estimate!!Total!!Total population!!AGE!!" + +_HEX64 = re.compile(r"[0-9a-f]{64}") + + +class ActivationError(ValueError): + """The pinned evidence does not reproduce the activation declaration.""" + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ActivationError(reason) + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +@dataclass(frozen=True, slots=True) +class AgeBand: + """One activated publisher cell and the household column that answers it. + + Attributes: + variable: The publisher cell, e.g. ``"S0101_C01_002"``. + label: The publisher's exact label for that cell's estimate. Binding + it here means an edited cell list cannot silently pull a + differently-defined published quantity. + low: Inclusive lower bound in completed years. + high: Inclusive upper bound, or ``None`` for the open top band. + column: The household count column the population operator owns. + """ + + variable: str + label: str + low: int + high: int | None + column: str + + def __post_init__(self) -> None: + _require( + isinstance(self.variable, str) + and isinstance(self.label, str) + and isinstance(self.column, str) + and self.variable + and self.label + and self.column, + "AGE_BAND_TEXT", + ) + _require( + type(self.low) is int + and self.low >= 0 + and ( + self.high is None or (type(self.high) is int and self.high >= self.low) + ), + f"AGE_BAND_BOUNDS:{self.variable}", + ) + + +def _bands() -> tuple[AgeBand, ...]: + quinary = [(low, low + 4) for low in range(0, 85, 5)] + bands = [] + for index, (low, high) in enumerate(quinary): + if low == 0: + label = "Under 5 years" + else: + label = f"{low} to {high} years" + bands.append( + AgeBand( + variable=f"S0101_C01_{index + 2:03d}", + label=_AGE_LABEL_PREFIX + label, + low=low, + high=high, + column=f"people_age_{low}_{high}", + ) + ) + bands.append( + AgeBand( + variable="S0101_C01_019", + label=_AGE_LABEL_PREFIX + "85 years and over", + low=85, + high=None, + column="people_age_85_plus", + ) + ) + return tuple(bands) + + +@dataclass(frozen=True, slots=True) +class NationalAgeActivation: + """The immutable authority for one calibration connection. + + Every field is normative: :func:`activation_digest` covers all of them, and + the digest travels into each produced :class:`TargetSpec`, so a changed + declaration produces a different registry version and a different node key. + + Attributes: + inventory_sha256: Digest of the accepted derived inventory's bytes. + table: The activated publisher table. + dataset: The Census API dataset the selected responses come from. + acs_year: The publisher's data year. + acs_release: The publisher's release. + geography: The direct national geography id. + geography_query: The publisher query that produced that single row. + published_name: The publisher's own name for that row. + confidence_level: The confidence level of the published margins. + role: What the activated cells are for. + universe: The population universe the cells count, stated in full. + entity: The entity whose weights the targets constrain. + period: The target period tag. + age_convention_id: The closed token naming the convention. The + population operator implements exactly this one and refuses any + other, so the declaration and the operator cannot drift apart. + age_convention: How model ages are read against the published bands. + allowed_reservation_status: The only inventory reservation status this + activation may act on. The same-source holdouts carry a different + one and can never satisfy it. + bands: The activated cells, in publisher order. + """ + + inventory_sha256: str + table: str + dataset: str + acs_year: int + acs_release: str + geography: str + geography_query: str + published_name: str + confidence_level: float + role: str + universe: str + entity: str + period: str + age_convention_id: str + age_convention: str + allowed_reservation_status: str + bands: tuple[AgeBand, ...] + + def __post_init__(self) -> None: + _require( + _HEX64.fullmatch(self.inventory_sha256) is not None, + "ACTIVATION_INVENTORY_DIGEST", + ) + _require(self.table not in RESERVED_FAMILIES, f"RESERVED_TABLE:{self.table}") + _require(self.table == "S0101", f"UNSUPPORTED_TABLE:{self.table}") + _require(self.role == "calibration", f"UNSUPPORTED_ROLE:{self.role}") + _require(self.universe.startswith("population"), "UNSUPPORTED_UNIVERSE") + _require(self.entity == "household", f"UNSUPPORTED_ENTITY:{self.entity}") + _require(self.period == "2024", f"UNSUPPORTED_PERIOD:{self.period}") + _require( + self.age_convention_id == "observed_interview_age_completed_years", + f"UNSUPPORTED_AGE_CONVENTION:{self.age_convention_id}", + ) + _require(self.acs_year == 2024, "UNSUPPORTED_ACS_YEAR") + _require(self.geography == "0100000US", "UNSUPPORTED_GEOGRAPHY") + _require( + self.allowed_reservation_status == "inventory_only", + "UNSUPPORTED_RESERVATION_STATUS", + ) + _require( + isinstance(self.bands, tuple) + and all(isinstance(band, AgeBand) for band in self.bands), + "ACTIVATION_BANDS_TYPE", + ) + _require(len(self.bands) == 18, f"ACTIVATION_BAND_COUNT:{len(self.bands)}") + _require( + len({band.variable for band in self.bands}) == len(self.bands) + and len({band.column for band in self.bands}) == len(self.bands), + "ACTIVATION_BANDS_NOT_DISTINCT", + ) + expected_low = 0 + for index, band in enumerate(self.bands): + _require(band.low == expected_low, f"ACTIVATION_BANDS_GAP:{band.variable}") + _require( + band.label.startswith(_AGE_LABEL_PREFIX), + f"ACTIVATION_BAND_NOT_AGE_DISTRIBUTION:{band.variable}", + ) + last = index == len(self.bands) - 1 + _require( + (band.high is None) is last, + f"ACTIVATION_BAND_OPEN_INTERVAL:{band.variable}", + ) + if band.high is not None: + expected_low = band.high + 1 + # Publisher cell order is the band order; nothing may be reordered or + # substituted without changing the digest. + _require( + [band.variable for band in self.bands] + == [f"S0101_C01_{index:03d}" for index in range(2, 20)], + "ACTIVATION_CELL_SELECTOR", + ) + + +#: The reviewed authority. Root's first engineering rung: observed interview +#: ages from pooled source cohorts, calibrated to the 2024 published national +#: distribution. No birth dates, no invented aging, and no claim that any +#: modelled person was interviewed in 2024. +NATIONAL_AGE_ACTIVATION = NationalAgeActivation( + inventory_sha256=( + "750b624e89557944c442600befb5557847dd75be6ce439c20daa9dfea5a0baf7" + ), + table="S0101", + dataset="acs/acs1/subject", + acs_year=2024, + acs_release="1-year", + geography="0100000US", + geography_query="us:*", + published_name="United States", + confidence_level=0.9, + role="calibration", + universe=( + "population: every resident person the publisher counts, household " + "and group-quarters alike; no household-population subsetting" + ), + entity="household", + period="2024", + age_convention_id="observed_interview_age_completed_years", + age_convention=( + "age in completed years as observed at each person's own source " + "interview, used verbatim; source cohorts are pooled and their " + "observed age distribution is calibrated to the 2024 published " + "counts; no birth dates, no aging, and no assertion that any person " + "was observed in 2024" + ), + allowed_reservation_status="inventory_only", + bands=_bands(), +) + + +def activation_digest(declaration: NationalAgeActivation) -> str: + """The content digest of a declaration's canonical form.""" + + # asdict recurses into the frozen AgeBand rows, so every bound, label and + # column name is inside the digest. + return hashlib.sha256( + _canonical({"schema_version": ACTIVATION_SCHEMA_VERSION, **asdict(declaration)}) + ).hexdigest() + + +def band_columns( + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> tuple[str, ...]: + """The household count columns the activation expects, in band order.""" + + return tuple(band.column for band in declaration.bands) + + +def _read_pinned(path: Path, digest: str, *, what: str) -> tuple[bytes, object]: + """Hash the bytes, then parse *those* bytes. Never re-read from disk.""" + + _require(_HEX64.fullmatch(digest) is not None, f"PINNED_DIGEST_SHAPE:{what}") + _require(path.is_file(), f"PINNED_FILE_MISSING:{what}") + payload = path.read_bytes() + actual = hashlib.sha256(payload).hexdigest() + _require(actual == digest, f"PINNED_DIGEST_MISMATCH:{what}:{actual}") + return payload, strict_json(payload) + + +def _selected_source(inventory: dict, declaration: NationalAgeActivation, kind: str): + sources = inventory.get("sources") + _require(isinstance(sources, dict), "INVENTORY_SOURCES") + census = sources.get("census") + _require(isinstance(census, list), "INVENTORY_CENSUS_SOURCES") + entries = [ + entry + for entry in census + if isinstance(entry, dict) + and entry.get("table") == declaration.table + and entry.get("kind") == kind + ] + _require(len(entries) == 1, f"SOURCE_ENTRY_NOT_UNIQUE:{kind}:{len(entries)}") + entry = entries[0] + digest = entry.get("sha256") + _require( + isinstance(digest, str) and _HEX64.fullmatch(digest) is not None, + f"SOURCE_DIGEST:{kind}", + ) + _require(entry.get("dataset") == declaration.dataset, f"SOURCE_DATASET:{kind}") + _require(entry.get("year") == declaration.acs_year, f"SOURCE_YEAR:{kind}") + _require( + entry.get("geography") == declaration.geography_query, + f"SOURCE_GEOGRAPHY:{kind}", + ) + url = entry.get("url") + _require(isinstance(url, str) and url.startswith("https://"), f"SOURCE_URL:{kind}") + _require( + f"/{declaration.acs_year}/{declaration.dataset}" in url, + f"SOURCE_URL_SCOPE:{kind}", + ) + _require( + entry.get("path") == f"raw/{entry.get('sha256')}.json", f"SOURCE_PATH:{kind}" + ) + return entry + + +def _authenticated_response( + root: Path, entry: dict, *, kind: str +) -> tuple[bytes, object]: + """Authenticate the request descriptor and the response it names.""" + + descriptor_path = ( + root / "requests" / f"{hashlib.sha256(entry['url'].encode()).hexdigest()}.json" + ) + _require(descriptor_path.is_file(), f"REQUEST_DESCRIPTOR_MISSING:{kind}") + descriptor = strict_json(descriptor_path.read_bytes()) + _require(descriptor == entry, f"REQUEST_DESCRIPTOR_MISMATCH:{kind}") + payload, document = _read_pinned( + root / entry["path"], entry["sha256"], what=f"response:{kind}" + ) + _require(len(payload) == entry.get("size_bytes"), f"RESPONSE_SIZE:{kind}") + return payload, document + + +def _national_row(document: object, declaration: NationalAgeActivation) -> dict: + _require( + isinstance(document, list) and len(document) == 2, + "RESPONSE_NOT_A_SINGLE_NATIONAL_ROW", + ) + header, values = document + _require( + isinstance(header, list) and all(isinstance(name, str) for name in header), + "RESPONSE_HEADER_SHAPE", + ) + _require(len(set(header)) == len(header), "RESPONSE_DUPLICATE_HEADER") + _require(isinstance(values, list) and len(values) == len(header), "RESPONSE_WIDTH") + row = dict(zip(header, values, strict=True)) + _require(row.get("GEO_ID") == declaration.geography, "RESPONSE_GEO_ID") + _require(row.get("us") == "1", "RESPONSE_NOT_THE_NATION") + _require(row.get("NAME") == declaration.published_name, "RESPONSE_PUBLISHED_NAME") + return row + + +def _published_variable(metadata: object, name: str, declaration) -> dict: + _require(isinstance(metadata, dict), "METADATA_SHAPE") + variables = metadata.get("variables") + _require(isinstance(variables, dict), "METADATA_VARIABLES") + entry = variables.get(name) + _require(isinstance(entry, dict), f"METADATA_VARIABLE_MISSING:{name}") + _require(entry.get("group") == declaration.table, f"METADATA_GROUP:{name}") + _require(entry.get("concept") == "Age and Sex", f"METADATA_CONCEPT:{name}") + return entry + + +def _count_estimate(cell: dict, variable: str) -> int: + estimate = cell["estimate"] + _require( + estimate["class"] == "numeric" and not estimate["annotation_conflict"], + f"NON_COUNT_ESTIMATE:{variable}:{estimate['class']}", + ) + value = estimate["numeric_value"] + _require( + value is not None + and math.isfinite(value) + and value >= 0 + and float(value).is_integer(), + f"ESTIMATE_NOT_A_NONNEGATIVE_COUNT:{variable}", + ) + return int(value) + + +def _standard_error( + cell: dict, variable: str, *, confidence_level: float +) -> float | None: + """A published margin becomes a standard error; a sentinel becomes null.""" + + moe = cell["moe"] + _require(not moe["annotation_conflict"], f"MOE_ANNOTATION_CONFLICT:{variable}") + if moe["class"] != "numeric": + # Controlled, suppressed and open-interval margins carry no published + # variance. Null says "unknown"; zero would claim certainty. + return None + _require(confidence_level == 0.9, "UNSUPPORTED_CONFIDENCE_LEVEL") + value = moe["numeric_value"] + _require( + value is not None and math.isfinite(value) and value >= 0, + f"MOE_NOT_NONNEGATIVE:{variable}", + ) + if value == 0: + # A zero margin is a statement about a controlled cell, not a claim + # that the count is certain. Null again, never a zero standard error. + return None + return float(value) / _MOE_90_TO_SE + + +def activate_national_age_targets( + bundle_dir: str | Path, + *, + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> TargetRegistry: + """Authenticate the pinned bundle and mint a new age-count registry. + + Args: + bundle_dir: The captured reference bundle root, holding + ``national-derived-inventory.json``, ``raw/`` and ``requests/``. + declaration: The activation authority. The default is the reviewed + one; a caller-supplied declaration is validated identically and + has no privileges of its own. + + Returns: + A new :class:`TargetRegistry` of 18 disjoint national age counts. + + Raises: + ActivationError: On any digest, identity, universe, label, schema, + reservation, selector or value violation. The bundle is never + written to, and a reserved same-source holdout can never be + activated. + """ + + root = Path(bundle_dir) + _, inventory = _read_pinned( + root / "national-derived-inventory.json", + declaration.inventory_sha256, + what="inventory", + ) + _require(isinstance(inventory, dict), "INVENTORY_SHAPE") + _require( + inventory.get("kind") == "us_national_acs_reference_inventory", + "INVENTORY_KIND", + ) + # The bundle states it is not an activation. Assert that rather than + # letting an edited descriptive field pass for authority. + _require(inventory.get("target_activation") is False, "INVENTORY_CLAIMS_ACTIVATION") + _require( + inventory.get("calibration_target_registry") == "not_created", + "INVENTORY_CLAIMS_A_REGISTRY", + ) + scope = inventory.get("scope") + _require(isinstance(scope, dict), "INVENTORY_SCOPE") + _require(scope.get("country") == "us", "INVENTORY_COUNTRY") + _require(scope.get("acs_year") == declaration.acs_year, "INVENTORY_ACS_YEAR") + _require(scope.get("acs_release") == declaration.acs_release, "INVENTORY_RELEASE") + _require( + scope.get("geography") == f"nation (for={declaration.geography_query})", + "INVENTORY_GEOGRAPHY", + ) + + tables = inventory.get("acs") + _require(isinstance(tables, dict), "INVENTORY_TABLES") + for reserved in sorted(RESERVED_FAMILIES): + held = tables.get(reserved) + _require(isinstance(held, dict), f"RESERVED_TABLE_ABSENT:{reserved}") + held_reservation = held.get("reservation") + _require( + isinstance(held_reservation, dict), + f"RESERVED_RESERVATION_SHAPE:{reserved}", + ) + _require( + held_reservation.get("status") == "reserved_same_source_holdout", + f"RESERVED_TABLE_RELABELLED:{reserved}", + ) + + entry = tables.get(declaration.table) + _require(isinstance(entry, dict), f"TABLE_ABSENT:{declaration.table}") + _require(entry.get("lineage") == "published", "TABLE_LINEAGE") + _require(entry.get("district_derived") is False, "TABLE_IS_DISTRICT_DERIVED") + _require(entry.get("geography_level") == "nation", "TABLE_GEOGRAPHY_LEVEL") + _require(entry.get("geography_id") == declaration.geography, "TABLE_GEOGRAPHY_ID") + _require( + entry.get("source_geography_query") == declaration.geography_query, + "TABLE_GEOGRAPHY_QUERY", + ) + _require(entry.get("published_name") == declaration.published_name, "TABLE_NAME") + _require(entry.get("year") == declaration.acs_year, "TABLE_YEAR") + _require( + entry.get("confidence_level") == declaration.confidence_level, + "TABLE_CONFIDENCE_LEVEL", + ) + _require( + entry.get("grouping") == "subject_age_and_sex_published_groups", + "TABLE_GROUPING", + ) + reservation = entry.get("reservation") + _require(isinstance(reservation, dict), "TABLE_RESERVATION") + _require( + reservation.get("status") == declaration.allowed_reservation_status, + f"TABLE_RESERVATION_STATUS:{reservation.get('status')}", + ) + # The bundle's own permission flags stay false and stay unread as + # authority; an inventory edited to claim permission is refused, and the + # digest above would already have rejected it. + _require(reservation.get("targets_allowed") is False, "INVENTORY_GRANTS_TARGETS") + _require(reservation.get("tuning_allowed") is False, "INVENTORY_GRANTS_TUNING") + + metadata_entry = _selected_source(inventory, declaration, "metadata") + data_entry = _selected_source(inventory, declaration, "data") + _, metadata = _authenticated_response(root, metadata_entry, kind="metadata") + _, data = _authenticated_response(root, data_entry, kind="data") + row = _national_row(data, declaration) + + inventory_cells = entry.get("cells") + _require(isinstance(inventory_cells, list), "TABLE_CELLS") + by_variable: dict[str, dict] = {} + for cell in inventory_cells: + _require(isinstance(cell, dict), "CELL_SHAPE") + variable = cell.get("variable") + _require(isinstance(variable, str), "CELL_VARIABLE") + _require(variable not in by_variable, f"DUPLICATE_CELL:{variable}") + by_variable[variable] = cell + + def derived_cell(variable: str) -> dict: + """Re-derive one cell from the authenticated bytes and cross-check it. + + The same annotation-precedence reader the district inventory uses runs + again over exactly the response that was hashed; the accepted inventory + must then agree with it cell for cell. + """ + + for suffix in ("E", "M", "EA", "MA"): + _require( + f"{variable}{suffix}" in row, + f"RESPONSE_CELL_MISSING:{variable}{suffix}", + ) + derived = { + "variable": variable, + "estimate": classify_acs_value(row[f"{variable}E"], row[f"{variable}EA"]), + "moe": classify_acs_value(row[f"{variable}M"], row[f"{variable}MA"]), + } + recorded = by_variable.get(variable) + _require(recorded is not None, f"CELL_ABSENT_FROM_INVENTORY:{variable}") + _require(derived == recorded, f"CELL_DISAGREES_WITH_INVENTORY:{variable}") + return derived + + specs: list[TargetSpec] = [] + digest = activation_digest(declaration) + for band in declaration.bands: + variable = band.variable + published = _published_variable(metadata, f"{variable}E", declaration) + _require(published.get("label") == band.label, f"PUBLISHED_LABEL:{variable}") + _require( + published.get("predicateType") == "int", f"PUBLISHED_NOT_A_COUNT:{variable}" + ) + derived = derived_cell(variable) + value = _count_estimate(derived, variable) + specs.append( + TargetSpec( + name=variable, + entity=declaration.entity, + measure=band.column, + value=float(value), + period=declaration.period, + se=_standard_error( + derived, variable, confidence_level=declaration.confidence_level + ), + source=f"{data_entry['url']} ({declaration.table} {variable}E)", + family=f"acs.{declaration.table}", + notes=( + f"activation={digest}; band={band.low}-" + f"{'+' if band.high is None else band.high}; " + f"label={band.label}; " + f"age_convention={declaration.age_convention}; " + f"uncertainty=published 90% MOE converted at " + f"{_MOE_90_TO_SE} ({_MOE_AUTHORITY}), not consumed " + f"by the current loss" + ), + metadata={ + "table": declaration.table, + "reference_sha256": declaration.inventory_sha256, + "geography": declaration.geography, + "universe": "population", + "role": declaration.role, + "evidence_scope": "source_documented", + }, + ) + ) + + # Coherence with the publisher's own total, checked and then discarded: + # the total is deliberately not activated as a nineteenth loss term. + total_variable = f"{declaration.table}_C01_001" + total = _count_estimate(derived_cell(total_variable), total_variable) + _require( + sum(int(spec.value) for spec in specs) == total, + "ACTIVATED_BANDS_DO_NOT_PARTITION_THE_PUBLISHED_TOTAL", + ) + return TargetRegistry(specs, country="us") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py new file mode 100644 index 000000000..72f27cfd7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py @@ -0,0 +1,666 @@ +"""Private native household origins, authenticated from original source members. + +ACS uses the existing archive codec. ASEC requires a separately reviewed original +HOUSEHOLD CSV member registry: prepared household HDF and PERSON-member pins are +not substitutes. The three registered members have an independently reviewed +archive/member audit and a complete accepted-population identifier bridge. + +This module owns source lineage, not population treatments or benchmark scoring. +Source capsules can be issued only by a reader. Graph transport is distinct from +source authentication; its consumer must retain the actual producer artifact edge. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, asdict, dataclass, field +from pathlib import Path + +import numpy as np + +from microcosm.build.cd_benchmark.origin import ( + AcsHouseholdOrigin, + AsecHouseholdOrigin, + SourceMember, + origin_key, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json + +from . import acs_housing_universe_source as acs +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .support_provenance import ( + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +_TOKEN = object() +_BOUND_TOKEN = object() +SOURCE_SCHEMA = "microcosm.us.native-household-origin-source.v1" +BINDING_SCHEMA = "microcosm.us.population-household-origins.v1" +MAX_SOURCE_BYTES = 512 * 1024**2 +_SHA = re.compile(r"[0-9a-f]{64}") + + +class NativeOriginError(ValueError): + """Bounded refusal code, without a private identifier or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise NativeOriginError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _integer(value: object) -> int: + _require( + not isinstance(value, (bool, np.bool_)) + and isinstance(value, (int, np.integer)), + "NATIVE_INTEGER", + ) + return int(value) + + +def _carried_native_integer(value: object) -> int: + # Union assembly may store a source-only integer column as canonical text + # beside the other arm's missing values. Accept only its lossless canonical + # decimal representation; source membership is still established against + # the independently authenticated integer H_SEQ roster, never this cast. + if isinstance(value, str): + _require( + re.fullmatch(r"[1-9][0-9]{0,17}", value) is not None, + "CARRIED_NATIVE_INTEGER", + ) + return int(value) + return _integer(value) + + +@dataclass(frozen=True) +class AsecNativeMemberPin: + """A reviewed original household member, not authority merely by construction. + + A root source audit must establish membership in the already-pinned Census + archive before adding this declaration to the closed registry below. + """ + + income_year: int + survey_year: int + canonical_member_id: str + member_name: str + archive_sha256: str + member_sha256: str + size_bytes: int + rows: int + + def __post_init__(self) -> None: + _require( + type(self.income_year) is int + and self.income_year in ASEC_EDUCATION_ASSISTANCE_ARCHIVES + and self.survey_year == self.income_year + 1, + "ASEC_MEMBER_COHORT", + ) + _require( + isinstance(self.canonical_member_id, str) + and bool(self.canonical_member_id) + and self.member_name == f"hhpub{str(self.survey_year)[-2:]}.csv", + "ASEC_MEMBER_IDENTITY", + ) + _require( + isinstance(self.member_sha256, str) + and _SHA.fullmatch(self.member_sha256) is not None + and self.archive_sha256 + == ASEC_EDUCATION_ASSISTANCE_ARCHIVES[self.income_year].zip_sha256, + "ASEC_MEMBER_ARCHIVE_BINDING", + ) + _require( + type(self.size_bytes) is int + and 0 < self.size_bytes <= MAX_SOURCE_BYTES + and type(self.rows) is int + and 0 < self.rows <= 1_000_000, + "ASEC_MEMBER_BOUNDS", + ) + + +# Root acceptance: us-asec-native-household-root-acceptance.json, SHA-256 +# f997bbdc3573ed346d0e84d29f90c7b7ded9a12720c036d2789a7d561a04d602. +# The original archive/member bytes and all accepted attachment/selected native +# household identifiers were independently verified. Tests use invented pins. +_ASEC_MEMBER_PINS: tuple[AsecNativeMemberPin, ...] = ( + AsecNativeMemberPin( + income_year=2022, + survey_year=2023, + canonical_member_id="census/cps/asec/2023/household/hhpub23.csv", + member_name="hhpub23.csv", + archive_sha256="d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + member_sha256="c88192cb3c963a90ea98022606112a01ade0eb8e2c17df4149052970a5d5246f", + size_bytes=30_259_450, + rows=88_978, + ), + AsecNativeMemberPin( + income_year=2023, + survey_year=2024, + canonical_member_id="census/cps/asec/2024/household/hhpub24.csv", + member_name="hhpub24.csv", + archive_sha256="cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + member_sha256="b12c1078e02766a4033ba1a031abdd6b9ef9df546aee5886dc8439a35abaa2ac", + size_bytes=30_448_089, + rows=89_473, + ), + AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="census/cps/asec/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256="318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + member_sha256="b5b7351d5d4e5d79ff189f1d90096b16b2d4749671c14328ec6474cfe83ce116", + size_bytes=33_298_479, + rows=88_932, + ), +) + + +def asec_member_registry() -> list[dict[str, object]]: + """Return the exact live registry for the separately scoped graph identity.""" + return [asdict(pin) for pin in _ASEC_MEMBER_PINS] + + +@dataclass(frozen=True) +class AuthenticatedNativeOriginSource: + """Owned source projection. Its constructor is not a public admission path.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self) -> dict: + _require(_sha(self.payload) == self._issued_sha256, "SOURCE_MUTATION") + return _source_document(self.payload) + + +def _source_document(payload: bytes) -> dict: + _require(type(payload) is bytes and len(payload) <= MAX_SOURCE_BYTES, "SOURCE_SIZE") + try: + value = json.loads(payload) + _require(canonical_json(value) == payload, "SOURCE_CANONICAL") + _require( + set(value) + == { + "schema", + "arm", + "members", + "records", + "source_receipt_sha256", + "release_eligible", + } + and value["schema"] == SOURCE_SCHEMA + and value["arm"] in ("acs", "asec") + and value["release_eligible"] is False + and _SHA.fullmatch(value["source_receipt_sha256"]) is not None, + "SOURCE_SHAPE", + ) + members = [SourceMember(**member) for member in value["members"]] + _require(bool(members) and len(set(members)) == len(members), "SOURCE_MEMBERS") + seen = set() + for period, native, member_index, key in value["records"]: + _require( + type(member_index) is int and 0 <= member_index < len(members), + "SOURCE_MEMBER_INDEX", + ) + origin = ( + AcsHouseholdOrigin(members[member_index], period, native) + if value["arm"] == "acs" + else AsecHouseholdOrigin(members[member_index], period, native) + ) + _require(origin_key(origin) == key, "SOURCE_ORIGIN_KEY") + identity = (period, native) + _require(identity not in seen, "SOURCE_NATIVE_KEY_AMBIGUOUS") + seen.add(identity) + _require(bool(seen), "SOURCE_EMPTY") + return value + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("SOURCE_SHAPE") from None + + +def _issue(arm, members, records, receipt_sha): + payload = canonical_json( + { + "schema": SOURCE_SCHEMA, + "arm": arm, + "members": members, + "records": records, + "source_receipt_sha256": receipt_sha, + "release_eligible": False, + } + ) + _source_document(payload) + return AuthenticatedNativeOriginSource(payload, _token=_TOKEN) + + +def produce_acs_native_origins(source_dir, *, snapshot_root, output_dir): + """Reuse the real ACS source reader; never accept caller-provided source rows.""" + source = acs.produce_acs_housing_source( + source_dir, + snapshot_root=snapshot_root, + output_dir=output_dir, + ) + receipt = json.loads(source.receipt_json) + table = source.households + members = [ + SourceMember( + # Original member identity is stable across staging aliases and + # recompression, and its original member bytes hash is explicit. + f"census/acs/pums/2024/household/{member['name']}", + member["sha256"], + ) + for member in receipt["members"]["household"] + if member["is_data"] + ] + names = [ + member["name"] + for member in receipt["members"]["household"] + if member["is_data"] + ] + indices = {name: i for i, name in enumerate(names)} + records = [] + for native, name in zip(table.SERIALNO, table.source_member, strict=True): + index = indices[name] + key = origin_key(AcsHouseholdOrigin(members[index], 2024, native)) + records.append([2024, native, index, key]) + return _issue( + "acs", + [member.as_payload() for member in members], + records, + _sha(source.receipt_json), + ) + + +def produce_asec_native_origins(member_paths: Mapping[int, Path]): + """Hash and parse registered original HOUSEHOLD CSV members only. + + No member name or digest is inferred from the person-side restoration. + Unregistered genuine sources refuse before any path is opened. + """ + pins = tuple(_ASEC_MEMBER_PINS) + _require(bool(pins), "ASEC_MEMBER_REGISTRY_UNAVAILABLE") + _require( + len({pin.income_year for pin in pins}) == len(pins) + and set(member_paths) == {pin.income_year for pin in pins}, + "ASEC_MEMBER_ROSTER", + ) + records, members = [], [] + with tempfile.TemporaryDirectory(prefix="native-asec-origin-") as tmp: + for index, pin in enumerate(pins): + # Revalidate the entire declared registry, including archive/cohort. + AsecNativeMemberPin(**asdict(pin)) + capture = Path(tmp) / f"{index}.csv" + _require( + _snapshot(member_paths[pin.income_year], capture, size=pin.size_bytes) + == pin.member_sha256, + "ASEC_MEMBER_SHA256", + ) + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + members.append(member.as_payload()) + with capture.open("r", encoding="utf-8", newline="") as handle: + rows = csv.reader(handle, strict=True) + header = next(rows, []) + _require( + bool(header) + and len(header) == len(set(header)) + and "H_SEQ" in header, + "ASEC_MEMBER_HEADER", + ) + position = header.index("H_SEQ") + count = 0 + for row in rows: + _require(len(row) == len(header), "ASEC_MEMBER_ROW_WIDTH") + token = row[position] + _require( + re.fullmatch(r"[0-9]{1,18}", token) is not None + and int(token) > 0, + "ASEC_MEMBER_NATIVE_KEY", + ) + native = int(token) + records.append( + [ + pin.income_year, + native, + index, + origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ] + ) + count += 1 + _require(count <= pin.rows, "ASEC_MEMBER_ROWS") + _require(count == pin.rows, "ASEC_MEMBER_ROWS") + _require( + _sha(capture.read_bytes()) == pin.member_sha256, "ASEC_CAPTURE_CHANGED" + ) + return _issue( + "asec", members, records, _sha(canonical_json(asec_member_registry())) + ) + + +def _population_content(frame: Frame) -> str: + # Defined over all cells/axes, effective weights and strata. Graph context + # deliberately has no Frame.metadata, graph ledger or original design anchors. + try: + normalized = Frame( + { + entity: frame.table(entity).loc[:, sorted(frame.table(entity))] + for entity in frame.entities + }, + frame.schema, + {entity: frame.resolve_weights(entity) for entity in frame.entities}, + frame.strata, + ) + return acs.frame_content_sha256(normalized) + except (ValueError, TypeError, KeyError, OverflowError): + raise NativeOriginError("POPULATION_CONTENT") from None + + +@dataclass(frozen=True) +class PopulationOriginBinding: + """Private population-grain evidence; its summary carries no native keys.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token): + _require(_token is _BOUND_TOKEN, "POPULATION_BINDING_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self): + _require(_sha(self.payload) == self._issued_sha256, "BINDING_MUTATION") + return json.loads(self.payload) + + @property + def summary(self): + return self.document["summary"] + + +def verify_population_origin_binding(frame: Frame, binding: PopulationOriginBinding): + """Reject evidence from any other full population content/ordered axes.""" + _require(type(binding) is PopulationOriginBinding, "POPULATION_BINDING_TYPE") + _require( + binding.document["population_content_sha256"] == _population_content(frame), + "POPULATION_CONTENT", + ) + + +def _entity_rows(frame, household_origins): + person = frame.person + household_ids = person.person_household_id.to_numpy() + rows = {} + for entity in frame.entities: + table = frame.table(entity) + ids = table[US_SCHEMA.entity_id_column(entity)].to_numpy() + if entity == "person": + membership = dict(zip(ids.tolist(), household_ids.tolist(), strict=True)) + elif entity == "household": + membership = dict(zip(ids.tolist(), ids.tolist(), strict=True)) + else: + pairs = person[ + [US_SCHEMA.membership_column(entity), "person_household_id"] + ].drop_duplicates() + _require( + not pairs.iloc[:, 0].duplicated().any(), "GROUP_CROSSES_HOUSEHOLDS" + ) + membership = dict(pairs.itertuples(index=False, name=None)) + _require(set(ids) == set(membership), "ENTITY_HOUSEHOLD_COVERAGE") + records = [] + effective_weights = frame.resolve_weights(entity).values + for i, identity in enumerate(ids): + hh = household_origins[_integer(membership[identity])] + channel = table[support_channel_column(entity)].iloc[i] + source_id = _integer(table[support_source_id_column(entity)].iloc[i]) + role = _integer(table[support_clone_index_column(entity)].iloc[i]) + _require( + channel == hh["channel"] and role == hh["clone_index"], + "ENTITY_HOUSEHOLD_LINEAGE", + ) + records.append( + { + "entity_id": _integer(identity), + "entity_source_id": source_id, + "clone_index": role, + "channel": channel, + "household_source_id": hh["entity_source_id"], + "origin_key": hh["origin_key"], + "weight_hex": float(effective_weights[i]).hex(), + } + ) + _require( + len({(r["entity_source_id"], r["clone_index"]) for r in records}) + == len(records), + "ENTITY_LINEAGE_DUPLICATE", + ) + rows[entity] = records + groups = { + entity: {row["entity_id"]: row for row in rows[entity]} + for entity in US_SCHEMA.group_entities + } + for position, row in enumerate(rows["person"]): + memberships = {} + for entity, lookup in groups.items(): + member_id = _integer( + person[US_SCHEMA.membership_column(entity)].iloc[position] + ) + member = lookup[member_id] + _require( + all( + member[k] == row[k] + for k in ( + "clone_index", + "channel", + "household_source_id", + "origin_key", + ) + ), + "PERSON_GROUP_LINEAGE", + ) + memberships[entity] = member["entity_source_id"] + row["membership_source_ids"] = memberships + return rows + + +def _bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Resolve every household and entity row, including all zero-weight records. + + Native columns identify original records; structural source IDs bind the + selection/clone ancestry and cannot substitute for that native identity. + """ + _require(frame.schema == US_SCHEMA and len(sources) == 2, "SOURCE_ARMS") + documents = [] + for source in sources: + _require(type(source) is AuthenticatedNativeOriginSource, "SOURCE_TYPE") + documents.append(source.document) + _require({d["arm"] for d in documents} == {"acs", "asec"}, "SOURCE_ARMS") + lookups = { + d["arm"]: {(p, n): key for p, n, _member, key in d["records"]} + for d in documents + } + household = frame.table("household") + person = frame.person + cohort_rows = person[["person_household_id", "source_year"]].drop_duplicates() + _require( + not cohort_rows.person_household_id.duplicated().any(), + "HOUSEHOLD_COHORT_AMBIGUOUS", + ) + cohorts = dict(cohort_rows.itertuples(index=False, name=None)) + weights = frame.resolve_weights("household").values + rows = [] + for i, hh in household.iterrows(): + identity = _integer(hh.household_id) + channel = hh[support_channel_column("household")] + _require(channel in lookups and identity in cohorts, "NATIVE_KEY_UNRESOLVED") + raw_period = cohorts[identity] + _require( + isinstance(raw_period, (str, int, np.integer)) + and re.fullmatch(r"[0-9]{4}", str(raw_period)) is not None, + "HOUSEHOLD_COHORT", + ) + period = int(raw_period) + native = ( + hh.SERIALNO if channel == "acs" else _carried_native_integer(hh.asec_H_SEQ) + ) + key = lookups[channel].get((period, native)) + _require(key is not None, "NATIVE_KEY_UNRESOLVED") + role = _integer(hh[support_clone_index_column("household")]) + _require(role in (0, 1), "CLONE_ROLE") + position = household.index.get_loc(i) + rows.append( + { + "household_id": identity, + "entity_source_id": _integer(hh[support_source_id_column("household")]), + "clone_index": role, + "channel": channel, + "origin_key": key, + "weight_hex": float(weights[position]).hex(), + } + ) + _require(len({r["household_id"] for r in rows}) == len(rows), "HOUSEHOLD_IDS") + entity_rows = _entity_rows(frame, {r["household_id"]: r for r in rows}) + cloned = any(r["clone_index"] for r in rows) + _require(not cloned or parent is not None, "CLONE_PARENT_REQUIRED") + pair_conserved = None + if parent is not None: + _require(type(parent) is PopulationOriginBinding, "PARENT_TYPE") + prior = parent.document + _require( + prior["source_payload_sha256"] == sorted(_sha(s.payload) for s in sources), + "PARENT_SOURCE_MISMATCH", + ) + _require( + not any(r["clone_index"] for r in prior["households"]), + "PARENT_MUST_BE_NATIVE", + ) + for entity, current in entity_rows.items(): + old = {r["entity_source_id"]: r for r in prior["entities"][entity]} + for row in current: + before = old.get(row["entity_source_id"]) + _require( + before is not None + and all( + row[k] == before[k] + for k in ("channel", "household_source_id", "origin_key") + ), + "PARENT_MEMBERSHIP_LINEAGE", + ) + if entity == "person": + _require( + row["membership_source_ids"] == before["membership_source_ids"], + "PARENT_MEMBERSHIP_LINEAGE", + ) + before_weight = float.fromhex(before["weight_hex"]) + if cloned: + half = before_weight * 0.5 + _require( + row["weight_hex"] == half.hex() and half * 2.0 == before_weight, + "CLONE_ENTITY_PAIR_WEIGHT", + ) + else: + _require( + row["weight_hex"] == before["weight_hex"], + "PARENT_ENTITY_WEIGHT", + ) + if cloned: + _require( + {(r["entity_source_id"], r["clone_index"]) for r in current} + == {(source_id, role) for source_id in old for role in (0, 1)}, + "CLONE_PAIR_COVERAGE", + ) + else: + retained_households = {r["entity_source_id"] for r in rows} + _require( + {r["entity_source_id"] for r in current} + == { + r["entity_source_id"] + for r in prior["entities"][entity] + if r["household_source_id"] in retained_households + }, + "PARENT_HOUSEHOLD_SELECTION", + ) + if cloned: + old = { + r["entity_source_id"]: float.fromhex(r["weight_hex"]) + for r in prior["households"] + } + _require( + all( + row["weight_hex"] == (old[row["entity_source_id"]] * 0.5).hex() + and (old[row["entity_source_id"]] * 0.5) * 2.0 + == old[row["entity_source_id"]] + for row in rows + ), + "CLONE_PAIR_WEIGHT", + ) + pair_conserved = True + content = _population_content(frame) + summary = { + "households": len(rows), + "original_household_origins": len({r["origin_key"] for r in rows}), + "zero_weight_households_with_lineage": sum( + float.fromhex(r["weight_hex"]) == 0 for r in rows + ), + "entity_rows": {entity: len(values) for entity, values in entity_rows.items()}, + "population_content_sha256": content, + "parent_pair_mass_conserved": pair_conserved, + "design_anchors": "not_in_frame_context_not_evaluated", + "release_eligible": False, + } + payload = canonical_json( + { + "schema": BINDING_SCHEMA, + "population_content_sha256": content, + "ordered_entity_ids_sha256": { + e: _sha(canonical_json([r["entity_id"] for r in records])) + for e, records in entity_rows.items() + }, + "source_payload_sha256": sorted(_sha(s.payload) for s in sources), + "parent_binding_sha256": None if parent is None else _sha(parent.payload), + "households": rows, + "entities": entity_rows, + "summary": summary, + } + ) + return PopulationOriginBinding(payload, _token=_BOUND_TOKEN) + + +def bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Bind every current row; malformed inputs refuse without leaking native IDs.""" + try: + return _bind_population_origins(frame, sources=sources, parent=parent) + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("POPULATION_BINDING_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py new file mode 100644 index 000000000..d5117016b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py @@ -0,0 +1,157 @@ +"""Source-visible operator column declarations, without operation imports. + +The operation modules re-export these same objects. Source boundary checks can +inspect ownership without loading transfer code or its reference tables. +""" + +from __future__ import annotations + +# Declared by qbi_inputs.py; operations retain their historical imports. + +_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", +) + +_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" + +US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( + *_GENERAL_QUALIFICATION_FLAGS, + _SSTB_QUALIFICATION_FLAG, + # Keep the classifier last so the chained QRF can condition the SSTB draw + # on the qualification flags it must agree with. + "business_is_sstb", +) + +US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + +US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( + *US_QBI_BOOLEAN_OUTPUT_COLUMNS, + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + + +# Declared by acs_transfer.py; operations retain their historical imports. + +#: Person columns the default transfer DERIVES deterministically after the +#: QRF fits (never fitted themselves). Coverage checks require them on the +#: recipient exactly like declared plan targets. +ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( + "schedule_d_capital_gain_distributions", +) + + +# Declared by puf_support.py; operations retain their historical imports. + +# Shared structural limit; importing it must not load the PUF donor operations. +PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 + +US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" + +PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + # The engine owns the realized contribution amounts through the + # IRA-limit scale and self-employment caps; the persistable leaves are + # the desired contributions, equal to the PUF's observed deductions at + # baseline (issue #278). + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + *US_QBI_OUTPUT_COLUMNS, +) + +PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", + "health_savings_account_ald", +) + + +# Declared by puf_capital_gains_tail.py; operations retain their historical imports. + +PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS = ( + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", +) + +PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS = ("unrecaptured_section_1250_gain",) + +PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN = "puf_capital_gains_tail_transfer_applied" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN = "puf_capital_gains_tail_donor_source_id" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN = ( + "puf_capital_gains_tail_donor_is_synthetic" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN = ( + "puf_capital_gains_tail_donor_filing_status_code" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN = ( + "puf_capital_gains_tail_donor_agi_band_index" +) + +PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN = "puf_capital_gains_tail_transfer_weight" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py new file mode 100644 index 000000000..530782446 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py @@ -0,0 +1,451 @@ +"""Fixture-only PUF price-result import and combined-host feature placement. + +The local envelope is transport, not scientific admission. A caller must bind +its exact producer result and source/price identities before using its arrays. +Genuine source definitions remain refused. No amount is repaired or allocated. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import numpy as np +import pandas as pd +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.model_input import encode_recipient_matrix +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights + +from . import native_household_origin as origin +from . import puf_growth as growth +from . import puf_monetary_agi_projection as agi +from . import puf_raw_source as raw +from . import support_provenance as provenance + +TARGET = "E00900" +OUTPUT = "dev_puf_schedule_c_profit_loss_2024_price" +MASK = "dev_puf_detail_recipient" +FEATURES = ("reported_wage_proxy", "mars_1", "mars_2", "mars_3", "mars_4") +SCOPE = "invented_fixture_machinery_only" +SOURCE_ADMISSION = "not_admitted_pending_basis_review" +MAGIC = b"local-puf-price-restatement-arrays-v1\n" +MAX_ROWS = 4096 +MAX_BYTES = 128 * 1024 * 1024 +MONEY_FIELDS = ( + "E00100", + "E00200", + "E00300", + "E00400", + "E00600", + "E00650", + "E00900", + "E01000", + "E01500", + "E01700", + "E02100", + "P22250", + "P23250", +) +ARRAY_DTYPES = { + **{name: " None: + """Refuse using a stable reason, without reporting source row values.""" + if not condition: + raise ValueError(reason) + + +def fixture_sources(definition_text: str, projection_text: str): + """Validate actual fixture definitions; packaged source pins cannot enter.""" + # Source documents and fit packets use distinct JSON canonical conventions. Validate each document against its own producer's exact bytes. + definition = raw.fixture_definition(json.loads(definition_text)) + projection = agi.fixture_agi_projection_document( + json.loads(projection_text), definition + ) + require( + definition.params_text == definition_text + and projection.params_text == projection_text, + "FIXTURE_SOURCE_CANONICAL", + ) + require(tuple(sorted(projection.fields)) == MONEY_FIELDS, "FIXTURE_MONEY_FIELDS") + require( + all(c.fit_admission == SOURCE_ADMISSION for c in projection.columns), + "SOURCE_ADMISSION_CHANGED", + ) + return definition, projection + + +def decode_price_arrays( + payload: bytes, *, expected: Mapping, expected_sha256: str, max_rows=MAX_ROWS +): + """Read the exact reviewed local helper envelope, with external identity pins. + + This parser does not create a GrownPufTable, repeat growth, or admit a fit. + The returned arrays are read-only views of the authenticated body bytes. + """ + require(type(max_rows) is int and 0 < max_rows <= 250_000, "PRICE_ROW_BOUND") + require( + isinstance(payload, bytes) + and len(payload) <= MAX_BYTES + and payload.startswith(MAGIC), + "PRICE_ENVELOPE", + ) + require( + codec._hash(expected_sha256) and codec.sha(payload) == expected_sha256, + "PRICE_PAYLOAD_SHA", + ) + offset = len(MAGIC) + require(len(payload) >= offset + 8, "PRICE_HEADER_LENGTH") + size = int.from_bytes(payload[offset : offset + 8], "big") + start = offset + 8 + require(0 < size <= 65536 and len(payload) >= start + size, "PRICE_HEADER_BOUND") + header = codec.decode_json(payload[start : start + size]) + require( + set(expected) == HEADER_FIELDS and set(header) == HEADER_FIELDS | {"columns"}, + "PRICE_HEADER_FIELDS", + ) + require({k: header[k] for k in HEADER_FIELDS} == dict(expected), "PRICE_IDENTITY") + require( + header["schema"] == "local.puf.price_restatement.arrays.v1" + and header["release_eligible"] is False, + "PRICE_SCOPE", + ) + for field in ("input_binding_sha256", "contract_sha256"): + require(codec._hash(header[field]), "PRICE_HEADER_SHA") + for field in ("source_head", "helper_head"): + require( + isinstance(header[field], str) + and len(header[field]) == 40 + and all(c in "0123456789abcdef" for c in header[field]), + "PRICE_CODE_ID", + ) + columns = header["columns"] + require( + isinstance(columns, dict) and set(columns) == set(ARRAY_DTYPES), + "PRICE_COLUMN_ROSTER", + ) + cursor, rows, result = start + size, None, {} + for name in sorted(ARRAY_DTYPES): + entry = columns[name] + require( + isinstance(entry, dict) and set(entry) == {"dtype", "rows", "sha256"}, + "PRICE_COLUMN_FIELDS", + ) + require( + entry["dtype"] == ARRAY_DTYPES[name] + and type(entry["rows"]) is int + and 0 < entry["rows"] <= max_rows, + "PRICE_COLUMN_TYPE_ROWS", + ) + rows = entry["rows"] if rows is None else rows + require(entry["rows"] == rows, "PRICE_ROW_ALIGNMENT") + end = cursor + rows * np.dtype(entry["dtype"]).itemsize + require(end <= len(payload), "PRICE_BODY_TRUNCATED") + body = payload[cursor:end] + require(codec.sha(body) == entry["sha256"], "PRICE_BODY_SHA") + values = np.frombuffer(body, dtype=entry["dtype"]) + require(np.isfinite(values).all(), "PRICE_NONFINITE") + result[name] = values + cursor = end + require(cursor == len(payload), "PRICE_TRAILING_BYTES") + return result + + +def donor_frame(arrays: Mapping, status, decoded, projection) -> Frame: + """One computational person/return wrapper, with explicit raw S006 DESIGN weights.""" + require(projection.route == "test_fixture", "GENUINE_FIT_NOT_ADMITTED") + return _donor_frame(arrays, status, decoded, projection, scope=SCOPE) + + +def _donor_frame(arrays, status, decoded, projection, *, scope): + """Shared mechanical checks; the entry point must qualify its source/recipe.""" + known = np.asarray(decoded.amount_known, dtype=bool) + require( + np.array_equal(status.typed["RECID"], decoded.typed["RECID"]), + "DONOR_STATUS_JOIN", + ) + require( + np.array_equal(status.typed["disclosure_aggregate"], ~known), "DONOR_UNIVERSE" + ) + ids = status.typed["RECID"][known] + require( + len(ids) == len(np.unique(ids)) + and np.array_equal(ids, arrays["RECID"]) + and np.array_equal(ids, arrays[growth.PROVENANCE_RECID_COLUMN]), + "DONOR_RECID_ORDER", + ) + aggregate_ids = status.typed["RECID"][~known] + require( + len(aggregate_ids) == 4 + and len(np.unique(aggregate_ids)) == 4 + and set(aggregate_ids) == set(raw.PUF_AGGREGATE_RECIDS), + "DONOR_AGGREGATE_ROSTER", + ) + for field, delivered in ( + ("S006", "S006_delivered_int64"), + ("FLPDYR", "FLPDYR_delivered_int16"), + ("demographic_status", "demographic_status_delivered_int8"), + ): + original = status.typed[field][known] + require( + np.array_equal(original, arrays[delivered]) + and np.array_equal(original, arrays[field]), + "DONOR_CARRIED_STATUS", + ) + require( + np.array_equal( + decoded.typed["E00100"][known], arrays[growth.PROVENANCE_SOURCE_AGI_COLUMN] + ), + "DONOR_SOURCE_AGI", + ) + weights = arrays["S006_delivered_int64"].astype("float64") + require( + np.all(arrays["S006_delivered_int64"] >= 0) + and np.all(arrays["S006_delivered_int64"] <= 2**53) + and np.array_equal(weights.astype("int64"), arrays["S006_delivered_int64"]) + and weights.sum() > 0, + "DONOR_DESIGN_WEIGHTS", + ) + wage = arrays["E00200"] + require(np.isfinite(wage).all() and np.all(wage >= 0), "DONOR_WAGE_FEATURE") + mars = status.typed["MARS"][known] + require(np.isin(mars, (1, 2, 3, 4)).all(), "DONOR_MARS") + table = pd.DataFrame( + { + "tax_unit_id": ids.copy(), + FEATURES[0]: wage.copy(), + **{f"mars_{i}": (mars == i).astype("float64") for i in range(1, 5)}, + TARGET: arrays[TARGET].copy(), + } + ) + return Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + {"person_id": ids.copy(), "person_tax_unit_id": ids.copy()} + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(weights, WeightKind.DESIGN)}, + metadata={ + "scope": scope, + "source_projection_sha256": projection.sha256, + "source_projection_document": projection.canonical.decode("utf-8"), + "carrier": "one technical person per return; not a taxpayer count", + "weight_units": "delivered_S006_integer_hundredths", + }, + ) + + +def population_content(frame: Frame) -> str: + """Full materialized cells/axes/effective weights/strata; no metadata shortcuts.""" + return origin._population_content(frame) + + +def recipient_matrix(frame: Frame, *, role_column=None, person_wages=None): + """Use every detail unit, including zero-weight rows, and actual filing roles.""" + require(frame.schema == US_SCHEMA, "HOST_SCHEMA") + # Build a manifest from the current native roster only for validation. The + # separate fixture source edge binds that roster's authority and full bytes. + tables = {e: frame.table(e) for e in frame.entities} + for entity, table in tables.items(): + role = table[provenance.support_clone_index_column(entity)] + require( + role.dtype == np.dtype("int64") and role.isin((0, 1)).all(), + "HOST_CLONE_TYPE", + ) + manifest = provenance.spine_assembly_manifest( + { + e: t.loc[t[provenance.support_clone_index_column(e)].eq(0)] + for e, t in tables.items() + }, + channels=("acs", "asec"), + ) + checked = Frame( + tables, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata, + metadata=manifest, + ) + provenance.validate_assembly_provenance(checked, boundary="invented_puf_detail") + person, units = tables["person"], tables["tax_unit"] + if person_wages is not None: + require( + type(person_wages) is pd.Series + and person_wages.dtype == np.dtype("float64") + and person_wages.index.dtype == np.dtype("int64") + and person_wages.index.is_unique + and np.array_equal(person_wages.index.to_numpy(), person.person_id), + "HOST_WAGE_ROW_AXIS", + ) + clone_role = person[provenance.support_clone_index_column("person")] + wage_values = person_wages.to_numpy() + require( + np.array_equal( + wage_values[clone_role.eq(0)].view("uint64"), + wage_values[clone_role.eq(1)].view("uint64"), + ), + "HOST_WAGE_PAIR_BYTES", + ) + pairs = {} + for entity, table in tables.items(): + src, role = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + require(table[src].dtype == np.dtype("int64"), "HOST_SOURCE_ID_TYPE") + require(not table.duplicated([src, role]).any(), "HOST_ROLE_PAIR_DUPLICATE") + native, detail = table.loc[table[role].eq(0)], table.loc[table[role].eq(1)] + require( + len(native) == len(detail) and np.array_equal(native[src], detail[src]), + "HOST_ROLE_PAIR_COVERAGE", + ) + structural = {US_SCHEMA.entity_id_column(entity), role} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + for name in table.columns: + if name not in structural: + require( + native[name] + .reset_index(drop=True) + .equals(detail[name].reset_index(drop=True)), + "HOST_COPIED_SOURCE_CELL", + ) + weights = frame.resolve_weights(entity).values + require( + np.array_equal( + weights[table[role].eq(0)].view("uint64"), + weights[table[role].eq(1)].view("uint64"), + ), + "HOST_PAIR_WEIGHTS", + ) + pairs[entity] = dict( + zip( + table[US_SCHEMA.entity_id_column(entity)], + zip(table[src], table[role], strict=True), + strict=True, + ) + ) + for group in US_SCHEMA.group_entities: + member = US_SCHEMA.membership_column(group) + for pid, gid in zip(person.person_id, person[member], strict=True): + require(pairs["person"][pid][1] == pairs[group][gid][1], "HOST_MEMBER_ROLE") + by_source = {} + for pid, gid in zip(person.person_id, person[member], strict=True): + source, role = pairs["person"][pid] + group_source = pairs[group][gid][0] + if role == 0: + by_source[source] = group_source + else: + require(by_source.get(source) == group_source, "HOST_CLONE_MEMBERSHIP") + require( + set(units[provenance.support_channel_column("tax_unit")]) == {"acs", "asec"}, + "HOST_BOTH_ARMS", + ) + mask = units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + selected = units.loc[mask] + values = [] + for row in selected.itertuples(index=False): + members = person.loc[person.person_tax_unit_id.eq(row.tax_unit_id)] + status = row.filing_status_input + require(status in MARS, "HOST_FILING_STATUS") + if role_column is None: + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ): + require( + name in members + and members[name].dtype == np.dtype("bool") + and not members[name].isna().any(), + "HOST_FILING_ROLE", + ) + head, spouse, dependent = ( + members[name].to_numpy() + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ) + ) + else: + require( + role_column == "tax_unit_role_input" and role_column in members, + "HOST_ROLE_INPUT_COLUMN", + ) + roles = members[role_column] + require( + pd.api.types.is_string_dtype(roles.dtype) + and not roles.isna().any() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all(), + "HOST_ROLE_INPUT_VALUES", + ) + head, spouse, dependent = ( + roles.eq(name).to_numpy(dtype=bool) + for name in ("HEAD", "SPOUSE", "DEPENDENT") + ) + require( + np.all(head.astype(int) + spouse.astype(int) + dependent.astype(int) == 1) + and head.sum() == 1 + and spouse.sum() == int(status == "JOINT"), + "HOST_FILING_STRUCTURE", + ) + channel = getattr(row, provenance.support_channel_column("tax_unit")) + name = ( + "asec_reported_wage_income_2024_price" + if channel == "asec" + else "employment_income_before_lsr" + ) + earners = head | spouse + if person_wages is None: + require(name in members, "HOST_MISSING_FEATURE") + wage = members.loc[earners, name] + else: + wage = person_wages.loc[members.loc[earners, "person_id"]] + require( + pd.api.types.is_numeric_dtype(wage) + and not pd.api.types.is_bool_dtype(wage) + and not pd.api.types.is_complex_dtype(wage) + and not wage.isna().any(), + "HOST_MISSING_FEATURE", + ) + a = wage.to_numpy(dtype="float64") + require(np.isfinite(a).all() and (a >= 0).all(), "HOST_NONFINITE_FEATURE") + amount = float(a.sum()) + require(np.isfinite(amount), "HOST_FEATURE_OVERFLOW") + values.append([amount, *(float(MARS[status] == i) for i in range(1, 5))]) + ids = selected.tax_unit_id.to_numpy(copy=True) + features = pd.DataFrame( + values, index=pd.Index(ids), columns=FEATURES, dtype="float64" + ) + return encode_recipient_matrix(features, entity="tax_unit", entity_ids=ids), mask diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py new file mode 100644 index 000000000..a2ac9942f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py @@ -0,0 +1,846 @@ +"""Qualified transport for one development-only PUF conditional diagnostic. + +Callers supply independently reviewed content pins. Receipt labels or fixture +definitions cannot replace those pins. No function opens a file or fits a model. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import numpy as np +import pandas as pd +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input + +from . import asec_current_money as current_money +from . import graph_asec_income as asec_income +from . import native_household_origin as origin +from . import puf_detail_transfer as detail +from . import puf_monetary_agi_projection as agi +from . import puf_raw_source as raw +from . import support_provenance as provenance +from .graph_context import _row_identity + +require = detail.require +RECIPE_ID = "us.puf.schedule_c_wage_mars_diagnostic.v1" +SCOPE = "development_conditional_support_not_tax_inputs" +SOURCE_HEAD = "c5c34a048fc38c01840b2da5454385e0a51d1c49" +HELPER_HEAD = "dea036278cc3f7c29c99da67bcadd968822c1d7a" +MAX_DONORS = 250_000 +ROLES = ( + "acceptance", + "proposal", + "config", + "completion", + "cold", + "required", + "definition", + "projection_definition", + "status", + "projection", + "arrays", +) + + +def recipe_document(): + """A diagnostic modeling choice, never a modification of source admission.""" + return { + "id": RECIPE_ID, + "scope": SCOPE, + "target": detail.TARGET, + "features": list(detail.FEATURES), + "output": detail.OUTPUT, + "trees": 2, + "seed": 578, + "zero_atol": 0.0, + "donor_weight": "delivered_S006_integer_hundredths_DESIGN", + "wage_proxy": "head_and_actual_JOINT_spouse_only_dependents_excluded", + "host_role_source": "microunit_tax_unit_role_input", + "filing_mapping": detail.MARS, + "missing_wage": "refuse_selected_earner_no_age_only_zero", + "prices": "2015_file_CPI_U_to_2024_development_prices", + "source_period": "unresolved_raw_FLPDYR", + "source_fit_admission": detail.SOURCE_ADMISSION, + "population_meaning": "potential_filing_unit_support_not_observed_filers", + "tax_input_admission": False, + "scientific_launch_admission": False, + "release_eligible": False, + } + + +def parse_document(payload): + """Strict JSON, allowing the reviewed helper's formatted receipt encoding.""" + require( + type(payload) is bytes and 0 < len(payload) <= 16_000_000, + "DIAGNOSTIC_DOCUMENT_BOUND", + ) + + def pairs(items): + result = {} + for key, value in items: + require(key not in result, "DIAGNOSTIC_DUPLICATE_KEY") + result[key] = value + return result + + def bad(_): + raise ValueError("DIAGNOSTIC_NONFINITE_JSON") + + value = json.loads(payload, object_pairs_hook=pairs, parse_constant=bad) + require(isinstance(value, dict), "DIAGNOSTIC_DOCUMENT_TYPE") + return value + + +def content_pin(payload): + return {"bytes": len(payload), "sha256": codec.sha(payload)} + + +def check_pin(payload, pin, reason): + require( + isinstance(pin, Mapping) + and type(pin.get("bytes")) is int + and codec._hash(pin.get("sha256")) + and content_pin(payload) == {k: pin[k] for k in ("bytes", "sha256")}, + reason, + ) + + +def qualify_price_transport(payloads, expected_pins, *, route, recipe): + """Validate DEA completion → config/source → grown-array transport. + + Expected pins are execution inputs established outside this parser. The + production route accepts only the packaged original source definitions; + tests use a visibly different acceptance/mode with actual fixture codecs. + """ + require( + codec.encode_json(recipe) == codec.encode_json(recipe_document()), + "DIAGNOSTIC_RECIPE", + ) + require(route in ("packaged", "test_fixture"), "DIAGNOSTIC_SOURCE_ROUTE") + require(set(payloads) == set(expected_pins) == set(ROLES), "PRICE_TRANSPORT_ROSTER") + for name in ROLES: + check_pin(payloads[name], expected_pins[name], "PRICE_TRANSPORT_PIN:" + name) + documents = {n: parse_document(payloads[n]) for n in ROLES[:6]} + acceptance, proposal, config, completed, cold, warm = ( + documents[n] for n in ROLES[:6] + ) + genuine = route == "packaged" + require( + acceptance["status"] + == ( + "ACCEPTED_GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT" + if genuine + else "ACCEPTED_INVENTED_DEVELOPMENTAL_PRICE_RESTATEMENT" + ) + and acceptance["release_eligible"] is False + and acceptance["fit_admission"] == detail.SOURCE_ADMISSION + and config["mode"] == ("genuine" if genuine else "bound_invented") + and completed["genuine_input_bound"] is genuine + and completed["status"] + == ( + "GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT_COMPLETED" + if genuine + else "INVENTED_PRICE_RESTATEMENT_PREPARATION_PASSED" + ), + "PRICE_TRANSPORT_AUTHORITY_SCOPE", + ) + for name in ( + "config", + "completion", + "cold", + "required", + "arrays", + "status", + "projection", + "definition", + "projection_definition", + ): + require( + any( + content_pin(payloads[name]) == {k: p[k] for k in ("bytes", "sha256")} + for p in acceptance["saved_file_pins_checked"] + ), + "PRICE_ACCEPTANCE_PIN:" + name, + ) + check_pin( + payloads["proposal"], acceptance["reviewed_proposal"], "PRICE_PROPOSAL_PIN" + ) + for name, receipt in (("cold", cold), ("required", warm)): + require( + completed[name]["sha256"] == codec.sha(payloads[name]), "PRICE_CHILD_PIN" + ) + require(receipt["phase"] == name, "PRICE_CHILD_PHASE") + check_pin(payloads["config"], completed["config_pin"], "PRICE_CONFIG_PIN") + require( + {k: v for k, v in config.items() if k != "execution"} == proposal["config"] + and proposal["price_execution_authorized"] is False + and proposal["release_eligible"] is False, + "PRICE_REVIEWED_CONFIGURATION", + ) + require( + config["inputs"] == cold["input_binding"] == warm["input_binding"], + "PRICE_SOURCE_INPUTS", + ) + require( + set(config["inputs"]) + == {"definition", "projection_definition", "status", "projection"}, + "PRICE_SOURCE_ROSTER", + ) + for name, pin in config["inputs"].items(): + check_pin(payloads[name], pin, "PRICE_SOURCE_PIN:" + name) + source = config["bound_source"] + require( + source["route"] == route and source["invented"] is (not genuine), + "PRICE_BOUND_ROUTE", + ) + source_sha = codec.sha(codec.encode_json(source)) + require( + source_sha + == config["binding_sha256"] + == cold["bound_source_identity"] + == warm["bound_source_identity"] + == completed["bound_source_identity"], + "PRICE_BOUND_SOURCE_SHA", + ) + runtime_keys = ( + "source_head", + "helper_head", + "python", + "executable", + "executable_sha256", + "versions", + ) + runtime = {k: cold["runtime"][k] for k in runtime_keys} + require( + all( + {k: item["runtime"][k] for k in runtime_keys} == runtime + for item in (warm, completed) + ), + "PRICE_RUNTIME_BINDING", + ) + require( + runtime["source_head"] == SOURCE_HEAD + and runtime["helper_head"] == HELPER_HEAD + and runtime["executable_sha256"] + == "bf4ea26231212e330b8e9b6f24969a30f894fec88665bf42754070d114ac7fc6" + and runtime["versions"] == {"numpy": "2.4.6", "pandas": "3.0.3"}, + "PRICE_REVIEWED_RUNTIME", + ) + for item in (cold, warm, completed): + isolation = item["runtime"]["bytecode_isolation"] + require( + isolation["isolated"] is True + and isolation["dont_write_bytecode"] is True + and isolation["absent_absolute_prefix"].startswith("/"), + "PRICE_BYTECODE_ISOLATION", + ) + require( + cold["runtime"]["bytecode_isolation"]["absent_absolute_prefix"] + != warm["runtime"]["bytecode_isolation"]["absent_absolute_prefix"], + "PRICE_FRESH_PROCESS_PREFIX", + ) + require( + cold["pid"] != warm["pid"] + and warm["all_hits"] + and warm["kernel_calls"] == [] + and warm["raising_replay_stubs"] is True + and all(v == 0 for v in warm["transform_calls"].values()), + "PRICE_REQUIRED_PROOF", + ) + for name in ( + "contract", + "array_identity", + "graph_artifacts", + "grown_artifact", + "excluded_lexical_artifact", + ): + require(cold[name] == warm[name], "PRICE_COLD_WARM_IDENTITY:" + name) + contract = cold["contract"] + require(source["contract"] == contract, "PRICE_SOURCE_CONTRACT") + require( + source["runtime"]["executable_sha256"] == runtime["executable_sha256"] + and source["runtime"]["package_versions"] == runtime["versions"], + "PRICE_SOURCE_RUNTIME", + ) + require( + contract["ratio"] == [313689, 237017] + and contract["ratio_bits"] == "0aa9e810012df53f" + and contract["release_eligible"] is False + and contract["per_record_preliminary_inflation"] is False + and contract["nominal_income_growth_claim"] is False, + "PRICE_BASIS", + ) + require( + acceptance["source_head"] == SOURCE_HEAD + and acceptance["helper_head"] == HELPER_HEAD + and config["helper_head"] == HELPER_HEAD, + "PRICE_REVIEWED_CODE", + ) + if genuine: + definition, projection = ( + raw.packaged_definition(), + agi.packaged_agi_projection(), + ) + else: + definition = raw.fixture_definition(parse_document(payloads["definition"])) + projection = agi.fixture_agi_projection_document( + parse_document(payloads["projection_definition"]), definition + ) + # Source files may use formatted JSON; their exact input pins were checked above. + require( + parse_document(payloads["definition"]) + == raw.definition_document_json(definition), + "PRICE_DEFINITION", + ) + require( + parse_document(payloads["projection_definition"]) + == json.loads(projection.canonical), + "PRICE_PROJECTION_DEFINITION", + ) + require( + cold["source_definition_sha256"] == definition.sha256 + and cold["projection_definition_sha256"] == projection.sha256, + "PRICE_DEFINITION_IDENTITY", + ) + require( + source["source"]["head"] == SOURCE_HEAD + and source["source"]["definition_canonical_sha256"] == definition.sha256 + and source["source"]["projection_canonical_sha256"] == projection.sha256, + "PRICE_DECLARED_SOURCE_IDENTITY", + ) + metadata = source["source_column_metadata"] + require( + set(metadata) == set(detail.MONEY_FIELDS) + and codec.sha(codec.encode_json(metadata)) == source["source_basis_sha256"] + and cold["source_column_metadata_preserved"] + == warm["source_column_metadata_preserved"] + == metadata, + "PRICE_SOURCE_METADATA", + ) + for value in metadata.values(): + require( + value["fit_admission"] == detail.SOURCE_ADMISSION + and value["period_semantics"] == "unresolved_raw_FLPDYR" + and value["amount_units"] == "whole_usd_source_year" + and value["grain"] == "source_return" + and value["projection_sha256"] == projection.sha256, + "PRICE_SOURCE_MEANING", + ) + rows = config["ordinary_rows"] + require( + type(rows) is int + and 0 < rows <= MAX_DONORS + and rows + == source["ordinary_rows"] + == acceptance["ordinary_rows"] + == completed["ordinary_rows"] + and acceptance["excluded_aggregate_rows"] == completed["aggregate_rows"] == 4, + "PRICE_ORDINARY_ROSTER", + ) + check_pin(payloads["arrays"], cold["grown_artifact"], "PRICE_EXPORT_PIN") + header = { + "schema": "local.puf.price_restatement.arrays.v1", + "input_binding_sha256": codec.sha(codec.encode_json(config["inputs"])), + "contract_sha256": contract["sha256"], + "source_head": SOURCE_HEAD, + "helper_head": HELPER_HEAD, + "release_eligible": False, + } + arrays = detail.decode_price_arrays( + payloads["arrays"], + expected=header, + expected_sha256=codec.sha(payloads["arrays"]), + max_rows=rows, + ) + require(len(arrays["RECID"]) == rows, "PRICE_EXACT_ROWS") + for name, values in arrays.items(): + require( + cold["array_identity"][name] + == { + "dtype": values.dtype.str, + "rows": len(values), + "sha256": codec.sha(values.tobytes()), + }, + "PRICE_ARRAY_RECEIPT", + ) + status = raw.decode_return_status(payloads["status"], definition) + decoded = agi.decode_agi_projection( + payloads["projection"], + projection, + expected_status_artifact_sha256=codec.sha(payloads["status"]), + ) + require( + metadata == agi._thawed(decoded.column_metadata), + "PRICE_ORIGINAL_METADATA_BINDING", + ) + frame = detail._donor_frame(arrays, status, decoded, projection, scope=SCOPE) + binding = { + "schema": "microcosm.us.puf_diagnostic_price_transport.v1", + "route": route, + "recipe": recipe, + "input_pins": {n: content_pin(payloads[n]) for n in ROLES}, + "source_column_metadata": metadata, + "source_projection_document": projection.canonical.decode(), + "original_source_admission": detail.SOURCE_ADMISSION, + "contract": contract, + "source_binding_sha256": source_sha, + "ordinary_rows": rows, + "donor_model_input_sha256": donor_model_input_sha256(frame), + "release_eligible": False, + } + return frame, codec.encode_json(binding) + + +def donor_model_input_sha256(frame): + """Exact donor IDs/features/target and DESIGN resampling weights.""" + table = frame.table("tax_unit").loc[:, [*detail.FEATURES, detail.TARGET]].copy() + table.index = pd.Index( + frame.table("tax_unit").tax_unit_id.to_numpy(dtype="int64"), name="tax_unit_id" + ) + payload = model_input.encode_recipient_matrix( + table.loc[:, [*detail.FEATURES, detail.TARGET]], + entity="tax_unit", + entity_ids=table.index.to_numpy(dtype="int64"), + ) + weight = frame.weights_for("tax_unit") + return codec.sha( + codec.encode_json( + { + "matrix_sha256": codec.sha(payload), + "weight_kind": weight.kind.value, + "weight_sha256": codec.sha(weight.values.astype("= 0) & (age <= 99) & (age == np.floor(age))).all() + and np.array_equal(age, _numeric(person.age, "ACS_WAGE_AGE")), + "ACS_WAGE_AGE", + ) + require(person.source_year.astype(str).eq("2024").all(), "ACS_WAGE_COHORT") + wage = _numeric(person.WAGP, "ACS_WAGE_RAW_TYPE", nullable=True) + adjustment = _numeric(person.ADJINC, "ACS_WAGE_ADJINC_TYPE", nullable=True) + carried = _numeric( + person.employment_income_before_lsr, "ACS_WAGE_CARRIED_TYPE", nullable=True + ) + observed = ~np.isnan(wage) + require( + np.isfinite(wage[observed]).all() + and (wage[observed] >= 0).all() + and np.isfinite(adjustment[observed]).all() + and (adjustment[observed] > 0).all(), + "ACS_WAGE_DOMAIN", + ) + require(not (observed & (age < 15)).any(), "ACS_WAGE_OUTSIDE_UNIVERSE_OBSERVED") + expected = wage * (adjustment / 1_000_000.0) + require( + np.array_equal(np.isnan(carried), ~observed) + and np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_WAGE_ADJUSTED_BINDING", + ) + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "age_sha256": codec.sha(age.astype("= 15).tolist(), + "missing_treatment": "preserved_no_zero_substitution", + "arithmetic": "WAGP*(ADJINC/1000000)", + "source_period_equivalence_claim": False, + } + + +def asec_wage_evidence(person, values): + """Reconstruct the accepted income artifact, then join its native person axis. + + ``values`` are executor-checked typed artifacts. H and its T1 descendant + have legitimate differing source ancestry; the accepted readers bind the + prepared receipt, current-money buffers, cohort and exact income payload. + """ + from . import graph_composed_asec_binding as asec_binding + from . import graph_composed_asec_measures as measures + + binding = measures._binding(values) + prepared = measures._prepared(values["prepared_receipt"]) + document = measures._document(values["frame_context"]) + measures._bound_population(document, binding) + selected = measures._selected_money(binding, prepared, values) + rows = asec_binding.bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + original = rows.array("person", "original_ids") + + def integer(name): + return asec_binding._int64_view(person[name], reason="ASEC_HOST:" + name) + + require( + np.array_equal(integer(provenance.spine_source_id_column("person")), original), + "ASEC_HOST_NATIVE_PERSON_ROSTER", + ) + composed = integer(provenance.support_source_id_column("person")) + require( + _row_identity(pd.DataFrame({"person_id": composed}), "person")[ + "ordered_ids_sha256" + ] + == binding["arm"]["person"]["composed_ordered_ids_sha256"], + "ASEC_HOST_COMPOSED_ROSTER", + ) + text_year = person.source_year.astype(str) + require(text_year.isin(("2022", "2023", "2024")).all(), "ASEC_HOST_COHORT") + years = text_year.to_numpy().astype("int64") + income = asec_income.bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + result = asec_income.read_reported_income( + values["reported_income"].payload, + selected, + income, + person_ids=original, + income_years=years, + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=codec.sha(values["prepared_receipt"].payload), + selection_sha256=codec.sha(values["asec_binding"].payload), + selected_money_sha256=codec.sha(values["selected_current_money"].payload), + income_payload_sha256=codec.sha(values["income_observations"].payload), + source_producer_key=values["prepared_receipt"].producer_key, + selection_producer_key=values["asec_binding"].producer_key, + frame_context_sha256=codec.sha(values["frame_context"].payload), + ) + require( + np.array_equal(integer("A_AGE"), result.array("A_AGE")) + and np.array_equal( + _numeric(person.age, "ASEC_HOST_AGE"), result.array("A_AGE") + ), + "ASEC_HOST_NATIVE_AGE", + ) + name = "asec_reported_wage_income_2024_price" + actual = _numeric(person[name], "ASEC_HOST_WAGE_TYPE") + require( + np.isfinite(actual).all() + and (actual >= 0).all() + and np.array_equal(actual.view("uint64"), result.array(name).view("uint64")), + "ASEC_HOST_WAGE_BINDING", + ) + require((result.array("WSAL_VAL.validity") == 1).all(), "ASEC_HOST_WAGE_VALIDITY") + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "source_native_person_sha256": codec.sha(original.tobytes()), + "income_year_sha256": codec.sha(years.tobytes()), + "reported_income_sha256": codec.sha(values["reported_income"].payload), + "wage_sha256": codec.sha(actual.tobytes()), + "evidence_axes": { + n: result.array("WSAL_VAL." + n).tolist() + for n in ("status", "validity", "zero_origin") + }, + "monetary_basis": json.loads(result.header)["monetary_basis"], + "missing_treatment": "refuse_no_evidence_upgrade", + "source_period_equivalence_claim": False, + } + + +def recipient_matrix(frame): + """Read the actual microunit role carrier; do not calculate policy roles.""" + return detail.recipient_matrix(frame, role_column="tax_unit_role_input") + + +def qualify_host_population(frame, values): + """Bind the complete untouched clone pool before selecting model features. + + Reconstruct native membership from typed original-source projections and + native-parent binding. Exact content comparison also catches a caller's + omitted declared column; the materialized verifier remains mandatory. + """ + sources = [] + for name in ("acs", "asec"): + value = values[name] + require( + origin._source_document(value.payload)["arm"] == name, "HOST_ORIGIN_ARM" + ) + sources.append( + origin.AuthenticatedNativeOriginSource(value.payload, _token=origin._TOKEN) + ) + parent_doc = codec.decode_json(values["parent_binding"].payload) + require(parent_doc["schema"] == origin.BINDING_SCHEMA, "HOST_PARENT_SCHEMA") + require( + parent_doc["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec") + }, + "HOST_NATIVE_ORIGIN_PRODUCERS", + ) + parent = origin.PopulationOriginBinding( + values["parent_binding"].payload, _token=origin._BOUND_TOKEN + ) + expected = origin.bind_population_origins( + frame, sources=sources, parent=parent + ).document + recorded = codec.decode_json(values["clone_binding"].payload) + require( + { + k: v + for k, v in recorded.items() + if k not in ("graph_parent_edges", "population_node") + } + == expected, + "HOST_FULL_ORIGIN_BINDING", + ) + require( + recorded["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec", "parent_binding") + }, + "HOST_ORIGIN_PRODUCERS", + ) + # Exact pair checks include every carried cell, linked entity and weight, + # including zero-weight rows. This also refuses incomplete selected wages. + matrix, mask = recipient_matrix(frame) + native = frame.person.loc[ + frame.person[provenance.support_clone_index_column("person")].eq(0) + ] + channel = native[provenance.support_channel_column("person")] + acs_person = native.loc[channel.eq("acs")].reset_index(drop=True) + asec_person = native.loc[channel.eq("asec")].reset_index(drop=True) + require(len(acs_person) > 0 and len(asec_person) > 0, "HOST_BOTH_ARMS") + evidence = { + "acs": acs_wage_evidence(acs_person), + "asec": asec_wage_evidence(asec_person, values), + } + return { + "schema": "microcosm.us.puf_diagnostic_host_features.v1", + "recipe": recipe_document(), + "full_population_content_sha256": detail.population_content(frame), + "origin_binding_sha256": codec.sha(values["clone_binding"].payload), + "wage_evidence": evidence, + "matrix_sha256": codec.sha(matrix), + "recipients": int(mask.sum()), + "all_clone_rows_including_zero_weights": True, + "release_eligible": False, + } + + +CURRENT_SURVEY_ROUTE = "authenticated_current_survey_v1" +CURRENT_SURVEY_PROJECTION_PROTOCOL = "microcosm.us.survey-puf-host-projection.v1" +CURRENT_SURVEY_MAX_BYTES = 64 * 1024**2 + + +def current_survey_recipient_matrix(frame, projection): + """Pure typed-transport consumer; live owner verification remains mandatory. + + Both this parser and the existing matrix validate every clone pair. The + projection retains current ASEC evidence bytes. ACS arithmetic/missingness + remains the existing WAGP/ADJINC contract, including under-15 refusal for a + missing selected earner. No serialized field confers source admission. + """ + require( + type(projection) is bytes and 0 < len(projection) <= CURRENT_SURVEY_MAX_BYTES, + "SURVEY_HOST_PROJECTION_BOUND", + ) + document = codec.decode_json(projection) + require( + set(document) + == { + "protocol", + "route", + "preparation_sha256", + "allocation_sha256", + "clone_sha256", + "asec", + "release_eligible", + }, + "SURVEY_HOST_PROJECTION_FIELDS", + ) + require( + document["protocol"] == CURRENT_SURVEY_PROJECTION_PROTOCOL + and document["route"] == CURRENT_SURVEY_ROUTE + and document["release_eligible"] is False + and all( + codec._hash(document[k]) + for k in ("preparation_sha256", "allocation_sha256", "clone_sha256") + ), + "SURVEY_HOST_PROJECTION_CONTRACT", + ) + asec = document["asec"] + require( + type(asec) is dict + and set(asec) + == { + "preparation_sha256", + "asec_native_sha256", + "money_header", + "money_header_sha256", + "domain", + "zero_origin_code", + "columns", + "rows", + }, + "SURVEY_WAGE_FIELDS", + ) + require( + asec["preparation_sha256"] == document["preparation_sha256"] + and codec._hash(asec["asec_native_sha256"]) + and codec.sha(codec.encode_json(asec["money_header"])) + == asec["money_header_sha256"] + and asec["money_header"]["target_year"] == 2024 + and asec["money_header"]["semantic"] == "annual_current_money" + and asec["columns"] + == [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "SURVEY_WAGE_CONTRACT", + ) + people = frame.person + native_role = people[provenance.support_clone_index_column("person")].eq(0) + native_acs = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("acs") + ] + native_asec = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("asec") + ] + acs_evidence = acs_wage_evidence(native_acs) + rows = asec["rows"] + require( + type(rows) is list and len(rows) == len(native_asec) > 0, "SURVEY_WAGE_ROSTER" + ) + require( + type(asec["domain"]) is dict + and set(asec["domain"]) + == {"name", "entity", "column", "grain", "minimum", "maximum", "zero_semantics"} + and asec["domain"]["name"] == "WSAL_VAL" + and asec["domain"]["entity"] == "person" + and type(asec["zero_origin_code"]) is int, + "SURVEY_WAGE_DOMAIN", + ) + amounts = {} + for row in rows: + require( + type(row) is list + and len(row) == 7 + and all(type(row[i]) is int for i in (0, 1, 2, 4, 5, 6)) + and row[2] == 2024 + and row[5] == 1 + and all(0 <= row[i] <= 255 for i in (4, 5, 6)) + and type(row[3]) is str + and len(row[3]) == 16 + and set(row[3]) <= set("0123456789abcdef"), + "SURVEY_WAGE_ROW", + ) + require(row[0] not in amounts, "SURVEY_WAGE_DUPLICATE") + amount = np.frombuffer(bytes.fromhex(row[3]), dtype="= 0, "SURVEY_WAGE_AMOUNT") + amounts[row[0]] = (row[1], amount) + # Reuse the maintained money owner's complete buffer/evidence/domain check. + # These value objects do not issue money/source authority. The live verifier + # compares the complete projection, including this source-owned domain. + current_money._validate_field( + current_money.MoneyField( + "WSAL_VAL", + b"".join(bytes.fromhex(row[3]) for row in rows), + bytes(row[4] for row in rows), + bytes(row[5] for row in rows), + bytes(row[6] for row in rows), + ), + current_money.MoneyDomain(**asec["domain"]), + len(rows), + nominal=True, + zero_origin_code=current_money.ZeroOrigin(asec["zero_origin_code"]), + ) + source_id = provenance.support_source_id_column("person") + original_id = provenance.spine_source_id_column("person") + require( + [r[0] for r in rows] == native_asec[source_id].tolist() + and [r[1] for r in rows] == native_asec[original_id].tolist() + and native_asec.source_year.astype(str).eq("2024").all(), + "SURVEY_WAGE_ORIGIN", + ) + wages = [] + columns = [ + provenance.support_channel_column("person"), + source_id, + original_id, + "employment_income_before_lsr", + ] + for channel, stacked, native, value in people[columns].itertuples( + index=False, name=None + ): + if channel == "asec": + require(stacked in amounts, "SURVEY_WAGE_COVERAGE") + original, amount = amounts[stacked] + require(original == native, "SURVEY_WAGE_ORIGIN") + wages.append(amount) + else: + require(channel == "acs", "SURVEY_WAGE_CHANNEL") + wages.append(np.nan if pd.isna(value) else value) + projected = pd.Series( + wages, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + matrix, mask = detail.recipient_matrix( + frame, role_column="tax_unit_role_input", person_wages=projected + ) + require(len(matrix) <= CURRENT_SURVEY_MAX_BYTES, "SURVEY_MATRIX_BOUND") + return ( + matrix, + mask, + { + "scope": SCOPE, + "host_route": CURRENT_SURVEY_ROUTE, + "projection_sha256": codec.sha(projection), + "acs_wage_evidence": acs_evidence, + "asec_money_header_sha256": asec["money_header_sha256"], + "selected_tax_units": int(mask.sum()), + "release_eligible": False, + "source_period_equivalence_claim": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py new file mode 100644 index 000000000..794486e8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py @@ -0,0 +1,1713 @@ +"""Pure per-field growth contract and transform for a raw-source PUF successor. + +This is the first bounded slice of `microcosm#530 +`_ item 2: *pin the raw +PUF asset and implement PUF aging as a Microcosm stage, under the same uprating +discipline as the other sources*. Nothing here reads a file, acquires an +artifact, imports a country engine, or generates a factor. It compiles an +explicit contract and applies it to a table that the caller already holds. + +What the contract makes explicit, per consumed field +---------------------------------------------------- +Source reference year, target year, role (money / code / identifier / count / +design weight), growth kind (nominal price restatement versus real per-return +aging), sign branch, and the exact factor-series identity. A field with no +declared rule is refused; a field with two rules that could both select the +same row is refused; there is no implicit factor of one and no dispatch by +string substring. Every factor is resolved by naming a **variable row** and two +**year columns** of the factor table explicitly, and is applied to a given row +at most once. + +Why the row/column axes are named rather than iterated +------------------------------------------------------ +The archived producer looped ``for variable in uprating`` over a DataFrame +pivoted to year columns, so it iterated years while believing it iterated +variable names (:data:`ARCHIVED_PRODUCER_COMMIT`, ``datasets/puf/puf.py`` +against ``utils/uprating.py``). Its 2015→2021 step separately listed two of its +three sign-branched fields in a fallback group as well, so for those two a +sign-specific factor was followed by a second growth factor. Both failure modes are structural +here: a series name is looked up in a variable-keyed mapping, a year is looked +up in a year-keyed mapping, and the sign-branch cover is validated to be exactly +disjoint before any array is touched. + +Authority +--------- +A factor table declares its own authority, and only +:data:`RELEASE_ELIGIBLE_AUTHORITIES` — ``reviewed_resource`` alone — can stand +behind a release. ``invented_fixture_nonauthority`` proves the contract and the +transform on invented levels. ``developmental_public_resource`` is for a +bounded experiment on levels that really were read from a public publisher: it +must carry the same receipt a reviewed table owes, plus the digest and +retrieval time of the exact resource bytes, and it must declare its own divisor +and index/total shape. Labelling such a table an invented fixture would be a +misdescription, so it gets its own authority rather than a lie. + +What it does **not** get is a promotion path. Release eligibility is keyed on +``reviewed_resource`` and nothing else, and a developmental table whose digest +appears in :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is refused outright, so an +allowlist entry cannot launder a developmental resource into a release recipe. +``reviewed_resource`` is itself refused until a reviewed table is generated, +pinned, and added to that deliberately empty tuple. No authority can be +promoted by passing a flag. + +Scope +----- +Money growth and design-weight growth are separate entry points, so a money +transform cannot move a weight. Codes, identifiers and counts are declared but +never written, and preservation is verified rather than assumed. The raw +``RECID`` and the source-year AGI used for SOI band assignment are copied into +separate provenance columns before any growth, so a later change of AGI basis +cannot destroy the band input. Nothing here decides whether the pinned +processed artifact is comparable to a rebuilt one; that artifact stays an +opaque legacy input, outside this contract. +""" + +from __future__ import annotations + +import datetime +import json +import struct +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, dataclass +from enum import StrEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.monetary_targets import MonetaryBasis +from microcosm.build.us_runtime.puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "DESIGN_WEIGHT_FIELD", + "GROWN_MONEY_VALUATION", + "MAX_EXACT_AMOUNT", + "PROVENANCE_COLUMNS", + "PROVENANCE_RECID_COLUMN", + "PROVENANCE_SOURCE_AGI_COLUMN", + "ARCHIVED_PRODUCER_COMMIT", + "DEVELOPMENTAL_PROVENANCE_KEYS", + "DEVELOPMENTAL_PROVENANCE_LITERALS", + "PROPOSED_TY2015_FIELD_ROSTER", + "RELEASE_ELIGIBLE_AUTHORITIES", + "ROSTER_CITATION", + "ROSTER_SOURCE_REFERENCE_YEAR", + "REVIEWED_FACTOR_TABLE_DIGESTS", + "REVIEWED_PROVENANCE_KEYS", + "SOURCE_AGI_FIELD", + "SOURCE_RECID_FIELD", + "CompiledPufGrowth", + "FactorAuthority", + "FactorSeries", + "FieldRole", + "GrowthFactorTable", + "GrowthKind", + "GrowthRecipe", + "GrowthRule", + "GrownPufTable", + "PufGrowthRefusalError", + "ResolvedFactor", + "RosterEntry", + "UNDOCUMENTED_SOURCE_FIELDS", + "SignBranch", + "apply_puf_design_weight_growth", + "apply_puf_growth", + "compile_puf_growth", + "parse_growth_recipe", + "roster_role", + "puf_growth_monetary_metadata", + "require_transformable", +] + +#: Schema version of the recipe identity this module emits. +RECIPE_SCHEMA_VERSION = 1 + +#: Schema version of the factor-table document this module parses. +FACTOR_TABLE_SCHEMA_VERSION = 1 + +#: Raw IRS PUF field carrying the return identifier. +SOURCE_RECID_FIELD = "RECID" + +#: Raw IRS PUF field carrying adjusted gross income. +SOURCE_AGI_FIELD = "E00100" + +#: Raw IRS PUF field carrying the sample weight (stored times 100). +DESIGN_WEIGHT_FIELD = "S006" + +#: Copy of ``RECID``, written from the source before any growth. No rule may +#: target this column or take its name, but the column itself is an ordinary +#: pandas column: what protects it is the contract, and in the graph the fact +#: that no node declares it as a rewrite. +PROVENANCE_RECID_COLUMN = "puf_source_recid" + +#: Copy of source-year AGI on the same terms, kept for the TY2015 SOI band +#: assignment in :mod:`microcosm.build.us_runtime.puf_source_agi`. +PROVENANCE_SOURCE_AGI_COLUMN = "puf_source_year_agi" + +#: The provenance columns this transform adds, in output order. +PROVENANCE_COLUMNS = (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + +#: SHA-256 digests of factor tables that a reviewed authority has accepted. +#: Empty: no factor table has been generated or reviewed for this lineage, so +#: ``reviewed_resource`` authority cannot compile. Adding a digest here is the +#: reviewed act; it is not something a caller can supply. +REVIEWED_FACTOR_TABLE_DIGESTS: tuple[str, ...] = () + +#: What a ``reviewed_resource`` factor table must state about itself. Nothing +#: can satisfy this yet, because :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is +#: empty; the set is here so the receipt a reviewed table owes is written down +#: executably rather than only in prose. +REVIEWED_PROVENANCE_KEYS = frozenset( + { + "dependency_versions", + "generated_by", + "generator_code_sha256", + "index_or_total", + "parameter_paths", + "population_divisor", + "rounding", + "source_url", + } +) + +#: What a ``developmental_public_resource`` factor table must state about +#: itself: everything a reviewed table owes, plus the digest of the exact +#: resource bytes its levels were read from and when they were retrieved. A +#: public resource is a moving distribution, so "which bytes, read when" is +#: part of the claim rather than a footnote. +DEVELOPMENTAL_PROVENANCE_KEYS = frozenset( + REVIEWED_PROVENANCE_KEYS | {"resource_sha256", "retrieved_utc"} +) + +#: Provenance values a developmental table must spell out rather than leave to +#: a reader's inference. A price index has no return or population divisor and +#: is an index, not a total; a table that says otherwise is describing a +#: different quantity and is refused here rather than downstream. +DEVELOPMENTAL_PROVENANCE_LITERALS = MappingProxyType( + {"index_or_total": "index", "population_divisor": "none"} +) + +#: Largest integer amount float64 represents exactly. A larger integer money +#: value is refused rather than silently rounded by the working conversion. +MAX_EXACT_AMOUNT = 2**53 + +#: Largest table this transform will accept, so a mistake is a refusal rather +#: than an unbounded allocation. +MAX_ROWS = 2_000_000 + +#: Largest factor document this transform will parse. +MAX_DOCUMENT_BYTES = 512 * 1024 + +_INTEGER_KINDS = frozenset("iu") +_MONEY_KINDS_ALLOWED = frozenset("iuf") +_RESULT_TOKEN = object() + + +class PufGrowthRefusalError(ValueError): + """Sanitized field/reason refusal; never carries row values or identifiers.""" + + def __init__(self, reason: str, field: str = "contract") -> None: + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(document: bytes) -> dict: + """Parse a bounded JSON object, refusing duplicate keys and non-finite values.""" + _require( + type(document) is bytes and 0 < len(document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + + def pairs(items): + result: dict = {} + for key, item in items: + _require(key not in result, "DUPLICATE_DOCUMENT_KEY") + result[key] = item + return result + + def constant(_): + raise PufGrowthRefusalError("NONFINITE_DOCUMENT") + + try: + parsed = json.loads(document, object_pairs_hook=pairs, parse_constant=constant) + except PufGrowthRefusalError: + raise + except (ValueError, RecursionError) as error: + raise PufGrowthRefusalError("MALFORMED_DOCUMENT") from error + _require(type(parsed) is dict, "DOCUMENT_OBJECT_REQUIRED") + return parsed + + +def _digest(value: object) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _identifier(value: object, reason: str, field: str = "contract") -> str: + _require( + type(value) is str and 0 < len(value) <= 128 and value.strip() == value, + reason, + field, + ) + return value # type: ignore[return-value] + + +def _year(value: object, reason: str, field: str = "contract") -> int: + _require(type(value) is int and 1900 <= value <= 2200, reason, field) + return value # type: ignore[return-value] + + +class FieldRole(StrEnum): + """What a consumed raw field is, which fixes what may happen to it.""" + + MONEY = "money" + CODE = "code" + IDENTIFIER = "identifier" + COUNT = "count" + DESIGN_WEIGHT = "design_weight" + + +class GrowthKind(StrEnum): + """How a declared factor is meant, which is not recoverable from its value.""" + + #: Restates the same nominal amount into another year's purchasing power. + NOMINAL_PRICE_RESTATEMENT = "nominal_price_restatement" + #: Ages an amount per return: an aggregate total ratio divided by a return + #: count ratio, so it carries real change as well as price change. + REAL_PER_RETURN_AGING = "real_per_return_aging" + #: Grows the design weight itself. Never applied to money. + DESIGN_WEIGHT_GROWTH = "design_weight_growth" + #: Declared and deliberately untouched. + NONE = "none" + + +class SignBranch(StrEnum): + """Which rows of a money field a rule may select.""" + + ANY = "any" + POSITIVE = "positive" + NEGATIVE = "negative" + + +class FactorAuthority(StrEnum): + """What a factor table claims about itself.""" + + INVENTED_FIXTURE = "invented_fixture_nonauthority" + #: Levels really read from a named public publisher, for a bounded + #: developmental experiment. Never release-eligible; see + #: :data:`RELEASE_ELIGIBLE_AUTHORITIES`. + DEVELOPMENTAL_PUBLIC_RESOURCE = "developmental_public_resource" + REVIEWED_RESOURCE = "reviewed_resource" + + +#: The only authority a release recipe may stand behind. Written as a set so +#: the invariant is one readable membership test rather than a comparison +#: repeated at each site, and so a test can assert its contents directly. +RELEASE_ELIGIBLE_AUTHORITIES = frozenset({FactorAuthority.REVIEWED_RESOURCE}) + + +_MONEY_KINDS = frozenset( + {GrowthKind.NOMINAL_PRICE_RESTATEMENT, GrowthKind.REAL_PER_RETURN_AGING} +) +_PRESERVED_ROLES = frozenset({FieldRole.CODE, FieldRole.IDENTIFIER, FieldRole.COUNT}) + +#: How a grown money column's valuation is named, per growth kind. The string +#: goes into :class:`~microcosm.build.monetary_targets.MonetaryBasis`, so a +#: later fit-boundary check compares a restated amount against an aged amount +#: as the different things they are. +GROWN_MONEY_VALUATION = MappingProxyType( + { + GrowthKind.NOMINAL_PRICE_RESTATEMENT: "nominal_price_restated_from_{source}", + GrowthKind.REAL_PER_RETURN_AGING: "real_per_return_aged_from_{source}", + } +) + + +@dataclass(frozen=True) +class GrowthRule: + """One rule: which rows of which field grow by which factor series. + + Attributes: + field: The raw source column this rule governs. + role: What the field is. Only :attr:`FieldRole.MONEY` may carry a sign + branch, and only money and the design weight may grow at all. + source_reference_year: The year the observed amount belongs to. + target_year: The year the output amount is stated in. Preserved fields + retain ``source_reference_year``; growing roles may also use that year. + growth_kind: What the factor means. Fixed by the role for every role + except money, which must choose restatement or aging explicitly. + sign_branch: Which rows this rule selects. ``ANY`` must be the field's + only rule; ``POSITIVE`` and ``NEGATIVE`` must both be present. + factor_series: The exact variable-row name in the factor table. Empty + only for a declared no-growth field. + citation: Where the field's meaning was read. Descriptive. + """ + + field: str + role: FieldRole + source_reference_year: int + target_year: int + growth_kind: GrowthKind + sign_branch: SignBranch = SignBranch.ANY + factor_series: str = "" + citation: str = "" + + def __post_init__(self) -> None: + field = _identifier(self.field, "RULE_FIELD_NAME") + _require(type(self.role) is FieldRole, "RULE_ROLE", field) + _require(type(self.growth_kind) is GrowthKind, "RULE_GROWTH_KIND", field) + _require(type(self.sign_branch) is SignBranch, "RULE_SIGN_BRANCH", field) + _require(type(self.citation) is str, "RULE_CITATION", field) + source = _year(self.source_reference_year, "RULE_SOURCE_YEAR", field) + target = _year(self.target_year, "RULE_TARGET_YEAR", field) + if self.role is FieldRole.MONEY: + _require(self.growth_kind in _MONEY_KINDS, "MONEY_GROWTH_KIND", field) + elif self.role is FieldRole.DESIGN_WEIGHT: + _require( + self.growth_kind is GrowthKind.DESIGN_WEIGHT_GROWTH, + "WEIGHT_GROWTH_KIND", + field, + ) + _require(self.sign_branch is SignBranch.ANY, "WEIGHT_SIGN_BRANCH", field) + else: + _require( + self.growth_kind is GrowthKind.NONE, "PRESERVED_GROWTH_KIND", field + ) + _require(self.sign_branch is SignBranch.ANY, "PRESERVED_SIGN_BRANCH", field) + _require(source == target, "PRESERVED_YEAR_SPAN", field) + if self.growth_kind is GrowthKind.NONE: + _require(self.factor_series == "", "PRESERVED_FACTOR_SERIES", field) + else: + _identifier(self.factor_series, "RULE_FACTOR_SERIES", field) + _require(target >= source, "RULE_YEAR_ORDER", field) + + @property + def grows(self) -> bool: + """Whether this rule multiplies anything at all.""" + return self.growth_kind is not GrowthKind.NONE + + def as_document(self) -> dict: + """Canonical, hashable description of this rule.""" + return { + "field": self.field, + "role": str(self.role), + "source_reference_year": self.source_reference_year, + "target_year": self.target_year, + "growth_kind": str(self.growth_kind), + "sign_branch": str(self.sign_branch), + "factor_series": self.factor_series, + } + + +@dataclass(frozen=True) +class GrowthRecipe: + """A closed set of rules covering every consumed field exactly once. + + Coverage is validated here, before any table is seen: a money field is + covered either by one ``ANY`` rule or by exactly one ``POSITIVE`` and one + ``NEGATIVE`` rule, and never by both shapes. That is the structural refusal + of the archived overlap, where a sign-branched field also sat in a fallback + group and could receive a second factor. + """ + + recipe_id: str + rules: tuple[GrowthRule, ...] + + def __post_init__(self) -> None: + _identifier(self.recipe_id, "RECIPE_ID") + _require( + type(self.rules) is tuple + and bool(self.rules) + and all(type(rule) is GrowthRule for rule in self.rules), + "RECIPE_RULES", + ) + seen: set[tuple[str, SignBranch]] = set() + branches: dict[str, set[SignBranch]] = {} + shape: dict[str, tuple[FieldRole, int, int, GrowthKind]] = {} + for rule in self.rules: + key = (rule.field, rule.sign_branch) + _require(key not in seen, "DUPLICATE_FIELD_RULE", rule.field) + seen.add(key) + branches.setdefault(rule.field, set()).add(rule.sign_branch) + declaration = ( + rule.role, + rule.source_reference_year, + rule.target_year, + rule.growth_kind, + ) + if rule.field in shape: + _require( + shape[rule.field] == declaration, + "AMBIGUOUS_FIELD_DECLARATION", + rule.field, + ) + else: + shape[rule.field] = declaration + for name, present in branches.items(): + if present == {SignBranch.ANY}: + continue + _require(SignBranch.ANY not in present, "OVERLAPPING_SIGN_RULES", name) + _require( + present == {SignBranch.POSITIVE, SignBranch.NEGATIVE}, + "INCOMPLETE_SIGN_COVER", + name, + ) + for rule in self.rules: + _require( + rule.field not in PROVENANCE_COLUMNS, + "PROVENANCE_COLUMN_NAME", + rule.field, + ) + money = [rule for rule in self.rules if rule.role is FieldRole.MONEY] + _require(bool(money), "NO_MONEY_FIELD") + identifiers = [ + rule.field for rule in self.rules if rule.role is FieldRole.IDENTIFIER + ] + _require(len(set(identifiers)) == 1, "PROVENANCE_RECID_FIELD") + _require( + any(rule.field == SOURCE_AGI_FIELD for rule in money), + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + money_targets = {rule.target_year for rule in money} + _require(len(money_targets) == 1, "MIXED_TARGET_YEAR") + _require( + len({rule.source_reference_year for rule in money}) == 1, + "MIXED_SOURCE_REFERENCE_YEAR", + ) + weights = [rule for rule in self.rules if rule.role is FieldRole.DESIGN_WEIGHT] + _require(len(weights) <= 1, "MULTIPLE_DESIGN_WEIGHTS") + if weights and money_targets: + _require( + weights[0].target_year == next(iter(money_targets)), + "WEIGHT_TARGET_YEAR", + weights[0].field, + ) + object.__setattr__(self, "rules", tuple(sorted(self.rules, key=_rule_order))) + + @property + def fields(self) -> tuple[str, ...]: + """Every declared field name, once each, in the rules' sorted order. + + ``__post_init__`` sorts the rules by ``(field, sign_branch)``, so two + recipes that declare the same rules in a different order are the same + recipe and hash the same. + """ + ordered: list[str] = [] + for rule in self.rules: + if rule.field not in ordered: + ordered.append(rule.field) + return tuple(ordered) + + def rules_for(self, field: str) -> tuple[GrowthRule, ...]: + """Every rule governing ``field``.""" + return tuple(rule for rule in self.rules if rule.field == field) + + @property + def identity(self) -> bytes: + """Canonical bytes identifying this recipe.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe_id": self.recipe_id, + "rules": [rule.as_document() for rule in self.rules], + } + ) + + +def parse_growth_recipe(document: bytes) -> GrowthRecipe: + """Rebuild a recipe from its own :attr:`GrowthRecipe.identity` bytes. + + Round-tripping through the identity document is how a recipe travels as a + graph node parameter without becoming an opaque blob: the bytes a node + carries are exactly the bytes the recipe hashes to. + + Args: + document: A recipe identity document. + + Returns: + The recipe those bytes describe. + + Raises: + PufGrowthRefusalError: If the document is malformed, is a different + schema version, or does not round-trip to itself. + """ + data = _parse(document) + _require( + set(data) == {"schema_version", "recipe_id", "rules"}, "RECIPE_DOCUMENT_SCHEMA" + ) + _require(data["schema_version"] == RECIPE_SCHEMA_VERSION, "RECIPE_SCHEMA_VERSION") + _require( + type(data["rules"]) is list and bool(data["rules"]), "RECIPE_DOCUMENT_RULES" + ) + rules = [] + for entry in data["rules"]: + _require( + type(entry) is dict + and set(entry) + == { + "field", + "role", + "source_reference_year", + "target_year", + "growth_kind", + "sign_branch", + "factor_series", + }, + "RECIPE_DOCUMENT_RULES", + ) + for key, values in ( + ("role", FieldRole), + ("growth_kind", GrowthKind), + ("sign_branch", SignBranch), + ): + _require( + entry[key] in tuple(str(value) for value in values), + "RECIPE_DOCUMENT_RULES", + entry["field"] if type(entry["field"]) is str else "contract", + ) + rules.append( + GrowthRule( + entry["field"], + FieldRole(entry["role"]), + entry["source_reference_year"], + entry["target_year"], + GrowthKind(entry["growth_kind"]), + SignBranch(entry["sign_branch"]), + entry["factor_series"], + ) + ) + recipe = GrowthRecipe(data["recipe_id"], tuple(rules)) + _require(recipe.identity == document, "RECIPE_DOCUMENT_ROUNDTRIP") + return recipe + + +def _rule_order(rule: GrowthRule) -> tuple[str, str]: + return (rule.field, str(rule.sign_branch)) + + +@dataclass(frozen=True) +class FactorSeries: + """One variable row of the factor table: a level per year column. + + Levels are index levels, not ratios. A factor is always a ratio of two + explicitly named year columns of one explicitly named row, so a table + cannot be read as if its year columns were variable names. + """ + + name: str + kind: GrowthKind + levels: tuple[tuple[int, str], ...] + + def __post_init__(self) -> None: + name = _identifier(self.name, "SERIES_NAME") + _require(type(self.kind) is GrowthKind, "SERIES_KIND", name) + _require(self.kind is not GrowthKind.NONE, "SERIES_KIND", name) + _require( + type(self.levels) is tuple and bool(self.levels), "SERIES_LEVELS", name + ) + _require( + all(type(pair) in (tuple, list) and len(pair) == 2 for pair in self.levels), + "SERIES_LEVELS", + name, + ) + # Own inner pairs too: frozen dataclasses do not freeze a caller's lists. + object.__setattr__(self, "levels", tuple(tuple(pair) for pair in self.levels)) + years = [year for year, _ in self.levels] + _require(len(set(years)) == len(years), "DUPLICATE_SERIES_YEAR", name) + _require(years == sorted(years), "UNORDERED_SERIES_YEARS", name) + for year, level in self.levels: + _year(year, "SERIES_YEAR", name) + _require(type(level) is str and level.strip() == level, "LEVEL_TEXT", name) + _require(_finite_positive(level), "LEVEL_VALUE", name) + + def level_text(self, year: int) -> str: + """The exact level text in the ``year`` column of this variable row.""" + for candidate, level in self.levels: + if candidate == year: + return level + raise PufGrowthRefusalError("MISSING_FACTOR_YEAR", self.name) + + @property + def years(self) -> tuple[int, ...]: + return tuple(year for year, _ in self.levels) + + +def _finite_positive(text: str) -> bool: + try: + value = float(text) + except ValueError: + return False + return bool(np.isfinite(value)) and value > 0.0 + + +def _freeze_provenance(value: object, depth: int = 0) -> object: + """Own a bounded JSON tree, including nested caller-supplied containers.""" + _require(depth <= 64, "TABLE_PROVENANCE_DEPTH") + if isinstance(value, Mapping): + _require(all(type(key) is str for key in value), "TABLE_PROVENANCE_KEY") + return MappingProxyType( + {key: _freeze_provenance(item, depth + 1) for key, item in value.items()} + ) + if type(value) in (tuple, list): + return tuple(_freeze_provenance(item, depth + 1) for item in value) + _require( + value is None or type(value) in (str, int, float, bool), + "TABLE_PROVENANCE_VALUE", + ) + if type(value) is float: + _require(bool(np.isfinite(value)), "TABLE_PROVENANCE_VALUE") + return value + + +def _provenance_document(value: object) -> object: + """Return detached JSON containers, never references into the owned tree.""" + if isinstance(value, Mapping): + return {key: _provenance_document(item) for key, item in value.items()} + if type(value) is tuple: + return [_provenance_document(item) for item in value] + return value + + +@dataclass(frozen=True) +class GrowthFactorTable: + """A parsed factor document: variable rows by year columns, plus authority. + + The document is retained verbatim so the table's identity is the bytes a + reviewer would read, not a re-serialization of this object's state. + """ + + document: bytes + table_id: str + authority: FactorAuthority + years: tuple[int, ...] + series: tuple[FactorSeries, ...] + provenance: Mapping[str, object] + + def __post_init__(self) -> None: + _require( + type(self.document) is bytes + and 0 < len(self.document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + _identifier(self.table_id, "TABLE_ID") + _require(type(self.authority) is FactorAuthority, "TABLE_AUTHORITY") + _require( + type(self.series) is tuple + and bool(self.series) + and all(type(item) is FactorSeries for item in self.series), + "TABLE_SERIES", + ) + names = [item.name for item in self.series] + _require(len(set(names)) == len(names), "DUPLICATE_SERIES") + _require(type(self.years) is tuple and bool(self.years), "TABLE_YEARS") + for year in self.years: + _year(year, "TABLE_YEARS") + _require(list(self.years) == sorted(set(self.years)), "TABLE_YEARS") + for item in self.series: + _require(item.years == self.years, "SERIES_YEAR_COVERAGE", item.name) + _require(isinstance(self.provenance, Mapping), "TABLE_PROVENANCE") + object.__setattr__(self, "provenance", _freeze_provenance(self.provenance)) + # Re-serializing proves the retained bytes are the canonical document + # for exactly this parsed state, so no field can drift from the bytes. + _require(self.document == _json(self.as_document()), "TABLE_DOCUMENT") + + @classmethod + def from_bytes(cls, document: bytes) -> GrowthFactorTable: + """Parse a factor document, refusing anything it does not fully close.""" + data = _parse(document) + _require( + set(data) + == { + "authority", + "provenance", + "schema_version", + "series", + "table_id", + "years", + }, + "TABLE_SCHEMA", + ) + _require( + data["schema_version"] == FACTOR_TABLE_SCHEMA_VERSION, + "TABLE_SCHEMA_VERSION", + ) + _require( + data["authority"] in tuple(str(value) for value in FactorAuthority), + "TABLE_AUTHORITY", + ) + _require(type(data["years"]) is list and bool(data["years"]), "TABLE_YEARS") + years = tuple(_year(value, "TABLE_YEARS") for value in data["years"]) + _require(type(data["series"]) is dict and bool(data["series"]), "TABLE_SERIES") + _require(type(data["provenance"]) is dict, "TABLE_PROVENANCE") + series = [] + for name in sorted(data["series"]): + entry = data["series"][name] + _require( + type(entry) is dict and set(entry) == {"kind", "levels"}, + "SERIES_SCHEMA", + name, + ) + _require( + entry["kind"] in tuple(str(value) for value in GrowthKind), + "SERIES_KIND", + name, + ) + levels = entry["levels"] + _require(type(levels) is dict and bool(levels), "SERIES_LEVELS", name) + _require( + set(levels) == {str(year) for year in years}, + "SERIES_YEAR_COVERAGE", + name, + ) + series.append( + FactorSeries( + name, + GrowthKind(entry["kind"]), + tuple((year, levels[str(year)]) for year in years), + ) + ) + return cls( + document, + data["table_id"], + FactorAuthority(data["authority"]), + years, + tuple(series), + data["provenance"], + ) + + def as_document(self) -> dict: + """The canonical document for this table's parsed state.""" + return { + "authority": str(self.authority), + "provenance": _provenance_document(self.provenance), + "schema_version": FACTOR_TABLE_SCHEMA_VERSION, + "series": { + item.name: { + "kind": str(item.kind), + "levels": {str(year): level for year, level in item.levels}, + } + for item in self.series + }, + "table_id": self.table_id, + "years": list(self.years), + } + + def series_for(self, name: str) -> FactorSeries: + """The variable row called ``name``; refuses a year column or a miss.""" + for item in self.series: + if item.name == name: + return item + raise PufGrowthRefusalError("UNKNOWN_FACTOR_SERIES", name) + + @property + def sha256(self) -> str: + return _sha(self.document) + + +def _check_developmental_provenance(factors: GrowthFactorTable) -> None: + """Close what a ``developmental_public_resource`` table must state. + + Everything the reviewed shape owes, plus the digest and retrieval time of + the exact resource bytes, plus the two literals that keep a price index + from being read as a per-return total. The reviewed allowlist is checked + negatively here: a developmental digest listed there is a refusal, so the + one act that promotes a reviewed table cannot promote this one. + """ + _require( + set(factors.provenance) == DEVELOPMENTAL_PROVENANCE_KEYS, + "DEVELOPMENTAL_PROVENANCE_SHAPE", + ) + _require( + factors.sha256 not in REVIEWED_FACTOR_TABLE_DIGESTS, + "DEVELOPMENTAL_TABLE_IN_REVIEWED_ALLOWLIST", + ) + for key, expected in DEVELOPMENTAL_PROVENANCE_LITERALS.items(): + _require(factors.provenance[key] == expected, "DEVELOPMENTAL_PROVENANCE_VALUE") + _require( + _digest(factors.provenance["resource_sha256"]), "DEVELOPMENTAL_RESOURCE_DIGEST" + ) + url = factors.provenance["source_url"] + _require( + type(url) is str and url.startswith("https://") and len(url) > len("https://"), + "DEVELOPMENTAL_SOURCE_URL", + ) + retrieved = factors.provenance["retrieved_utc"] + _require(type(retrieved) is str, "DEVELOPMENTAL_RETRIEVED_UTC") + try: + stamp = datetime.datetime.fromisoformat(retrieved) + except ValueError as error: + raise PufGrowthRefusalError("DEVELOPMENTAL_RETRIEVED_UTC") from error + _require( + stamp.tzinfo is not None and stamp.utcoffset() == datetime.timedelta(0), + "DEVELOPMENTAL_RETRIEVED_UTC", + ) + + +@dataclass(frozen=True) +class ResolvedFactor: + """One rule's factor, resolved once from two named year columns.""" + + field: str + sign_branch: SignBranch + series: str + kind: GrowthKind + source_year: int + target_year: int + source_level: str + target_level: str + ratio_bits: str + + def __post_init__(self) -> None: + field = _identifier(self.field, "RESOLVED_FIELD") + _require(type(self.sign_branch) is SignBranch, "RESOLVED_SIGN_BRANCH", field) + _require( + type(self.kind) is GrowthKind and self.kind is not GrowthKind.NONE, + "RESOLVED_KIND", + field, + ) + _identifier(self.series, "RESOLVED_SERIES", field) + _year(self.source_year, "RESOLVED_SOURCE_YEAR", field) + _year(self.target_year, "RESOLVED_TARGET_YEAR", field) + # The stored bits are the factor. Recomputing them here means a + # hand-built ResolvedFactor cannot claim a factor its own levels do not + # produce, so the bits are evidence rather than an assertion. + expected = _factor_ratio_bits(self.source_level, self.target_level, field) + _require(self.ratio_bits == expected, "RESOLVED_RATIO_BITS", field) + + @property + def value(self) -> float: + """The exact float64 this factor is, recovered from its bit pattern.""" + return struct.unpack(" dict: + return { + "field": self.field, + "kind": str(self.kind), + "ratio_bits": self.ratio_bits, + "series": self.series, + "sign_branch": str(self.sign_branch), + "source_level": self.source_level, + "source_year": self.source_year, + "target_level": self.target_level, + "target_year": self.target_year, + } + + +@dataclass(frozen=True) +class CompiledPufGrowth: + """A recipe bound to a factor table, with every factor already resolved.""" + + recipe: GrowthRecipe + factors: GrowthFactorTable + resolved: tuple[ResolvedFactor, ...] + identity: bytes + + def __post_init__(self) -> None: + _require(type(self.recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(self.factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + # The authority gate lives here, not only in the compile function, so + # that no construction path can produce a release-eligible contract the + # compile chokepoint would have refused. + if self.factors.authority is FactorAuthority.REVIEWED_RESOURCE: + _require( + self.factors.sha256 in REVIEWED_FACTOR_TABLE_DIGESTS, + "UNREVIEWED_FACTOR_TABLE", + ) + _require( + set(self.factors.provenance) == REVIEWED_PROVENANCE_KEYS, + "REVIEWED_PROVENANCE_SHAPE", + ) + elif self.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE: + _check_developmental_provenance(self.factors) + _require( + type(self.resolved) is tuple + and all(type(item) is ResolvedFactor for item in self.resolved), + "RESOLVED_TYPE", + ) + # Exactly one resolved factor per growing rule, agreeing with it, and + # none left over: a directly constructed contract cannot describe a + # different computation than its own recipe declares. + expected = tuple( + ( + rule.field, + rule.sign_branch, + rule.factor_series, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + ) + for rule in self.recipe.rules + if rule.grows + ) + actual = tuple( + ( + factor.field, + factor.sign_branch, + factor.series, + factor.kind, + factor.source_year, + factor.target_year, + ) + for factor in self.resolved + ) + _require(actual == expected, "RESOLVED_COVERAGE") + _require( + self.resolved == _resolve_factors(self.recipe, self.factors), + "RESOLVED_FACTOR_BINDING", + ) + _require( + type(self.identity) is bytes + and self.identity + == _contract_identity(self.recipe, self.factors, self.resolved), + "CONTRACT_BINDING", + ) + + @property + def sha256(self) -> str: + return _sha(self.identity) + + @property + def release_eligible(self) -> bool: + """Whether this contract may stand behind a release recipe. + + ``False`` for every invented fixture and every developmental public + resource, and there is no override: the answer is membership of + :data:`RELEASE_ELIGIBLE_AUTHORITIES`, which no caller, flag or + allowlist entry can widen. + """ + return self.factors.authority in RELEASE_ELIGIBLE_AUTHORITIES + + @property + def money_fields(self) -> tuple[str, ...]: + """Declared money fields, once each, in rule order.""" + return tuple( + dict.fromkeys( + rule.field for rule in self.recipe.rules if rule.role is FieldRole.MONEY + ) + ) + + @property + def money_view_fields(self) -> tuple[str, ...]: + """The columns :func:`apply_puf_growth` consumes: everything but weights. + + The design weight is excluded so a money transform neither reads nor + depends on it. In the graph that keeps the money node and the + design-weight node siblings rather than one ordering the other. + """ + weight = self.design_weight_rule + return tuple( + name + for name in self.recipe.fields + if weight is None or name != weight.field + ) + + @property + def preserved_fields(self) -> tuple[str, ...]: + """Declared code/identifier/count fields, which this contract never writes.""" + return tuple( + dict.fromkeys( + rule.field + for rule in self.recipe.rules + if rule.role in _PRESERVED_ROLES + ) + ) + + @property + def design_weight_rule(self) -> GrowthRule | None: + for rule in self.recipe.rules: + if rule.role is FieldRole.DESIGN_WEIGHT: + return rule + return None + + @property + def target_year(self) -> int: + for rule in self.recipe.rules: + if rule.role is FieldRole.MONEY: + return rule.target_year + raise PufGrowthRefusalError("NO_MONEY_FIELD") + + def factor_for(self, field: str, sign_branch: SignBranch) -> ResolvedFactor: + for factor in self.resolved: + if factor.field == field and factor.sign_branch is sign_branch: + return factor + raise PufGrowthRefusalError("NO_RESOLVED_FACTOR", field) + + +def compile_puf_growth( + recipe: GrowthRecipe, factors: GrowthFactorTable +) -> CompiledPufGrowth: + """Bind a recipe to a table using the same resolver as constructor validation. + + Args: + recipe: The closed rule set. Already coverage-validated. + factors: The factor table the rules name. + + Returns: + An immutable compiled contract carrying its own identity bytes. + + Raises: + PufGrowthRefusalError: If the table's authority is not compilable, if a + rule names a series the table does not have, if a named series + means a different growth kind than the rule declares, or if either + declared year is not a column of that series. + """ + _require(type(recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + resolved = _resolve_factors(recipe, factors) + return CompiledPufGrowth( + recipe, factors, resolved, _contract_identity(recipe, factors, resolved) + ) + + +def _factor_ratio_bits(source_level: str, target_level: str, field: str) -> str: + """One finite positive float64 ratio for compiler and public constructors.""" + _require( + all( + type(level) is str and level.strip() == level and _finite_positive(level) + for level in (source_level, target_level) + ), + "RESOLVED_LEVEL", + field, + ) + ratio = float(target_level) / float(source_level) + _require(bool(np.isfinite(ratio)) and ratio > 0.0, "FACTOR_VALUE", field) + return struct.pack(" tuple[ResolvedFactor, ...]: + """Resolve the complete recipe from the table's immutable named series.""" + resolved = [] + for rule in recipe.rules: + if not rule.grows: + continue + series = factors.series_for(rule.factor_series) + _require(series.kind is rule.growth_kind, "FACTOR_KIND_MISMATCH", rule.field) + source_level = series.level_text(rule.source_reference_year) + target_level = series.level_text(rule.target_year) + resolved.append( + ResolvedFactor( + rule.field, + rule.sign_branch, + series.name, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + source_level, + target_level, + _factor_ratio_bits(source_level, target_level, rule.field), + ) + ) + return tuple(resolved) + + +def _contract_identity( + recipe: GrowthRecipe, + factors: GrowthFactorTable, + resolved: tuple[ResolvedFactor, ...], +) -> bytes: + """Canonical bytes identifying one recipe bound to one factor table.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe": json.loads(recipe.identity), + "factor_table_sha256": factors.sha256, + "factor_table_id": factors.table_id, + "factor_authority": str(factors.authority), + "resolved": [factor.as_document() for factor in resolved], + } + ) + + +def puf_growth_monetary_metadata( + compiled: CompiledPufGrowth, +) -> Mapping[str, MonetaryBasis]: + """Declared output basis per money field, for a later fit-boundary check. + + The check itself is not in this slice. What is here is the metadata it + needs: currency, target period, and a valuation string that distinguishes a + nominal restatement from a real per-return aging, so a donor field and a + recipient predictor cannot be silently matched across different bases. + + Args: + compiled: A compiled contract. + + Returns: + Money field name to its declared output + :class:`~microcosm.build.monetary_targets.MonetaryBasis`. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + result: dict[str, MonetaryBasis] = {} + for rule in compiled.recipe.rules: + if rule.role is not FieldRole.MONEY or rule.field in result: + continue + result[rule.field] = MonetaryBasis( + currency="USD", + unit="base_currency", + period=str(rule.target_year), + temporal_basis="annual_flow", + sector="households", + perimeter="irs_individual_income_tax_returns", + valuation=GROWN_MONEY_VALUATION[rule.growth_kind].format( + source=rule.source_reference_year + ), + ) + return MappingProxyType(result) + + +@dataclass(frozen=True, eq=False) +class GrownPufTable: + """A grown table and its receipt; only :func:`apply_puf_growth` builds one.""" + + table: pd.DataFrame + contract_sha256: str + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token: object) -> None: + _require(_token is _RESULT_TOKEN, "RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self) -> dict: + """The parsed receipt describing exactly what was applied.""" + return _parse(self._receipt) + + @property + def receipt_bytes(self) -> bytes: + return self._receipt + + +def _checked_source_table( + table: pd.DataFrame, compiled: CompiledPufGrowth +) -> pd.DataFrame: + _require(type(table) is pd.DataFrame, "SOURCE_TABLE_TYPE") + rows = len(table) + _require(0 < rows <= MAX_ROWS, "SOURCE_TABLE_ROWS") + _require(all(type(name) is str for name in table.columns), "SOURCE_TABLE_COLUMNS") + _require(len(set(table.columns)) == len(table.columns), "DUPLICATE_SOURCE_COLUMN") + declared = set(compiled.money_view_fields) + present = set(table.columns) + weight = compiled.design_weight_rule + if weight is not None and weight.field in present: + raise PufGrowthRefusalError("DESIGN_WEIGHT_NOT_IN_MONEY_VIEW", weight.field) + missing = sorted(declared - present) + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + undeclared = sorted(present - declared) + if undeclared: + raise PufGrowthRefusalError("UNDECLARED_SOURCE_COLUMN", undeclared[0]) + return table + + +def _money_values(table: pd.DataFrame, field: str) -> np.ndarray: + """A money column as float64, refusing anything it would have to coerce.""" + series = table[field] + # An explicit kind allow-list rather than ``is_numeric_dtype``, which also + # admits complex, whose imaginary part a float64 cast would drop in silence. + _require( + series.dtype.kind in _MONEY_KINDS_ALLOWED + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint", "float")) + ), + "FIELD_DTYPE", + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + integral = series.dtype.kind in _INTEGER_KINDS or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ) + if integral: + # float64 is the working type, so an integer amount it cannot represent + # exactly is refused rather than silently rounded. + _require( + bool((np.abs(series.to_numpy(dtype=object)) <= MAX_EXACT_AMOUNT).all()), + "MONEY_MAGNITUDE", + field, + ) + values = series.to_numpy(dtype=np.float64, copy=True) + _require(bool(np.isfinite(values).all()), "FIELD_NONFINITE", field) + # A negative zero would pass every arithmetic guard untouched and then + # compare equal to zero while carrying a different bit pattern. + _require(not bool(np.signbit(values[values == 0.0]).any()), "NEGATIVE_ZERO", field) + return values + + +def _integral_values(table: pd.DataFrame, field: str, reason: str) -> np.ndarray: + series = table[field] + _require( + series.dtype.kind in _INTEGER_KINDS + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ), + reason, + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + return series.to_numpy(copy=True) + + +def _selection(values: np.ndarray, branch: SignBranch) -> np.ndarray: + """Rows this branch selects. Zero is never selected, so it never grows.""" + if branch is SignBranch.POSITIVE: + return values > 0.0 + if branch is SignBranch.NEGATIVE: + return values < 0.0 + return values != 0.0 + + +def require_transformable(compiled: CompiledPufGrowth) -> None: + """Check every precondition :func:`apply_puf_growth` needs, without a table. + + A caller that is about to declare work — a graph node factory, say — uses + this so a contract that could never run is refused before anything is + loaded, rather than at the moment the money transform reaches for a field + that is not there. + + Args: + compiled: The contract to check. + + Raises: + PufGrowthRefusalError: If the contract declares no money field, no + single identifier, or no source-year AGI field. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _ = compiled.target_year + _source_reference_year(compiled) + _recid_field(compiled) + _agi_field(compiled) + + +def apply_puf_growth(table: pd.DataFrame, compiled: CompiledPufGrowth) -> GrownPufTable: + """Grow every declared money field once, preserving everything else. + + The input is never mutated: every refusal happens before any array is + written, and the output is a new frame built from copies. + + Args: + table: One row per raw source record, whose columns are exactly + :attr:`CompiledPufGrowth.money_view_fields` — every declared field + except the design weight. + compiled: The compiled contract. + + Row correspondence is positional, not by pandas label: the output carries a + fresh ``RangeIndex`` and row *i* of the output is row *i* of the input. A + caller that needs the input's labels back must reattach them. + + The input is never mutated and no partial result is ever returned. Most + refusals happen before any arithmetic; the product guards + (``GROWN_NONFINITE``, ``SIGN_CHANGED``, ``FACTOR_APPLIED_TWICE``) + necessarily run after a candidate array has been computed in local memory. + + Returns: + A :class:`GrownPufTable` whose frame carries the grown money columns, + the byte-identical code/identifier/count columns, and the two + provenance columns copied from the source before any growth. The design + weight is not part of this view at all; it grows only through + :func:`apply_puf_design_weight_growth`. + + Raises: + PufGrowthRefusalError: On any contract, dtype, value, or coverage + violation. Nothing is written when this is raised. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + source = _checked_source_table(table, compiled) + rows = len(source) + + # Validate every column and resolve every factor before writing anything. + grown: dict[str, np.ndarray] = {} + applications: dict[str, int] = {} + for name in compiled.money_view_fields: + rules = compiled.recipe.rules_for(name) + role = rules[0].role + if role is FieldRole.MONEY: + values = _money_values(source, name) + applied = np.zeros(rows, dtype=np.int64) + result = values.copy() + for rule in rules: + factor = compiled.factor_for(name, rule.sign_branch) + selected = _selection(values, rule.sign_branch) + applied += selected.astype(np.int64) + # The product is checked below; a warning here would only + # duplicate a refusal that is already explicit. + with np.errstate(over="ignore", under="ignore"): + result[selected] = values[selected] * factor.value + _require(bool((applied <= 1).all()), "FACTOR_APPLIED_TWICE", name) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", name) + _require( + bool((np.sign(result) == np.sign(values)).all()), "SIGN_CHANGED", name + ) + grown[name] = result + applications[name] = int(applied.sum()) + else: + _integral_values(source, name, "PRESERVED_FIELD_DTYPE") + + recid_field = _recid_field(compiled) + agi_field = _agi_field(compiled) + provenance_recid = _integral_values(source, recid_field, "PROVENANCE_RECID_DTYPE") + _require(len(np.unique(provenance_recid)) == rows, "DUPLICATE_RECID", recid_field) + provenance_agi = _money_values(source, agi_field) + + output = pd.DataFrame(index=pd.RangeIndex(rows)) + for name in source.columns: + output[name] = ( + pd.Series(grown[name], index=output.index, dtype="float64") + if name in grown + else source[name].reset_index(drop=True).copy(deep=True) + ) + output[PROVENANCE_RECID_COLUMN] = pd.Series( + provenance_recid, index=output.index, dtype=source[recid_field].dtype + ) + output[PROVENANCE_SOURCE_AGI_COLUMN] = pd.Series( + provenance_agi, index=output.index, dtype="float64" + ) + + receipt = _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "artifact_kind": "microcosm.us_puf_growth", + "contract_sha256": _sha(compiled.identity), + "factor_authority": str(compiled.factors.authority), + "factor_table_sha256": compiled.factors.sha256, + "release_eligible": compiled.release_eligible, + "rows": rows, + "source_reference_year": _source_reference_year(compiled), + "target_year": compiled.target_year, + "grown_money_fields": sorted(grown), + "grown_row_counts": {name: applications[name] for name in sorted(grown)}, + "preserved_fields": sorted(compiled.preserved_fields), + "design_weight_field": ( + compiled.design_weight_rule.field + if compiled.design_weight_rule is not None + else None + ), + "design_weight_in_money_view": False, + "provenance_columns": list(PROVENANCE_COLUMNS), + "provenance_recid_source": recid_field, + "provenance_agi_source": agi_field, + } + ) + return GrownPufTable(output, _sha(compiled.identity), receipt, _token=_RESULT_TOKEN) + + +def apply_puf_design_weight_growth( + weights: Sequence[float] | np.ndarray, compiled: CompiledPufGrowth +) -> np.ndarray: + """Grow design weights by their own declared factor, and nothing else. + + Kept apart from :func:`apply_puf_growth` on purpose: money growth and + weight growth are different operations with different factor series, and a + single call that did both could silently move population with prices. + + Args: + weights: The source design weights, one per record. + compiled: The compiled contract, which must declare a design weight. + + Returns: + A new float64 array of grown weights. + + Raises: + PufGrowthRefusalError: If no design-weight rule is declared, or the + weights are not finite and positive. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + rule = compiled.design_weight_rule + _require(rule is not None, "NO_DESIGN_WEIGHT_RULE") + values = np.asarray(weights, dtype=np.float64) + _require(values.ndim == 1 and 0 < values.size <= MAX_ROWS, "WEIGHT_SHAPE") + _require(bool(np.isfinite(values).all()), "WEIGHT_NONFINITE") + _require(bool((values > 0.0).all()), "NONPOSITIVE_DESIGN_WEIGHT") + factor = compiled.factor_for(rule.field, SignBranch.ANY) + _require( + factor.kind is GrowthKind.DESIGN_WEIGHT_GROWTH, "WEIGHT_FACTOR_KIND", rule.field + ) + with np.errstate(over="ignore", under="ignore"): + grown = values * factor.value + _require(bool(np.isfinite(grown).all()), "GROWN_WEIGHT_NONFINITE", rule.field) + _require(bool((grown > 0.0).all()), "GROWN_WEIGHT_NONPOSITIVE", rule.field) + return grown + + +def _source_reference_year(compiled: CompiledPufGrowth) -> int: + years = { + rule.source_reference_year + for rule in compiled.recipe.rules + if rule.role is FieldRole.MONEY + } + _require(len(years) == 1, "MIXED_SOURCE_REFERENCE_YEAR") + return years.pop() + + +def _recid_field(compiled: CompiledPufGrowth) -> str: + """The one declared identifier. :class:`GrowthRecipe` already required it.""" + candidates = [ + rule.field + for rule in compiled.recipe.rules + if rule.role is FieldRole.IDENTIFIER + ] + _require(len(candidates) == 1, "PROVENANCE_RECID_FIELD") + return candidates[0] + + +def _agi_field(compiled: CompiledPufGrowth) -> str: + _require( + SOURCE_AGI_FIELD in compiled.recipe.fields, + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + return SOURCE_AGI_FIELD + + +@dataclass(frozen=True) +class RosterEntry: + """One raw TY2015 PUF field this lineage already names, and what it is. + + A roster entry states a **role** and a documented **meaning**, never a + factor. Which series a field should grow by is a producer decision that + belongs to a reviewed recipe, not to this file. + """ + + field: str + role: FieldRole + meaning: str + citation: str = "" + + def __post_init__(self) -> None: + _identifier(self.field, "ROSTER_FIELD") + _require(type(self.role) is FieldRole, "ROSTER_ROLE", self.field) + _require(bool(self.meaning), "ROSTER_MEANING", self.field) + + +#: The archived producer commit every historical claim in this module is read +#: from. The retired US data package is named by commit rather than by package +#: name because this repository's launch contract forbids naming it in the live +#: tree (``test_no_incumbent_data_package_references_in_live_tree``); paths in +#: the citations below are relative to that repository's Python package +#: directory. Read a blob with ``git show :/``. +ARCHIVED_PRODUCER_COMMIT = "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + +#: Where the roster's field meanings come from. The archived file is a comment +#: block that itself cites an IRS booklet; that booklet was not read here, so a +#: meaning below is "the label the archived producer attached to this code", +#: not an independently authenticated IRS definition. +ROSTER_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_totals.yaml" + " (its own cited source: 2014 PUF General Description Booklet, pp. 33-36," + " June 2020)" +) + +_STRUCTURAL_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/puf.py preprocess_puf" +) + +#: Field code to the meaning the archived codebook records for it. Money only. +_DOCUMENTED_MONEY_MEANINGS: tuple[tuple[str, str], ...] = ( + ("E00100", "AGI"), + ("E00200", "Wages and salaries"), + ("E00300", "Taxable interest"), + ("E00400", "Tax-exempt interest"), + ("E00600", "Ordinary dividends"), + ("E00650", "Qualified dividends"), + ("E00700", "State/local tax refund"), + ("E00800", "Alimony received"), + ("E00900", "Business income/loss"), + ("E01000", "Net capital gain/loss"), + ("E01100", "Capital gain distributions"), + ("E01200", "Other gains/losses"), + ("E01400", "Taxable IRA distributions"), + ("E01500", "Total pensions and annuities"), + ("E01700", "Taxable pensions and annuities"), + ("E02100", "Farm income/loss (Sch F)"), + ("E02300", "Unemployment compensation"), + ("E02400", "Social security benefits"), + ("E03150", "IRA deduction"), + ("E03210", "Student loan interest"), + ("E03220", "Educator expenses"), + ("E03230", "Tuition and fees"), + ("E03240", "Domestic production deduction"), + ("E03270", "Self-employed health insurance"), + ("E03290", "HSA deduction"), + ("E03300", "Self-employed SEP/SIMPLE"), + ("E03400", "Early withdrawal penalty"), + ("E03500", "Alimony paid"), + ("E07240", "Saver's credit"), + ("E07260", "Residential energy credit"), + ("E07300", "Foreign tax credit"), + ("E07400", "General business credit"), + ("E07600", "Prior year min tax credit"), + ("E09600", "AMT"), + ("E09700", "Recapture of investment credit"), + ("E09800", "Unreported SE tax"), + ("E09900", "Penalty on early withdrawal"), + ("E11200", "Excess FICA withheld"), + ("E17500", "Medical expenses"), + ("E18400", "State/local income tax"), + ("E18500", "Real estate taxes"), + ("E19200", "Interest paid"), + ("E19800", "Cash contributions"), + ("E20100", "Non-cash contributions"), + ("E20400", "Misc itemized deductions"), + ("E20500", "Casualty/theft loss"), + ("E22250", "Short-term capital gain/loss"), + ("E24515", "Unrecaptured Sec. 1250 gain"), + ("E24518", "Collectibles gain/loss"), + ("E25850", "Rental income (Sch E gross)"), + ("E25860", "Rental loss (Sch E gross)"), + ("E25940", "Partnership income (gross)"), + ("E25960", "Partnership loss (gross)"), + ("E25980", "Partnership net income"), + ("E26180", "S-corp loss (gross)"), + ("E26190", "S-corp net income"), + ("E26270", "Partnership/S-corp net"), + ("E26390", "Estate/trust income (gross)"), + ("E26400", "Estate/trust loss (gross)"), + ("E27200", "Farm rental income (Sch E)"), + ("E30400", "SE income (taxpayer)"), + ("E30500", "SE income (spouse)"), + ("E32800", "Child care credit expenses"), + ("E58990", "Investment interest (4952)"), + ("E62900", "AMT foreign tax credit"), + ("E87521", "American Opportunity Credit"), + ("P08000", "Other credits"), + ("P23250", "Long-term capital gain/loss"), + ("T27800", "Farm income (Sch J)"), +) + +_SOI_MEASURE_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:utils/soi.py puf_measures" +) + +_SOI_RENAME_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/uprate_puf.py SOI_TO_PUF_STRAIGHT_RENAMES" +) + +_SCREENED_FIELD_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_utils.py SCREENED_FIELDS" +) + +#: The raw TY2015 PUF fields this lineage names, with the role each one plays. +#: A **proposal**, not an authenticated raw-source manifest: every money code +#: the archived codebook describes, plus the four structural codes, the one +#: screened code, and five money codes the codebook omits that only the +#: archived SOI rename/measure tables name -- each cited to where its meaning +#: was actually read. It is not every code this repository reads, and not every +#: code the delivered file contains. It declares no factor for any field. +#: Adding or removing a field is a reviewed change, not something a recipe may +#: do implicitly. +PROPOSED_TY2015_FIELD_ROSTER: tuple[RosterEntry, ...] = ( + RosterEntry( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + "Record id (the archived producer copies it to household_id)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + "Sample weight, stored times 100 (the archived producer divides by 100)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "MARS", + FieldRole.CODE, + "Filing status: 0 marks four aggregate disclosure records; " + "1 single, 2 joint, 3 separate, 4 head of household", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "XTOT", + FieldRole.COUNT, + "Exemptions count", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "P22250", + FieldRole.MONEY, + "Short-term capital gains", + _SCREENED_FIELD_CITATION, + ), + RosterEntry( + "E02500", + FieldRole.MONEY, + "Taxable social security (the archived SOI rename binds the SOI series" + " taxable_social_security to this code; the codebook has no entry)", + _SOI_RENAME_CITATION, + ), + # Four money codes the codebook omits and only the archived SOI measure + # table names. E06500 is bound to two different SOI series there, which is + # itself an open question a successor must settle rather than inherit. + RosterEntry("E04800", FieldRole.MONEY, "Taxable income", _SOI_MEASURE_CITATION), + RosterEntry( + "E06500", + FieldRole.MONEY, + "Income tax; the archived measure table binds it to both" + " total_income_tax and income_tax_before_credits", + _SOI_MEASURE_CITATION, + ), + RosterEntry( + "E08800", FieldRole.MONEY, "Income tax after credits", _SOI_MEASURE_CITATION + ), + RosterEntry( + "E19700", + FieldRole.MONEY, + "Charitable contributions deduction", + _SOI_MEASURE_CITATION, + ), + *( + RosterEntry(field, FieldRole.MONEY, meaning, ROSTER_CITATION) + for field, meaning in _DOCUMENTED_MONEY_MEANINGS + ), +) + +#: Codes the archived producer names in executable code at +#: :data:`ARCHIVED_PRODUCER_COMMIT` while nothing there says what they mean: +#: ``DSI`` and ``EIC`` appear only in a predictor list, ``E25920`` only as a +#: subtracted term, and ``E87530`` only inside two helpers whose names imply a +#: Lifetime-Learning qualified-tuition meaning that no comment states. They are +#: deliberately absent from the roster: a successor must resolve them against +#: real source documentation rather than inherit a guess. This is **not** a +#: complete inventory of undocumented codes -- it is what this slice tripped +#: over, and closing it is a documented remaining obligation. ``E22250`` is in +#: the roster because the codebook documents it, but no archived code path +#: reads it -- every path uses ``P22250`` -- so whether both codes exist in the +#: delivered file is open. +UNDOCUMENTED_SOURCE_FIELDS: tuple[str, ...] = ("DSI", "E25920", "E87530", "EIC") + +#: The source tax year those roster fields are observed in. +ROSTER_SOURCE_REFERENCE_YEAR = PUF_SOURCE_YEAR + + +def roster_role(field: str) -> FieldRole: + """The proposed role of ``field``; refuses a field the roster does not name.""" + for entry in PROPOSED_TY2015_FIELD_ROSTER: + if entry.field == field: + return entry.role + raise PufGrowthRefusalError("FIELD_NOT_IN_ROSTER", field) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py new file mode 100644 index 000000000..0c727a5aa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py @@ -0,0 +1,375 @@ +"""Graph nodes that run the pure PUF growth transform inside a real population. + +Three nodes, in the shape the US build already uses for a source operation: + +``.boundary`` + A ``FILTER`` node that keeps every person. It exists because a column a + ``CREATE`` node loaded can only be rewritten once a structural node has + opened a new version. Each kernel reconstructs the declared contract from + its parameters before its computation. The boundary's receipt states the + contract digest, the factor authority, and whether that authority could + ever stand behind a release. + +``.money`` + Owns each declared money column as a rewrite, plus the two provenance + columns. Its whole computation is + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`. + +``.design_weight`` + Owns only the declared design-weight column, through + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + A separate node so the graph itself records that money growth and weight + growth are different operations reading different factor series. + +The contract travels as node parameters — the recipe identity document and the +factor-table document, verbatim — so a changed contract changes the node key +and nothing is served from the store across a contract change. Codes, +identifiers and counts are read but never owned. Two executor mechanisms are +what actually keep them byte-identical: a kernel's returned coordinates must be +exactly the node's declared outputs, so it cannot write them at all; and the +projected input view is frozen and its digest compared before and after the +kernel runs, so it cannot mutate them in place either. + +The design weight this stage grows is the **raw source column**. Promoting it +into a frame's typed design weights is a separate, later obligation with its +own mass policy; nothing here touches ``Frame`` weights. The weight column is +owned as ``float64``, as are the money rewrites; a base version carrying any +of those columns as an integer would need a +declared dtype change, which is a reviewed decision rather than a cast here. + +**The closed read set is the transform's, not the frame's.** +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` refuses a table +carrying a column no rule declares. These nodes cannot make that promise about +the *population*: a ``CREATE`` node upstream may have loaded raw money columns +this contract never mentions, and they pass through the stage ungrown, because a +node only sees the slices it declares. Closing that gap needs a source-stage +contract that declares the full raw column inventory and checks the population +against it; that is a documented remaining obligation, not something this stage +silently covers. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from importlib import import_module + +import pandas as pd + +from microcosm.build.us_runtime.puf_growth import ( + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + CompiledPufGrowth, + GrowthFactorTable, + PufGrowthRefusalError, + apply_puf_design_weight_growth, + apply_puf_growth, + compile_puf_growth, + parse_growth_recipe, + require_transformable, +) +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) + +__all__ = [ + "PUF_GROWTH_BOUNDARY_NODE", + "PUF_GROWTH_DESIGN_WEIGHT_NODE", + "PUF_GROWTH_MONEY_NODE", + "PUF_GROWTH_PROVENANCE_RECID_DTYPE", + "PUF_GROWTH_STAGE", + "PufGrowthBoundaryKernel", + "PufGrowthDesignWeightKernel", + "PufGrowthMoneyKernel", + "register_us_puf_growth_kernels", + "us_puf_growth_nodes", +] + +#: Stage name, and the node ids derived from it. +PUF_GROWTH_STAGE = "us_puf_growth" +PUF_GROWTH_BOUNDARY_NODE = f"{PUF_GROWTH_STAGE}.boundary" +PUF_GROWTH_MONEY_NODE = f"{PUF_GROWTH_STAGE}.money" +PUF_GROWTH_DESIGN_WEIGHT_NODE = f"{PUF_GROWTH_STAGE}.design_weight" + +#: The dtype the provenance record identifier is owned as. The raw identifier +#: is an integer record id; a float copy of it would not be a faithful key. +PUF_GROWTH_PROVENANCE_RECID_DTYPE = "int64" + +#: Modules whose bytes are attested by every kernel's implementation hash. The +#: pure contract is included because a change there changes what these kernels +#: compute even when this file is untouched. +_IMPLEMENTATION_MODULES = ( + "microcosm.build.monetary_targets", + "microcosm.build.us_runtime.puf_growth", + "microcosm.build.us_runtime.puf_growth_graph", + "microcosm.build.us_runtime.puf_source_agi", +) + +_PARAMS = ("entity", "factor_table", "recipe") + + +class _PufGrowthKernel(KernelBase): + """Shared parameter contract and code identity for the three kernels.""" + + def implementation_hash(self) -> str: + # Keep the declared dependencies in the hash the way KernelBase does: + # without them a declared distribution would bind only its name into + # the node key, never its installed version. + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + @staticmethod + def contract(params: Mapping[str, object]) -> tuple[str, CompiledPufGrowth]: + """Recompile the declared contract from the node's own parameters.""" + if set(params) != set(_PARAMS): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + entity = params["entity"] + recipe_document = params["recipe"] + factor_document = params["factor_table"] + if not ( + type(entity) is str + and type(recipe_document) is str + and type(factor_document) is str + ): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + recipe = parse_growth_recipe(recipe_document.encode("ascii")) + factors = GrowthFactorTable.from_bytes(factor_document.encode("ascii")) + return entity, compile_puf_growth(recipe, factors) + + +def _identifier_index(context: KernelContext, entity: str) -> pd.Index: + column = f"{entity}_id" + table = context.tables[entity] + if column not in table: + raise PufGrowthRefusalError("ENTITY_ID_COLUMN", entity) + return pd.Index(table[column], name=column) + + +def _declared_source(context: KernelContext, entity: str, fields: Sequence[str]): + table = context.tables[entity] + missing = [name for name in fields if name not in table.columns] + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + return table.loc[:, list(fields)] + + +def _contract_receipt(compiled: CompiledPufGrowth) -> dict[str, object]: + return { + "contract_sha256": compiled.sha256, + "factor_table_sha256": compiled.factors.sha256, + "factor_table_id": compiled.factors.table_id, + "factor_authority": str(compiled.factors.authority), + "release_eligible": compiled.release_eligible, + "target_year": compiled.target_year, + } + + +class PufGrowthBoundaryKernel(_PufGrowthKernel): + """Open a version for the rewrite, and publish the contract it will use.""" + + ref = "us.puf.growth.boundary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + _, compiled = self.contract(context.params) + person = context.tables["person"] + return KernelResult( + keep=pd.Series( + True, index=pd.Index(person.person_id, name="person_id"), dtype=bool + ), + receipt={"puf_growth_contract": _contract_receipt(compiled)}, + ) + + +class PufGrowthMoneyKernel(_PufGrowthKernel): + """Grow every declared money field once, and copy the provenance columns.""" + + ref = "us.puf.growth.money@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + source = _declared_source(context, entity, compiled.money_view_fields) + result = apply_puf_growth(source, compiled) + ids = _identifier_index(context, entity) + grown = result.table + if grown[PROVENANCE_RECID_COLUMN].dtype != PUF_GROWTH_PROVENANCE_RECID_DTYPE: + raise PufGrowthRefusalError( + "PROVENANCE_RECID_DTYPE", PROVENANCE_RECID_COLUMN + ) + columns = { + (entity, name): pd.Series( + grown[name].to_numpy(copy=True), index=ids, dtype="float64" + ) + for name in compiled.money_fields + } + columns[(entity, PROVENANCE_RECID_COLUMN)] = pd.Series( + grown[PROVENANCE_RECID_COLUMN].to_numpy(copy=True), + index=ids, + dtype=PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ) + columns[(entity, PROVENANCE_SOURCE_AGI_COLUMN)] = pd.Series( + grown[PROVENANCE_SOURCE_AGI_COLUMN].to_numpy(copy=True), + index=ids, + dtype="float64", + ) + return KernelResult( + columns=columns, + receipt={"puf_growth": json.loads(result.receipt_bytes)}, + ) + + +class PufGrowthDesignWeightKernel(_PufGrowthKernel): + """Grow the raw design-weight column, and nothing else.""" + + ref = "us.puf.growth.design_weight@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + rule = compiled.design_weight_rule + if rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + source = _declared_source(context, entity, (rule.field,)) + grown = apply_puf_design_weight_growth( + source[rule.field].to_numpy(dtype="float64"), compiled + ) + factor = compiled.factor_for(rule.field, rule.sign_branch) + return KernelResult( + columns={ + (entity, rule.field): pd.Series( + grown, index=_identifier_index(context, entity), dtype="float64" + ) + }, + receipt={ + "puf_design_weight_growth": { + **_contract_receipt(compiled), + "field": rule.field, + "series": factor.series, + "source_year": factor.source_year, + "ratio_bits": factor.ratio_bits, + "rows": int(len(grown)), + } + }, + ) + + +def us_puf_growth_nodes( + compiled: CompiledPufGrowth, + *, + base: str, + entity: str, + person_boundary_columns: Sequence[str], + stage: str = PUF_GROWTH_STAGE, +) -> tuple[Node, ...]: + """Declare the boundary, money and design-weight nodes for one contract. + + Args: + compiled: The contract these nodes carry. Its recipe identity and its + factor document become node parameters verbatim, so the node keys + move whenever the contract moves. + base: Id of the population version to filter. + entity: Entity whose table carries the raw source fields. + person_boundary_columns: Person columns the boundary reads. A ``FILTER`` + node returns a person mask, so it must declare a person slice. + stage: Node-id prefix, for a graph that runs more than one contract. + + Returns: + The boundary node, the money node, and the design-weight node. + + Raises: + PufGrowthRefusalError: If the contract declares no design weight, names + no person boundary column, or could not run at all + (:func:`~microcosm.build.us_runtime.puf_growth.require_transformable`). + """ + if type(compiled) is not CompiledPufGrowth: + raise PufGrowthRefusalError("COMPILED_TYPE") + weight_rule = compiled.design_weight_rule + if weight_rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + if not person_boundary_columns: + raise PufGrowthRefusalError("PERSON_BOUNDARY_COLUMNS") + # Everything the money kernel needs is checked here, at declaration time, + # rather than after a graph run has already loaded a source and written the + # design-weight column. + require_transformable(compiled) + params = { + "entity": entity, + "factor_table": compiled.factors.document.decode("ascii"), + "recipe": compiled.recipe.identity.decode("ascii"), + } + money_view = Slice(entity, compiled.money_view_fields) + boundary = f"{stage}.boundary" + return ( + Node( + boundary, + PufGrowthBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", tuple(person_boundary_columns)),), + params=params, + description="Open a version so the declared money columns can be rewritten.", + ), + Node( + f"{stage}.money", + PufGrowthMoneyKernel.ref, + population=boundary, + inputs=(money_view,), + outputs=( + *( + Owned(entity, name, "float64", rewrite=True) + for name in compiled.money_fields + ), + Owned( + entity, + PROVENANCE_RECID_COLUMN, + PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ), + Owned(entity, PROVENANCE_SOURCE_AGI_COLUMN, "float64"), + ), + params=params, + description="Grow declared money once and carry source provenance.", + ), + Node( + f"{stage}.design_weight", + PufGrowthDesignWeightKernel.ref, + population=boundary, + inputs=(Slice(entity, (weight_rule.field,)),), + outputs=(Owned(entity, weight_rule.field, "float64", rewrite=True),), + params=params, + description="Grow the raw design weight by its own declared series.", + ), + ) + + +def register_us_puf_growth_kernels(registry: KernelRegistry) -> None: + """Register the three PUF growth kernels.""" + registry.register(PufGrowthBoundaryKernel()) + registry.register(PufGrowthMoneyKernel()) + registry.register(PufGrowthDesignWeightKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py new file mode 100644 index 000000000..73e16294b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py @@ -0,0 +1,689 @@ +"""The thirteen-field 2015 PUF monetary source projection: the twelve accepted +columns plus the publisher's reported ``E00100``. + +The reviewed twelve-field increment +(:mod:`microcosm.build.us_runtime.puf_monetary_source`) decodes wages, the two +interest columns, the two dividend columns, the Schedule C and Schedule F net +results, the two pension columns and the three Schedule D quantities. The PUF +growth contract additionally requires ``E00100`` as a declared money field +(:class:`~microcosm.build.us_runtime.puf_growth.GrowthRecipe`), and the +provenance AGI column that contract writes is a *copy of its own input*, not an +independent source. So a growth run over this lineage needs a projection that +carries reported AGI, and this module is that projection. + +Why a separate module, and what it preserves +-------------------------------------------- +The twelve-field kernel hashes the bytes of ``puf_monetary_source.py`` and +``puf_raw_source.py`` into its implementation hash, and the executor folds that +hash into the artifact store key. Editing either module would move the accepted +twelve-field artifact's key. A repository test also pins the twelve-field +generator's own sha256 to the packaged document. **Nothing here edits any of +them.** This module imports the reviewed decoder, the reviewed envelope +helpers and the reviewed fact derivation, so the thirteenth column is decoded +by exactly the code that decodes the other twelve rather than by a second +implementation of the same contract. The private names it imports are imported +deliberately, and each one is a piece of the reviewed contract this edition +must not restate. + +What is the same +---------------- +Every property of the twelve-field contract holds here unchanged: whole-dollar +signed integer tokens on individual returns, the separately bounded +disclosure-aggregate lexical grammar, the minimum-int64 out-of-universe +sentinel, the ``amount_known == (disclosure_aggregate == 0)`` rule derived from +the authenticated flag on every access, all 207,696 delivered rows retained, +the three disjoint row classes, the binding to the reviewed status artifact, +and ``target_year_use: refused_pending_independent_growth_decision`` on every +column including ``E00100``. + +What is different +----------------- +One more column, and a different artifact identity. The type is +``microcosm.us.puf_2015_monetary_agi_source_projection`` version 1, with its +own magic line. A twelve-field payload fed to this reader is refused **by +name** rather than reinterpreted, because the two artifacts carry different +column sets and silently reading one as the other is exactly the failure the +twelve-field module's superseded-magic list exists to prevent. Nothing here +derives, reconstructs or reconciles AGI: ``E00100`` is the delivered reported +field, and the document records that the booklets state no component identity +between it and the twelve. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +# The reviewed twelve-field contract, imported rather than restated. The +# private names are deliberate: the thirteenth column is authenticated, +# encoded, read back and described by exactly the reviewed code, so a change +# there cannot leave a stale second copy of the same contract here. What this +# module owns is its own artifact identity, not its own parser. +from .puf_monetary_source import ( + _AGGREGATE_COLUMN_FACT_KEYS, + _ENVELOPE_FACT_KEYS, + _ENVELOPE_HEADER_KEYS, + _LEXICAL_KIND, + _RETURN_COLUMN_FACT_KEYS, + _STRUCTURAL_DTYPES, + _TYPED_KIND, + AGGREGATE_ROW_CLASS, + BODY_MAX_BYTES, + HEADER_MAX_BYTES, + MONETARY_PROJECTION_MAGIC, + ROW_CLASSES, + SUPERSEDED_MAGIC, + PufMonetaryProjection, + PufMonetarySourceProjection, + _check_envelope_sources, + _envelope_column_order, + _envelope_columns, + _envelope_int, + _exact_keys, + _facts_from_arrays, + _fixed_ascii, + _hex64, + _projection_from_document, + _read_fixed_ascii, + _refuse, + _thawed, + _verify_readback, + decode_puf_monetary_source, + packaged_projection, + payload_bound, +) +from .puf_raw_source import ( + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGI_FIELD", + "AGI_PROJECTION_EDITION", + "AGI_PROJECTION_MAGIC", + "AGI_PROJECTION_NODE", + "AGI_PROJECTION_STAGE", + "AGI_PROJECTION_TYPE", + "TWELVE_FIELD_MAGIC", + "USPufMonetaryAgiSourceKernel", + "decode_agi_projection", + "encode_agi_projection", + "fixture_agi_projection_document", + "packaged_agi_projection", + "register_us_puf_monetary_agi_source_kernels", + "us_puf_monetary_agi_source_node", +] + +#: The one column this edition adds to the accepted twelve. +AGI_FIELD = "E00100" + +#: The edition marker the packaged document must declare about itself. +AGI_PROJECTION_EDITION = "thirteen_field_reported_agi" + +AGI_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_agi_source_projection", 1 +) +AGI_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_agi_source_projection/1\n" + +#: Magic lines this reader refuses by name rather than reinterpreting. These +#: are the twelve-field artifact's own magics: those payloads carry a +#: different column set, and reading one here would report a twelve-column +#: artifact as a thirteen-column one. +TWELVE_FIELD_MAGIC = (MONETARY_PROJECTION_MAGIC, *SUPERSEDED_MAGIC) + +AGI_PROJECTION_STAGE = "us_puf_monetary_agi_source" +AGI_PROJECTION_NODE = f"{AGI_PROJECTION_STAGE}.projection" + +_AGI_RESOURCE = "puf_2015_monetary_agi_source_projection.json" +_AGI_PACKAGE = "microcosm.build.us_runtime" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_agi_projection" + + +# -------------------------------------------------------------------------- +# The packaged thirteen-field document +# -------------------------------------------------------------------------- + + +def _check_agi_edition( + document: Mapping[str, object], projection: PufMonetaryProjection +) -> None: + """Bind this edition to the accepted twelve-field document, at load time. + + The edition block is not decoration: it names the parent document by + digest and lists the parent's projected fields, so a thirteen-field + document that quietly dropped or renamed one of the accepted twelve — or + that was generated against a different parent — is refused here rather + than discovered downstream. + """ + + edition = document.get("projection_edition") + if not isinstance(edition, Mapping): + raise _refuse("AGI_PROJECTION_EDITION_MISSING") + if edition.get("edition") != AGI_PROJECTION_EDITION: + raise _refuse("AGI_PROJECTION_EDITION") + if list(edition.get("adds", ())) != [AGI_FIELD]: + raise _refuse("AGI_PROJECTION_EDITION_ADDS") + parent = packaged_projection() + if edition.get("parent_document_canonical_sha256") != parent.sha256: + raise _refuse("AGI_PROJECTION_PARENT_DOCUMENT") + if tuple(edition.get("parent_projected_fields", ())) != parent.fields: + raise _refuse("AGI_PROJECTION_PARENT_FIELDS") + fields = projection.fields + if AGI_FIELD not in fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if set(fields) != set(parent.fields) | {AGI_FIELD}: + raise _refuse("AGI_PROJECTION_FIELD_SET") + if len(fields) != len(parent.fields) + 1: + raise _refuse("AGI_PROJECTION_FIELD_COUNT") + # Every accepted column must still declare exactly what it declared + # upstream. This edition adds a column; it does not restate the twelve. + upstream = {column.field: column for column in parent.columns} + for column in projection.columns: + if column.field == AGI_FIELD: + continue + if column != upstream[column.field]: + raise _refuse("AGI_PROJECTION_PARENT_COLUMN", column.field) + + +def _packaged_agi_bytes() -> bytes: + return resources.files(_AGI_PACKAGE).joinpath(_AGI_RESOURCE).read_bytes() + + +_PACKAGED_AGI: PufMonetaryProjection | None = None + + +def packaged_agi_projection() -> PufMonetaryProjection: + """The one closed thirteen-field document a production run may use.""" + + global _PACKAGED_AGI + if _PACKAGED_AGI is None: + document = json.loads(_packaged_agi_bytes().decode("utf-8")) + projection = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + _check_agi_edition(document, projection) + _PACKAGED_AGI = projection + return _PACKAGED_AGI + + +def fixture_agi_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a thirteen-field projection on the explicit non-production route. + + Same rules as the twelve-field fixture route: ``route="test_fixture"``, + the invented-fixture authority, and no restatement of a packaged source + pin. It must still name ``E00100``, because a fixture without it would be + testing the twelve-field contract under this module's name. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def encode_agi_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode a thirteen-field projection into its own bounded envelope. + + Byte-for-byte the twelve-field layout — magic line, big-endian header + length, canonical JSON header, little-endian typed bodies then fixed-width + lexical bodies — under a different magic and type, with the column bodies + the shared order helper produces for this projection. + """ + + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": AGI_PROJECTION_TYPE.schema_version, + "type": [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (AGI_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _resolve_agi_projection( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> PufMonetaryProjection: + """The thirteen-field projection this payload may be checked against.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + if route == "packaged": + packaged = packaged_agi_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + # A fixture-route payload has no reconstructible projection document, so + # there is nothing to close its column set against. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + + +def decode_agi_projection( + payload: bytes, + projection: PufMonetaryProjection | None = None, + *, + expected_status_artifact_sha256: str | None = None, +) -> PufMonetarySourceProjection: + """Decode a thirteen-field payload and re-prove it, never trusting it. + + Every reported fact is recomputed from the arrays actually read back by + the reviewed derivation and must agree exactly, and every row re-parses + under the universe its authenticated flag selects. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(AGI_PROJECTION_MAGIC): + if any(payload.startswith(magic) for magic in TWELVE_FIELD_MAGIC): + raise _refuse("ENVELOPE_TWELVE_FIELD_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(AGI_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != AGI_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_agi_projection(header, projection) + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=MappingProxyType(dict(recomputed["column_metadata"])), + ) + + +# -------------------------------------------------------------------------- +# The graph producer +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", + "microcosm.build.us_runtime.puf_monetary_agi_projection", +) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_agi_source/implementation/1\n" + + +class USPufMonetaryAgiSourceKernel(KernelBase): + """Read the pinned main delivery and produce the thirteen-field projection.""" + + ref = "us.puf.monetary_agi_source.projection@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = ( + packaged_agi_projection() if projection is None else projection + ) + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if AGI_FIELD not in self.projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if self.projection.route == "packaged": + packaged = packaged_agi_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_agi_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_agi_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "projection_edition": AGI_PROJECTION_EDITION, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + "agi_field": AGI_FIELD, + "agi_is_derived": False, + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +def us_puf_monetary_agi_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = AGI_PROJECTION_STAGE, +) -> Node: + """Declare the thirteen-field producer inside an explicitly named host.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_agi_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + if AGI_FIELD not in resolved.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return Node( + f"{stage}.projection", + USPufMonetaryAgiSourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),), + description=( + "Decode the twelve accepted 2015 PUF monetary source columns plus " + "reported E00100 at return grain, bound to the reviewed status " + "artifact. Owns no population columns and admits no target year." + ), + ) + + +def register_us_puf_monetary_agi_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the thirteen-field producer kernel into ``registry``.""" + + registry.register( + USPufMonetaryAgiSourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py new file mode 100644 index 000000000..5c91c5a55 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py @@ -0,0 +1,2036 @@ +"""The first monetary amounts typed out of the 2015 PUF delivery bytes. + +The reviewed raw source slice +(:mod:`microcosm.build.us_runtime.puf_raw_source`) types six of the 222 +delivered main columns and the six demographic codes, reports +``amounts_decoded: False``, and drops the other 216 main columns after +checking their header name, order and record width. This module is the next +bounded increment: it authenticates the **whole** delivered layout against the +retained IRS booklets and decodes a small, explicitly enumerated set of +monetary columns at source-return grain. + +Two things it produces, and one it refuses. + +*The full layout is authenticated, not merely counted.* The packaged document +``puf_2015_monetary_source_projection.json`` locates every one of the 222 main +columns and 7 demographic columns in the September 2022 booklet by the +publisher's own position number and name, retains that record-layout row +verbatim with its exact character span and span digest, and corroborates each +against the February 2023 edition. Nothing about a column's role is inferred +from an ``E``, ``P``, ``S`` or ``T`` prefix or from membership of the +amount-field list: the publisher's own "Misc Codes" block sits inside that list +and holds ``RECID``, ``S006``, ``S008``, ``S009``, ``WSAMP`` and ``TXRT``, none +of which is money and one of which (``TXRT``) carries an implied decimal point. + +*Twelve monetary columns are decoded, each with its own declaration.* Wages, +taxable and tax-exempt interest, ordinary and qualified dividends, the +Schedule C and Schedule F net results, pensions received and pensions in AGI, +and the three Schedule D quantities the booklets define. Each declares exact +units, sign evidence, missingness policy, return grain, source-period status +and source pin identity, and each is refused for target-year use. + +*No amount is grown, restated or combined.* The row period is unresolved — the +cover says Tax Year 2015, the code definitions print FLPDYR 2011-2014, the +disclosure section removed pre-2012 returns, and the reviewed header audit +observed 2012-2015 — so every column carries +``target_year_use: refused_pending_independent_growth_decision``. Component +relationships the booklets do not state (qualified dividends inside ordinary +dividends, pensions in AGI inside pensions received, ``E01000`` out of +``P22250`` and ``P23250``) are recorded as unresolved and never computed. +``E22250`` is absent from the delivered layout and from both booklets and is +never aliased onto ``P22250``. An uncapped self-employment earnings derivation +from ``E30400``/``E30500`` is refused with the publisher text that refuses it. + +Three properties are load-bearing. + +*Every delivered main record is projected.* All 207,696 rows, including the +88,021 with no demographic row and the four disclosure aggregate records. The +aggregate rows are tagged and every derived fact is reported for three +disjoint row classes — aggregate, demographic-matched and +demographic-unmatched — so an aggregate can never be averaged into a +per-return statistic and an unmatched row is never a zero. + +*The individual-return amount universe is explicit.* A row is inside it if and +only if its authenticated disclosure-aggregate flag is 0, and the decoded +object exposes that rule as a read-only ``amount_known`` mask derived from the +flag on every access. Individual returns keep the unchanged integer, +whole-dollar, twelve-digit grammar, and a fractional token on one of them +still refuses. The four disclosure aggregates are outside that universe: their +typed slots hold the minimum-int64 out-of-universe sentinel — never a zero, +which is a delivered value in these columns — and their delivered tokens are +retained exactly under a separate, explicitly bounded lexical grammar that is +an operational input bound for this pinned file rather than a publisher +specification. Nothing converts an aggregate token to a number, rounds it, +grows it or drops its row. The aggregate row class carries only token-class +counts and a token width, so no header, receipt or summary can state an +aggregate amount, and the exact row/token pairs are available only through a +local accessor on the decoded object. + +*The projection is bound to the reviewed status artifact.* The node takes +``microcosm.us.puf_2015_raw_return_status`` as an artifact input and refuses +unless its own ``RECID`` sequence and aggregate flags are element-wise +identical to that artifact's. ``demographic_status`` is carried from it rather +than re-joined, so the two artifacts cannot disagree about which returns +matched. + +*The raw producer has an explicit packaging revision.* This module imports +:mod:`puf_raw_source` for its definition, codec and artifact decoder. Moving +the unchanged definition bytes into ``us_runtime`` changed the raw resource +loader and its implementation identity. New runs use that revised identity; +previously accepted raw artifacts retain their original keys and evidence. +The CSV reader helpers here remain local to this monetary projection. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +from .puf_raw_source import ( + PUF_AGGREGATE_RECIDS, + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGGREGATE_ROW_CLASS", + "AGGREGATE_SENTINEL", + "AMOUNT_KNOWN_RULE", + "BODY_MAX_BYTES", + "HEADER_MAX_BYTES", + "LEXICAL_WIDTH_MAX", + "MONETARY_PROJECTION_NODE", + "MONETARY_PROJECTION_STAGE", + "MONETARY_PROJECTION_SUMMARY_NODE", + "MONETARY_PROJECTION_TYPE", + "PufMonetaryProjection", + "PufMonetaryRefusalError", + "PufMonetarySourceDocument", + "ProjectedColumn", + "RETURN_ROW_CLASSES", + "SUPERSEDED_MAGIC", + "USPufMonetaryProjectionSummaryKernel", + "USPufMonetarySourceKernel", + "decode_monetary_projection", + "decode_puf_monetary_source", + "encode_monetary_projection", + "fixture_projection_document", + "packaged_projection", + "projection_document_json", + "register_us_puf_monetary_source_kernels", + "us_puf_monetary_projection_summary_node", + "us_puf_monetary_source_node", +] + +MONETARY_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_source_projection", 3 +) +MONETARY_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_source_projection/3\n" + +#: Magic lines this reader refuses by name instead of reinterpreting. No +#: genuine schema-1 or schema-2 monetary artifact was ever completed — the +#: first genuine run refused before writing one — but invented artifacts of +#: both do exist in test evidence, and the amount universe differs, so an old +#: payload gets its own refusal rather than falling through the generic magic +#: check as an unrecognised blob. +SUPERSEDED_MAGIC = ( + b"microcosm.us.puf_2015_monetary_source_projection/1\n", + b"microcosm.us.puf_2015_monetary_source_projection/2\n", +) + +MONETARY_PROJECTION_STAGE = "us_puf_monetary_source" +MONETARY_PROJECTION_NODE = f"{MONETARY_PROJECTION_STAGE}.projection" +MONETARY_PROJECTION_SUMMARY_NODE = f"{MONETARY_PROJECTION_STAGE}.projection_summary" + +#: Bound on the encoded artifact's column bodies, matching the raw slice's own +#: private bound. Twelve projected columns over the delivered 207,696 rows is +#: 262 bytes a row, so about 51.9 MiB; a thirteenth column of the same shape +#: would still fit and a much wider selection would not. +BODY_MAX_BYTES = 64 * 1024 * 1024 + +#: Bound on the canonical JSON header. The header carries per-column facts for +#: three row classes, so it is larger than the status artifact's. +HEADER_MAX_BYTES = 64 * 1024 + +#: The widest lexical allocation a projected column may declare. +LEXICAL_WIDTH_MAX = 64 + +_PROJECTION_RESOURCE = "puf_2015_monetary_source_projection.json" +_PROJECTION_PACKAGE = "microcosm.build.us_runtime" +_PROJECTION_SCHEMA = "microcosm.us.puf_2015_monetary_source_projection" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STRUCTURAL_COLUMNS = ("RECID", "disclosure_aggregate", "demographic_status") +_STRUCTURAL_DTYPES = { + "RECID": " None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufMonetaryRefusalError: + return PufMonetaryRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed projection document +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ProjectedColumn: + """One enumerated monetary column, with everything the parse needs.""" + + field: str + concept: str + concept_group: str + delivered_index: int + publisher_position: int + publisher_label: str + dtype: str + lexical_width: int + grammar: str + max_digits: int + publisher_signed_marker: bool + negative_values: str + fit_admission: str + unresolved: tuple[str, ...] + + @property + def numpy_dtype(self) -> str: + return _DECLARED_DTYPE[self.dtype] + + +@dataclass(frozen=True) +class PufMonetaryProjection: + """A closed layout authentication plus the enumerated projection.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + columns: tuple[ProjectedColumn, ...] + raw_definition_sha256: str + main_delivered_header: tuple[str, ...] + demographic_delivered_header: tuple[str, ...] + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("utf-8") + + @property + def fields(self) -> tuple[str, ...]: + return tuple(column.field for column in self.columns) + + +#: Kept as a separate public name so a caller can talk about the document +#: without holding a parsed projection. +PufMonetarySourceDocument = Mapping[str, object] + + +def _frozen(value: object) -> object: + """A deeply immutable view, as the raw source definition already does.""" + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _thawed(value: object) -> object: + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("PROJECTION_DIGEST", label) + return value + + +def _positive_int(value: object, label: str, *, minimum: int = 1) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("PROJECTION_INTEGER", label) + return value + + +def _layout_names( + entry: Mapping[str, object], label: str, reasons: frozenset[str] +) -> tuple[str, ...]: + columns = entry["columns"] + if not isinstance(columns, (list, tuple)) or not columns: + raise _refuse("PROJECTION_LAYOUT", label) + names: list[str] = [] + for position, column in enumerate(columns): + if not isinstance(column, Mapping): + raise _refuse("PROJECTION_LAYOUT", label) + name = column["name"] + if not isinstance(name, str) or not name: + raise _refuse("PROJECTION_LAYOUT_NAME", label) + if column["delivered_index"] != position: + raise _refuse("PROJECTION_LAYOUT_INDEX", name) + admission = column["admission"] + if admission not in _ADMISSIONS: + raise _refuse("PROJECTION_LAYOUT_ADMISSION", name) + if admission == "not_selected": + # An unselected column carries a reason from the document's own + # closed list, so nothing is dropped by omission. + if column.get("not_selected_reason") not in reasons: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + elif "not_selected_reason" in column: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + row = column["layout_row"] + if not isinstance(row, Mapping): + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + text = row["text"] + if not isinstance(text, str) or not text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + if hashlib.sha256(text.encode("utf-8")).hexdigest() != row["text_sha256"]: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_DIGEST", name) + if name not in text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_NAME", name) + names.append(name) + if entry["delivered_header_width"] != len(names): + raise _refuse("PROJECTION_LAYOUT_WIDTH", label) + if len(set(names)) != len(names): + raise _refuse("PROJECTION_LAYOUT_DUPLICATE", label) + return tuple(names) + + +def _projected_column( + entry: Mapping[str, object], layout: Mapping[str, Mapping[str, object]] +) -> ProjectedColumn: + field = entry["field"] + if not isinstance(field, str) or field not in layout: + raise _refuse("PROJECTION_UNDELIVERED_FIELD", field) + for key in _FORBIDDEN_COLUMN_KEYS: + if key in entry: + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + column = layout[field] + if column["admission"] != "monetary_source_projection": + raise _refuse("PROJECTION_ADMISSION_DISAGREEMENT", field) + if entry["delivered_index"] != column["delivered_index"]: + raise _refuse("PROJECTION_INDEX_DISAGREEMENT", field) + if entry["publisher_position"] != column["publisher_position"]: + raise _refuse("PROJECTION_POSITION_DISAGREEMENT", field) + if entry["publisher_label"] != column["publisher_label"]: + raise _refuse("PROJECTION_LABEL_DISAGREEMENT", field) + if entry["dtype"] not in _DECLARED_DTYPE: + raise _refuse("PROJECTION_DTYPE", field) + if entry["grammar"] != _GRAMMAR: + raise _refuse("PROJECTION_GRAMMAR", field) + width = _positive_int(entry["lexical_width"], f"{field}.lexical_width") + if width > LEXICAL_WIDTH_MAX: + raise _refuse("PROJECTION_LEXICAL_WIDTH", field) + units = entry["units"] + digits = _positive_int(units["field_width_digits"], f"{field}.field_width_digits") + if digits > _MAX_AMOUNT_DIGITS or width < digits + 1: + raise _refuse("PROJECTION_WIDTH_BELOW_GRAMMAR", field) + if units["currency"] != "USD" or units["scale"] != "whole_dollars": + raise _refuse("PROJECTION_UNITS", field) + if units["implied_decimals"] != 0: + raise _refuse("PROJECTION_IMPLIED_DECIMALS", field) + sign = entry["sign"] + marker = sign["publisher_signed_marker"] + if not isinstance(marker, bool) or marker != column["publisher_signed_marker"]: + raise _refuse("PROJECTION_SIGN_MARKER", field) + negative = sign["negative_values"] + if negative not in _NEGATIVE_STATES: + raise _refuse("PROJECTION_SIGN_STATE", field) + if marker and negative != "documented_by_record_layout_marker": + raise _refuse("PROJECTION_SIGN_STATE", field) + if sign["negative_zero"] != "refused": + raise _refuse("PROJECTION_NEGATIVE_ZERO_POLICY", field) + missing = entry["missingness"] + if missing["empty_token"] != "refused": + raise _refuse("PROJECTION_EMPTY_TOKEN_POLICY", field) + if missing["zero_meaning"] != "delivered_zero_in_edited_public_file": + raise _refuse("PROJECTION_ZERO_POLICY", field) + if entry["return_grain"]["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN", field) + if entry["source_period_status"]["status"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS", field) + if entry["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE", field) + if entry["derived_combinations"] != "refused": + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + admission = entry["fit_admission"] + if admission not in _FIT_ADMISSIONS: + raise _refuse("PROJECTION_FIT_ADMISSION", field) + unresolved = tuple(item["code"] for item in entry["unresolved"]) + return ProjectedColumn( + field=field, + concept=entry["concept"], + concept_group=entry["concept_group"], + delivered_index=entry["delivered_index"], + publisher_position=entry["publisher_position"], + publisher_label=entry["publisher_label"], + dtype=entry["dtype"], + lexical_width=width, + grammar=entry["grammar"], + max_digits=digits, + publisher_signed_marker=marker, + negative_values=negative, + fit_admission=admission, + unresolved=unresolved, + ) + + +def _projection_from_document( + document: Mapping[str, object], + *, + route: str, + definition: PufRawSourceDefinition, +) -> PufMonetaryProjection: + """Validate a projection document against a raw source definition.""" + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("PROJECTION_DEFINITION_TYPE") + if document.get("schema") != _PROJECTION_SCHEMA: + raise _refuse("PROJECTION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("PROJECTION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("PROJECTION_ROUTE") + + raw = document["raw_source_definition"] + raw_sha = _hex64(raw["sha256"], "raw_source_definition.sha256") + if raw_sha != definition.sha256: + raise _refuse("PROJECTION_RAW_DEFINITION_MISMATCH") + + layout = document["layout"] + reasons = layout["not_selected_reasons"] + if not isinstance(reasons, Mapping) or not reasons: + raise _refuse("PROJECTION_NOT_SELECTED_REASONS") + allowed = frozenset(reasons) + main_names = _layout_names(layout["main"], "main", allowed) + demographic_names = _layout_names(layout["demographic"], "demographic", allowed) + if main_names != definition.main.delivered_header: + raise _refuse("PROJECTION_MAIN_LAYOUT_HEADER") + if demographic_names != definition.demographic.delivered_header: + raise _refuse("PROJECTION_DEMOGRAPHIC_LAYOUT_HEADER") + + for name, entry in ( + ("main", definition.main), + ("demographic", definition.demographic), + ): + pin = document["sources"][name] + if ( + pin["sha256"] != entry.sha256 + or pin["git_blob_sha1"] != entry.git_blob_sha1 + or pin["bytes"] != entry.bytes + or pin["header_record_canonical_sha256"] + != entry.header_record_canonical_sha256 + or pin["delivered_header_width"] != len(entry.delivered_header) + or pin["data_records"] != entry.data_records + ): + raise _refuse("PROJECTION_SOURCE_PIN", name) + + by_name = {column["name"]: column for column in layout["main"]["columns"]} + for refused in document["refused_fields"]: + if refused in main_names or refused in demographic_names: + raise _refuse("PROJECTION_REFUSED_FIELD_DELIVERED", refused) + + projection = document["projection"] + if projection["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN") + if projection["period_semantics"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS") + if projection["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE") + if projection["amounts_decoded"] is not True: + raise _refuse("PROJECTION_AMOUNTS_DECODED") + + entries = projection["columns"] + if not isinstance(entries, (list, tuple)) or not entries: + raise _refuse("PROJECTION_EMPTY") + columns = tuple(_projected_column(entry, by_name) for entry in entries) + fields = [column.field for column in columns] + if len(set(fields)) != len(fields): + raise _refuse("PROJECTION_DUPLICATE_FIELD") + for field in fields: + if field in document["refused_fields"]: + raise _refuse("PROJECTION_REFUSED_FIELD", field) + admitted = { + name + for name, column in by_name.items() + if column["admission"] == "monetary_source_projection" + } + if admitted != set(fields): + raise _refuse("PROJECTION_ADMISSION_SET") + if projection["column_count"] != len(columns): + raise _refuse("PROJECTION_COLUMN_COUNT") + if sorted(fields, key=lambda name: by_name[name]["delivered_index"]) != fields: + raise _refuse("PROJECTION_COLUMN_ORDER") + + if any(value is not False for value in document["non_claims"].values()): + raise _refuse("PROJECTION_NON_CLAIMS") + + canonical = canonical_json(_thawed(document)) + return PufMonetaryProjection( + route=route, + document=_frozen(json.loads(canonical.decode("utf-8"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + columns=columns, + raw_definition_sha256=raw_sha, + main_delivered_header=main_names, + demographic_delivered_header=demographic_names, + ) + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_PROJECTION_PACKAGE).joinpath(_PROJECTION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufMonetaryProjection | None = None + + +def packaged_projection() -> PufMonetaryProjection: + """The one closed projection document a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("utf-8")) + _PACKAGED = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + return _PACKAGED + + +def projection_document_json( + projection: PufMonetaryProjection | None = None, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a projection document.""" + + resolved = packaged_projection() if projection is None else projection + return json.loads(resolved.canonical.decode("utf-8")) + + +def fixture_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a projection on the explicit non-production test route. + + A fixture projection must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. It still has to authenticate the whole delivered layout, because a + fixture that dropped the layout would be testing a different contract. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return projection + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMonetarySourceProjection: + """Decoded source values with explicit, immutable fitting refusals.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + column_metadata: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + @property + def amount_known(self) -> np.ndarray: + """Whether a row is inside the individual-return amount universe. + + Derived from the authenticated disclosure flag on every access and + handed back read-only, so there is no stored mask to tamper with and + nothing a caller holds can contradict the flags. A false entry means + *outside the universe*: the typed slot holds the sentinel and the + delivered token is the authority. + """ + + return _amount_known(self.typed["disclosure_aggregate"]) + + def aggregate_lexical_rows( + self, field: str | None = None + ) -> Mapping[str, tuple[tuple[int, str], ...]]: + """Exact ``(RECID, token)`` pairs for the disclosure-aggregate rows. + + A local accessor, for diagnosis at the console or in a test. Public + summaries, receipts and reports stay aggregate-only: nothing in the + kernels calls this. No token is parsed, totalled, ordered by value or + compared with another token. + """ + + if field is None: + fields = tuple(self.lexical) + elif field in self.lexical: + fields = (field,) + else: + raise _refuse("AGGREGATE_LEXICAL_FIELD", field) + aggregate = np.flatnonzero(~self.amount_known).tolist() + recids = self.typed["RECID"] + return MappingProxyType( + { + name: tuple( + (int(recids[position]), self.lexical[name][position]) + for position in aggregate + ) + for name in fields + } + ) + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one.""" + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _canonical_header_digest(header: Sequence[str]) -> str: + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _key_token(value: str, *, width: int) -> str: + """The unsigned decimal ``RECID`` token, as the raw slice reads it.""" + + if not isinstance(value, str): + raise _refuse("KEY_TOKEN_TYPE") + if not value.isascii(): + raise _refuse("KEY_TOKEN_NON_ASCII") + if "\x00" in value: + raise _refuse("KEY_TOKEN_EMBEDDED_NUL") + if not value: + raise _refuse("KEY_TOKEN_EMPTY") + if len(value) > width: + raise _refuse("KEY_TOKEN_OVER_WIDTH") + if any(character not in _DIGITS for character in value): + raise _refuse("KEY_TOKEN_GRAMMAR") + number = int(value) + if number > _INT64_MAX: + raise _refuse("KEY_TOKEN_INT64_OVERFLOW") + return value + + +def _amount(value: str, column: ProjectedColumn) -> int: + """Parse one delivered amount token, or refuse. + + The grammar is exactly what the retained booklets support: an optional + leading minus, then decimal digits, no wider than the publisher's declared + amount-field width. There is no missing-value sentinel anywhere in either + booklet, so an empty token is a refusal to be preserved and reviewed + rather than a zero to be invented, and a plus sign, a decimal point, + surrounding whitespace or an exponent is a refusal rather than a silently + widened grammar. ``-0`` is refused outright: a signed zero would compare + equal to zero while spelling differently. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AMOUNT_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AMOUNT_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AMOUNT_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AMOUNT_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AMOUNT_TOKEN_OVER_WIDTH", label) + digits = value[1:] if value.startswith("-") else value + if not digits: + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if any(character not in _DIGITS for character in digits): + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AMOUNT_TOKEN_OVER_DIGITS", label) + number = -int(digits) if value.startswith("-") else int(digits) + if number == 0 and value.startswith("-"): + raise _refuse("AMOUNT_TOKEN_NEGATIVE_ZERO", label) + if not _INT64_MIN <= number <= _INT64_MAX: + raise _refuse("AMOUNT_TOKEN_INT64_OVERFLOW", label) + return number + + +def _aggregate_token(value: str, column: ProjectedColumn) -> str: + """Validate one disclosure-aggregate token, and return it unchanged. + + This is a **separate, explicitly bounded operational grammar** for the + delivered aggregate rows of this pinned file. It is not a newly discovered + publisher specification, and it is never applied to an individual return. + Optional leading minus, one or more digits, at most one point followed by + one or more digits: no plus sign, exponent, whitespace, embedded NUL, + empty token or negative zero in either spelling, at most the column's + declared digit bound and at most its declared width. + + The token is returned exactly as delivered. Nothing here converts it to a + number, rounds it, or compares it against another token. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AGGREGATE_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AGGREGATE_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AGGREGATE_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AGGREGATE_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AGGREGATE_TOKEN_OVER_WIDTH", label) + negative = value.startswith("-") + body = value[1:] if negative else value + if not body: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if body.count(".") > 1: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + whole, point, fraction = body.partition(".") + if point and (not whole or not fraction): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + digits = whole + fraction + if not digits or any(character not in _DIGITS for character in digits): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AGGREGATE_TOKEN_OVER_DIGITS", label) + if negative and set(digits) == {"0"}: + raise _refuse("AGGREGATE_TOKEN_NEGATIVE_ZERO", label) + return value + + +def _amount_known(flags: np.ndarray) -> np.ndarray: + """The individual-return amount universe, derived from the flag alone. + + Returned read-only and recomputed at every call, so no caller can hold a + mask that contradicts the disclosure flags it came from. + """ + + known = np.asarray(flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"]) == 0 + # A read-only flag on an owning ndarray can be reversed with setflags. + # An immutable bytes backing store makes that reversal impossible. + return np.frombuffer(known.tobytes(), dtype=np.bool_) + + +def _typed_column( + tokens: Sequence[str], known: np.ndarray, column: ProjectedColumn +) -> np.ndarray: + """Parse by universe: an amount where known, the sentinel where not.""" + + if len(tokens) != int(known.shape[0]): + raise _refuse("COLUMN_LEXICAL_ROWS", column.field) + values: list[int] = [] + for position, token in enumerate(tokens): + if bool(known[position]): + values.append(_amount(token, column)) + else: + _aggregate_token(token, column) + values.append(AGGREGATE_SENTINEL) + return np.asarray(values, dtype=column.numpy_dtype) + + +def _check_header( + reader, projection: PufMonetaryProjection, definition: PufRawSourceDefinition +) -> list[str]: + pin = definition.main + profile = definition.document["csv_profile"] + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING") from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE") from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT") + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH") + if tuple(header) != projection.main_delivered_header: + raise _refuse("HEADER_NOT_AUTHENTICATED_LAYOUT") + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST") + refused = set(projection.document["refused_fields"]) & set(header) + if refused: + raise _refuse("HEADER_REFUSED_FIELD", sorted(refused)) + return header + + +def _records(reader, definition: PufRawSourceDefinition): + profile = definition.document["csv_profile"] + width = len(definition.main.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE") from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP") + if len(record) != width: + raise _refuse("RECORD_WIDTH") + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP") + yield record + + +def decode_puf_monetary_source( + main_bytes: bytes, + status: object, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, +) -> PufMonetarySourceProjection: + """Decode the enumerated monetary columns, bound to the status artifact. + + ``status`` is the decoded ``puf_2015_raw_return_status``. Its ``RECID`` + sequence and aggregate flags must be element-wise identical to this + parse's, and its ``demographic_status`` is carried through rather than + re-joined, so the two artifacts cannot disagree about which returns + matched. Every delivered main record is projected: nothing is filtered, + and the four disclosure aggregates stay tagged rather than dropped. + + Amount tokens are collected before that alignment runs and parsed after + it, by universe: the unchanged integer grammar on individual returns, the + bounded aggregate lexical grammar plus the out-of-universe sentinel on + disclosure aggregates. A parse therefore cannot decide which grammar a row + answers to before the status artifact has proved which row it is. + """ + + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.raw_definition_sha256 != definition.sha256: + raise _refuse("DECODE_DEFINITION_MISMATCH") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + key_width = next( + field["lexical_width"] + for field in definition.document["fields"] + if field["name"] == "RECID" + ) + + reader = _reader(main_bytes, profile) + header = _check_header(reader, projection, definition) + positions = { + column.field: header.index(column.field) for column in projection.columns + } + for column in projection.columns: + if positions[column.field] != column.delivered_index: + raise _refuse("DELIVERED_INDEX_DISAGREEMENT", column.field) + recid_position = header.index("RECID") + + keys: list[int] = [] + seen: set[str] = set() + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + flags: list[int] = [] + tokens: dict[str, list[str]] = {column.field: [] for column in projection.columns} + + # Tokens are collected first and parsed afterwards. Which grammar a row's + # amount answers to is decided only once the status artifact has proved + # the RECID sequence and the aggregate flags, so "authenticate the + # alignment before interpreting a row's amount universe" is the literal + # order of operations rather than a claim about one. + for record in _records(reader, definition): + key = _key_token(record[recid_position], width=key_width) + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = int(key) + keys.append(recid) + flags.append(1 if recid in aggregates else 0) + for column in projection.columns: + tokens[column.field].append(record[positions[column.field]]) + rows = len(keys) + if rows == 0: + raise _refuse("MAIN_NO_RECORDS") + if rows != definition.main.data_records: + raise _refuse("MAIN_RECORD_COUNT") + + typed: dict[str, np.ndarray] = { + "RECID": np.asarray(keys, dtype=_STRUCTURAL_DTYPES["RECID"]), + "disclosure_aggregate": np.asarray( + flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"] + ), + } + if len(np.unique(typed["RECID"])) != rows: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + typed["demographic_status"] = _status_alignment(status, typed, rows) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + typed[column.field] = _typed_column(tokens[column.field], known, column) + + lexical = {name: tuple(items) for name, items in tokens.items()} + facts = _facts_from_arrays(typed, lexical, projection) + # Match envelope readback's immutable byte-backed arrays. In particular, + # disclosure flags cannot change after an amount-known mask is returned. + typed = { + name: np.frombuffer(array.tobytes(), dtype=array.dtype) + for name, array in typed.items() + } + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(facts), + column_metadata=_frozen(facts["column_metadata"]), + ) + + +def _status_alignment( + status: object, typed: Mapping[str, np.ndarray], rows: int +) -> np.ndarray: + """Bind this parse to the reviewed status artifact, element by element.""" + + try: + status_typed = status.typed + except AttributeError as error: + raise _refuse("STATUS_ARTIFACT_TYPE") from error + for name in ("RECID", "disclosure_aggregate", "demographic_status"): + if name not in status_typed: + raise _refuse("STATUS_ARTIFACT_COLUMN", name) + if int(status_typed["RECID"].shape[0]) != rows: + raise _refuse("STATUS_ROW_COUNT") + if not np.array_equal( + np.asarray(status_typed["RECID"], dtype=" dict[str, np.ndarray]: + """The three disjoint row classes, which must partition the rows exactly.""" + + aggregate = typed["disclosure_aggregate"] == 1 + status = typed["demographic_status"] == 1 + masks = { + "disclosure_aggregate": aggregate, + "demographic_matched": (~aggregate) & status, + "demographic_unmatched": (~aggregate) & (~status), + } + total = sum(int(np.count_nonzero(mask)) for mask in masks.values()) + if total != int(typed["RECID"].shape[0]): + raise _refuse("ROW_CLASS_PARTITION") + return masks + + +def _column_facts(values: np.ndarray, mask: np.ndarray) -> dict[str, object]: + selected = values[mask] + count = int(selected.shape[0]) + if count == 0: + return { + "rows": 0, + "zero": 0, + "negative": 0, + "positive": 0, + "minimum": None, + "maximum": None, + "sum": 0, + } + total = int(sum(int(item) for item in selected.tolist())) + if not _INT64_MIN <= total <= _INT64_MAX: + raise _refuse("COLUMN_SUM_OVERFLOW") + return { + "rows": count, + "zero": int(np.count_nonzero(selected == 0)), + "negative": int(np.count_nonzero(selected < 0)), + "positive": int(np.count_nonzero(selected > 0)), + "minimum": int(selected.min()), + "maximum": int(selected.max()), + "sum": total, + } + + +def _aggregate_column_facts( + tokens: Sequence[str], mask: np.ndarray +) -> dict[str, object]: + """Token classes for the aggregate rows, and nothing that reads as money. + + Counts of token shapes and one token width. There is no zero, sign census, + minimum, maximum or sum key here, because the delivered token is the + authority for these rows and this module never interprets it. A diagnostic + may count token classes; it may not state an aggregate amount. + """ + + selected = [ + token for token, taken in zip(tokens, mask.tolist(), strict=True) if taken + ] + if not selected: + return { + "rows": 0, + "integer_tokens": 0, + "fractional_tokens": 0, + "negative_tokens": 0, + "max_token_width": None, + } + return { + "rows": len(selected), + "integer_tokens": sum(1 for token in selected if "." not in token), + "fractional_tokens": sum(1 for token in selected if "." in token), + "negative_tokens": sum(1 for token in selected if token.startswith("-")), + "max_token_width": max(len(token) for token in selected), + } + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, Sequence[str]], + projection: PufMonetaryProjection, +) -> dict[str, object]: + """Derive every reported fact from the arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count, a sum or a sign census the + arrays do not carry. Layout counts and column semantics come from the + authenticated projection document, rather than being inferred from rows. + The lexical arrays are an input because the aggregate class is described + from its delivered tokens and from nothing else. + """ + + rows = int(typed["RECID"].shape[0]) + masks = _row_masks(typed) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + if len(lexical[column.field]) != rows: + raise _refuse("FACTS_LEXICAL_ROWS", column.field) + return { + "rows": rows, + "main_records": rows, + "projected_columns": len(projection.columns), + "layout_columns_authenticated": len(projection.main_delivered_header), + "demographic_layout_columns": len(projection.demographic_delivered_header), + "aggregate_records": int(np.count_nonzero(masks["disclosure_aggregate"])), + "demographic_matched_rows": int( + np.count_nonzero(typed["demographic_status"] == 1) + ), + "demographic_unmatched_rows": int( + np.count_nonzero(typed["demographic_status"] == 0) + ), + "row_class_rows": { + name: int(np.count_nonzero(masks[name])) for name in ROW_CLASSES + }, + "column_facts": { + column.field: { + AGGREGATE_ROW_CLASS: _aggregate_column_facts( + lexical[column.field], masks[AGGREGATE_ROW_CLASS] + ), + **{ + name: _column_facts(typed[column.field], masks[name]) + for name in RETURN_ROW_CLASSES + }, + } + for column in projection.columns + }, + "column_metadata": { + column.field: { + "concept": column.concept, + "grain": _GRAIN, + # The packaged projection document's integer, whole-dollar, + # twelve-digit semantics describe individual returns. That + # document's bytes are unchanged; this metadata says who they + # apply to, and what happens to the rows they do not. + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_universe": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_token_max_digits": column.max_digits, + "aggregate_token_max_width": column.lexical_width, + "aggregate_token_grammar_authority": _AGGREGATE_GRAMMAR_AUTHORITY, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "period_semantics": _PERIOD_SEMANTICS, + "fit_admission": column.fit_admission, + "unresolved": list(column.unresolved), + "negative_values": column.negative_values, + "zero_meaning": "delivered_zero_in_edited_public_file", + "target_year_use": _TARGET_YEAR_USE, + "projection_sha256": projection.sha256, + } + for column in projection.columns + }, + "grain": _GRAIN, + "period_semantics": _PERIOD_SEMANTICS, + "amounts_decoded": True, + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "amount_known_rows": int(np.count_nonzero(known)), + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "target_year_use": _TARGET_YEAR_USE, + } + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def payload_bound( + rows: int, projection: PufMonetaryProjection +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact allocates, and its per-column parts.""" + + sizes = { + name: rows * np.dtype(_STRUCTURAL_DTYPES[name]).itemsize + for name in _STRUCTURAL_COLUMNS + } + for column in projection.columns: + sizes[column.field] = rows * np.dtype(column.numpy_dtype).itemsize + sizes[f"{column.field}_lexical"] = rows * column.lexical_width + return sum(sizes.values()), sizes + + +def _envelope_column_order(projection: PufMonetaryProjection) -> list[str]: + return [ + *_STRUCTURAL_COLUMNS, + *(column.field for column in projection.columns), + *(f"{column.field}_lexical" for column in projection.columns), + ] + + +def encode_monetary_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode the projection into the bounded, self-describing envelope.""" + + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": MONETARY_PROJECTION_TYPE.schema_version, + "type": [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (MONETARY_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_projection( + header: Mapping[str, object], + projection: PufMonetaryProjection | None, +) -> PufMonetaryProjection | None: + """The projection this payload may be checked against, or ``None``.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + return projection + if route == "packaged": + packaged = packaged_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if projection is None: + continue + pin = projection.document["sources"][name] + if ( + entry["sha256"] != pin["sha256"] + or entry["git_blob_sha1"] != pin["git_blob_sha1"] + or entry["bytes"] != pin["bytes"] + or entry["header_record_canonical_sha256"] + != pin["header_record_canonical_sha256"] + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], rows: int, projection: PufMonetaryProjection +) -> list[Mapping[str, object]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = _envelope_column_order(projection) + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + resolved: list[Mapping[str, object]] = [] + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + entry = _exact_keys(column, _ENVELOPE_LEXICAL_COLUMN_KEYS, name) + if entry["kind"] != _LEXICAL_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + width = _envelope_int(entry["width"], f"{name}.width", minimum=1) + if width > LEXICAL_WIDTH_MAX or width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + else: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # An exact dtype string closes width, signedness and endianness in + # one comparison, so `` BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + projection: PufMonetaryProjection, +) -> None: + """Re-prove the projection's own invariants from the arrays read back.""" + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufMonetarySourceProjection: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema; every reported fact is recomputed from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a negative census, a class partition or a column sum + it does not carry. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(MONETARY_PROJECTION_MAGIC): + # A superseded magic is named, not reinterpreted: those payloads + # declare a different amount universe, and silently reading one under + # this contract would report aggregates as individual amounts. + if any(payload.startswith(magic) for magic in SUPERSEDED_MAGIC): + raise _refuse("ENVELOPE_SUPERSEDED_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(MONETARY_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != MONETARY_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_projection(header, projection) + if resolved is None: + # A fixture-route payload has no reconstructible projection document, + # so there is nothing to close its column set against. That is a + # caller error here rather than a weaker structural pass: the + # projection is exactly what makes the columns meaningful. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=_frozen(recomputed["column_metadata"]), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", +) +#: The implementation identity moves with the contract: a node written +#: against the version-2 amount universe must not bind to this kernel. +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_source/implementation/3\n" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_projection" + + +class USPufMonetarySourceKernel(KernelBase): + """Read the pinned main delivery and produce the monetary projection.""" + + ref = "us.puf.monetary_source.projection@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = packaged_projection() if projection is None else projection + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if self.projection.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the projection must be the packaged file's own canonical bytes, + # and the document it carries must still be those exact bytes. + packaged = packaged_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions and the effective CSV profile.""" + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_monetary_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +class USPufMonetaryProjectionSummaryKernel(KernelBase): + """A downstream consumer: decode the projection and report its census.""" + + ref = "us.puf.monetary_source.projection_summary@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__(self, *, projection: PufMonetaryProjection | None = None) -> None: + self.projection = packaged_projection() if projection is None else projection + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != MONETARY_PROJECTION_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != MONETARY_PROJECTION_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + decoded = decode_monetary_projection(value.payload, self.projection) + facts = decoded.facts + # The per-return negative census covers the two return classes only. + # Summing it across all three would have counted a token class as a + # monetary sign, which is exactly the conflation this contract ends. + per_return_negatives = { + field: sum(int(classes[name]["negative"]) for name in RETURN_ROW_CLASSES) + for field, classes in facts["column_facts"].items() + } + # Aggregate-only: token-class counts and a width, never a token, a + # RECID or anything that could be read as an aggregate amount. The + # exact row/token pairs stay behind the local decoded accessor. + aggregate_token_classes = { + field: dict(classes[AGGREGATE_ROW_CLASS]) + for field, classes in facts["column_facts"].items() + } + return KernelResult( + receipt={ + "puf_monetary_projection_summary": { + "producer_key": value.producer_key, + "rows": decoded.rows, + "projected_columns": facts["projected_columns"], + "row_class_rows": dict(facts["row_class_rows"]), + "amount_known_rows": facts["amount_known_rows"], + "amount_known_rule": facts["amount_known_rule"], + "per_return_negative_values_by_field": per_return_negatives, + "aggregate_rows_are_lexical_only": True, + "aggregate_typed_sentinel": facts["aggregate_typed_sentinel"], + "aggregate_token_grammar": facts["aggregate_token_grammar"], + "aggregate_lexical_authority": facts["aggregate_lexical_authority"], + "aggregate_numeric_interpretation": facts[ + "aggregate_numeric_interpretation" + ], + "aggregate_lexical_token_classes": aggregate_token_classes, + "period_semantics": facts["period_semantics"], + "amount_units": facts["amount_units"], + "amount_units_scope": facts["amount_units_scope"], + "target_year_use": facts["target_year_use"], + "column_metadata": _thawed(decoded.column_metadata), + } + } + ) + + +def us_puf_monetary_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + return Node( + f"{stage}.projection", + USPufMonetarySourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),), + description=( + "Decode the enumerated 2015 PUF monetary source columns at return " + "grain, bound to the reviewed status artifact. Owns no population " + "columns and admits no target year." + ), + ) + + +def us_puf_monetary_projection_summary_node( + *, + population: str, + producer: str = MONETARY_PROJECTION_NODE, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare an ordinary consumer of the projection artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.projection_summary", + USPufMonetaryProjectionSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + _PROJECTION_ALIAS, + producer, + _PROJECTION_ALIAS, + MONETARY_PROJECTION_TYPE, + ), + ), + description="Read the monetary projection artifact and report its census.", + ) + + +def register_us_puf_monetary_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufMonetarySourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + registry.register(USPufMonetaryProjectionSummaryKernel(projection=projection)) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py new file mode 100644 index 000000000..ba565c369 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py @@ -0,0 +1,1214 @@ +"""The developmental CPI-U price-restatement baseline for the 2015 PUF. + +One exact positive ratio of two published CPI-U all-items annual averages — +313689/237017, 2015 to 2024 — applied to every known money field of the +thirteen-field source projection. It restates 2015-file purchasing power at +2024 prices. It is **not** income aging: it carries no category-specific +nominal growth, no return-mass transition, and no claim that any field's real +level changed. ``S006`` is declared with an explicit identity series so the +no-mass-transition choice is recorded rather than implied. + +Three things this module is, and three it refuses +------------------------------------------------- +*It pins the resource.* The series is ``CUUR0000SA0``, U.S. city average, all +items, not seasonally adjusted, ``M13`` annual average, base 1982-84=100. The +levels are the publisher's own decimal text, copied verbatim, and the factor +table it builds carries the ``developmental_public_resource`` authority: the +retained bytes are checked against their named digest and their two records +against the declared level text. Retrieval location and time remain caller +metadata, not claims proved by hashing. The authority cannot become +release-eligible. + +*It declares the recipe.* One ``nominal_price_restatement`` series for all +thirteen money fields, one source reference year, one target year, ``RECID`` +as the identifier, ``S006`` as a design weight whose declared factor is +exactly one. + +*It adapts the artifact to the money view.* The projection is an artifact and +the growth transform consumes a table, so +:func:`adapt_projection_to_money_view` joins the projection to the reviewed +status artifact on ``RECID``, removes only the disclosure-aggregate rows, +records what it removed, and hands back ``FLPDYR``, the demographic status and +the unaltered integer ``S006`` separately. The growth itself is +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`; nothing here +multiplies an amount. + +It refuses to interpret an aggregate token, to put the out-of-universe +sentinel anywhere near a numeric input, and to declare that this baseline +under- or over-states any 2024 amount. The public universes that would support +such a statement are not matched to this file. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import struct +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from .puf_growth import ( + DESIGN_WEIGHT_FIELD, + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + SOURCE_AGI_FIELD, + SOURCE_RECID_FIELD, + CompiledPufGrowth, + FactorAuthority, + FieldRole, + GrownPufTable, + GrowthFactorTable, + GrowthKind, + GrowthRecipe, + GrowthRule, + PufGrowthRefusalError, + SignBranch, + compile_puf_growth, +) +from .puf_monetary_agi_projection import AGI_FIELD +from .puf_monetary_source import AGGREGATE_SENTINEL, PufMonetarySourceProjection +from .puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "CPI_U_ANNUAL_AVERAGE_PERIOD", + "CPI_U_INDEX_BASE", + "CPI_U_SERIES_ID", + "CPI_U_SERIES_TITLE", + "DESIGN_WEIGHT_IDENTITY_SERIES", + "PRICE_BASELINE_LEVELS", + "PRICE_BASELINE_RATIO", + "PRICE_BASELINE_RECIPE_ID", + "PRICE_BASELINE_SERIES_NAME", + "PRICE_BASELINE_SOURCE_YEAR", + "PRICE_BASELINE_TABLE_ID", + "PRICE_BASELINE_TARGET_YEAR", + "PRICE_BASELINE_FACTOR_BITS", + "PAIRWISE_OBSERVATIONS", + "PufMoneyViewAdaptation", + "adapt_projection_to_money_view", + "compile_price_baseline", + "price_baseline_factor_document", + "price_baseline_invariants", + "price_baseline_recipe", + "require_price_baseline_contract", + "verify_cpi_resource_bytes", +] + +#: The primary BLS series this baseline is read from. +CPI_U_SERIES_ID = "CUUR0000SA0" +CPI_U_SERIES_TITLE = ( + "All items in U.S. city average, all urban consumers, not seasonally adjusted" +) +CPI_U_ANNUAL_AVERAGE_PERIOD = "M13" +CPI_U_INDEX_BASE = "1982-84=100" + +#: The source file's representative year, and the target the restatement +#: states amounts in. The source year is the growth slice's own PUF source +#: year, not a second opinion about it. +PRICE_BASELINE_SOURCE_YEAR = PUF_SOURCE_YEAR +PRICE_BASELINE_TARGET_YEAR = 2024 + +#: The publisher's own decimal text for the two annual averages. These are +#: level strings, never floats: the factor table stores text and the ratio is +#: computed once, in one place, by the growth contract. +PRICE_BASELINE_LEVELS = MappingProxyType( + {PRICE_BASELINE_SOURCE_YEAR: "237.017", PRICE_BASELINE_TARGET_YEAR: "313.689"} +) + +#: The exact rational factor, as ``(numerator, denominator)``. Both published +#: levels carry exactly three decimals, so the ratio of the thousandths-scaled +#: integers is the exact ratio of the published levels. +PRICE_BASELINE_RATIO = (313689, 237017) + +#: The one float64 the adopted factor is, as its little-endian bit pattern. +#: Derived from the exact rational here rather than written down, so it cannot +#: drift from :data:`PRICE_BASELINE_RATIO`; a test separately checks that the +#: division of the two published level *texts* lands on the same double. +#: :func:`require_price_baseline_contract` compares an incoming contract's +#: resolved bits against this, which is the check that makes every downstream +#: statement about "the factor" a statement about the contract in hand. +PRICE_BASELINE_FACTOR_BITS = struct.pack( + " None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _module_sha256() -> str: + """This module's own bytes, so the factor table names the code that built it.""" + return hashlib.sha256(Path(__file__).resolve().read_bytes()).hexdigest() + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _cpi_annual_average_records( + resource_bytes: bytes, years: Sequence[int] +) -> dict[int, bytes]: + """The CPI-U all-items ``M13`` level text for each of *years*, or fewer. + + BLS pads ``series_id`` and ``value`` with spaces for monospace column + alignment; the character that actually delimits a field is the tab, so + every field is stripped before it is compared or returned — the same + convention this lane's own retrieval script used to extract and hash + these two lines. A row whose first field is this series but which does + not split into the publisher's five tab-delimited fields + (``series_id``, ``year``, ``period``, ``value``, ``footnote_codes``) is + refused rather than silently skipped, because a truncated or reshuffled + row is exactly the failure mode this check exists to catch. Rows for a + year not in *years* are ignored: this baseline does not require the file + to be otherwise well-formed, only that its own two records are. + """ + series = CPI_U_SERIES_ID.encode("ascii") + period = CPI_U_ANNUAL_AVERAGE_PERIOD.encode("ascii") + wanted = {str(year).encode("ascii") for year in years} + found: dict[int, bytes] = {} + for line in resource_bytes.split(b"\n"): + if not line.strip(): + continue + fields = line.split(b"\t") + if fields[0].strip() != series: + continue + _require(len(fields) == 5, "PRICE_BASELINE_RESOURCE_LINE_SHAPE") + _, year_field, period_field, value_field, _ = ( + field.strip() for field in fields + ) + if period_field != period or year_field not in wanted: + continue + year = int(year_field) + _require( + year not in found, "PRICE_BASELINE_RESOURCE_DUPLICATE_RECORD", str(year) + ) + found[year] = value_field + return found + + +def verify_cpi_resource_bytes( + resource_bytes: bytes, *, resource_sha256: str +) -> Mapping[str, object]: + """Refuse unless *resource_bytes* is this baseline's named public resource. + + The whole-file digest is bound first — the caller's ``resource_sha256`` + is no longer a string nobody checks. Then the bytes themselves are parsed + with the publisher's own tab-delimited, space-padded convention (see + :func:`_cpi_annual_average_records`) and must contain **exactly** the two + ``CUUR0000SA0`` ``M13`` records this baseline reads, at the two years + :data:`PRICE_BASELINE_LEVELS` names, each with exactly + the declared decimal text after removing BLS column padding. Different + numeric spellings and near-equal floats are refused: source lexical + identity is stronger than arithmetic agreement. + + Args: + resource_bytes: The exact bytes retrieved from ``source_url``. + resource_sha256: The digest the caller claims for those bytes. + + Returns: + A receipt naming the series, period, verified digest and the two + matched records. + + Raises: + PufGrowthRefusalError: On a type, size, digest, shape, duplicate, + missing-record or value mismatch. + """ + _require( + isinstance(resource_bytes, (bytes, bytearray)), "PRICE_BASELINE_RESOURCE_TYPE" + ) + _require(len(resource_bytes) > 0, "PRICE_BASELINE_RESOURCE_EMPTY") + _require( + len(resource_bytes) <= _CPI_RESOURCE_MAX_BYTES, + "PRICE_BASELINE_RESOURCE_OVERSIZE", + ) + body = bytes(resource_bytes) + actual_sha256 = hashlib.sha256(body).hexdigest() + _require(actual_sha256 == resource_sha256, "PRICE_BASELINE_RESOURCE_DIGEST") + + years = tuple(PRICE_BASELINE_LEVELS) + found = _cpi_annual_average_records(body, years) + records: dict[str, object] = {} + for year in years: + _require(year in found, "PRICE_BASELINE_RESOURCE_RECORD_MISSING", str(year)) + _require( + found[year] == PRICE_BASELINE_LEVELS[year].encode("ascii"), + "PRICE_BASELINE_RESOURCE_RECORD_VALUE", + str(year), + ) + # Decode only after exact ASCII equality; malformed values must refuse + # by the same domain reason rather than leaking a parser exception. + level_text = found[year].decode("ascii") + found_value = float(level_text) + records[str(year)] = MappingProxyType( + {"level_text": level_text, "level_float64": found_value} + ) + return MappingProxyType( + { + "series_id": CPI_U_SERIES_ID, + "period": CPI_U_ANNUAL_AVERAGE_PERIOD, + "resource_sha256": actual_sha256, + "resource_bytes": len(body), + "records": MappingProxyType(records), + } + ) + + +def price_baseline_factor_document( + *, source_url: str, resource_sha256: str, retrieved_utc: str, resource_bytes: bytes +) -> bytes: + """The developmental factor table, as the exact bytes it hashes to. + + Args: + source_url: The publisher endpoint the levels were read from. + resource_sha256: Digest of the exact resource bytes that were read. + retrieved_utc: When they were read, as an ISO 8601 UTC instant. + resource_bytes: The exact bytes retrieved from ``source_url``. Checked + against ``resource_sha256`` and parsed for the two annual-average + records this baseline names — see + :func:`verify_cpi_resource_bytes`. Without this the two prior + arguments were caller assertions nothing in this module verified. + + Returns: + A canonical factor-table document carrying the price series and the + identity design-weight series, with the + ``developmental_public_resource`` authority. + """ + verify_cpi_resource_bytes(resource_bytes, resource_sha256=resource_sha256) + return _canonical( + { + "authority": "developmental_public_resource", + "provenance": { + "dependency_versions": {}, + "generated_by": ( + "microcosm.build.us_runtime.puf_price_baseline." + "price_baseline_factor_document" + ), + "generator_code_sha256": _module_sha256(), + "index_or_total": "index", + "parameter_paths": [], + "population_divisor": "none", + "resource_sha256": resource_sha256, + "retrieved_utc": retrieved_utc, + "rounding": ( + "none; the two published levels are copied verbatim as " + "decimal text and the factor is one float64 division of them" + ), + "source_url": source_url, + }, + "schema_version": 1, + "series": { + DESIGN_WEIGHT_IDENTITY_SERIES: { + "kind": str(GrowthKind.DESIGN_WEIGHT_GROWTH), + "levels": { + str(PRICE_BASELINE_SOURCE_YEAR): "1", + str(PRICE_BASELINE_TARGET_YEAR): "1", + }, + }, + PRICE_BASELINE_SERIES_NAME: { + "kind": str(GrowthKind.NOMINAL_PRICE_RESTATEMENT), + "levels": { + str(year): level + for year, level in PRICE_BASELINE_LEVELS.items() + }, + }, + }, + "table_id": PRICE_BASELINE_TABLE_ID, + "years": [PRICE_BASELINE_SOURCE_YEAR, PRICE_BASELINE_TARGET_YEAR], + } + ) + + +def price_baseline_recipe( + money_fields: Sequence[str], *, recipe_id: str = PRICE_BASELINE_RECIPE_ID +) -> GrowthRecipe: + """One price series for every money field, plus the two structural rules. + + Args: + money_fields: The projection's money fields. Must contain + ``E00100``, because the growth contract requires it, and must not + contain ``RECID`` or ``S006``, which are not money. + recipe_id: Recipe identity, for a caller running more than one. + + Raises: + PufGrowthRefusalError: If the field set is not a money field set. + """ + _require(bool(money_fields), "PRICE_BASELINE_NO_MONEY_FIELD") + fields = tuple(money_fields) + _require(len(set(fields)) == len(fields), "PRICE_BASELINE_DUPLICATE_FIELD") + _require(SOURCE_AGI_FIELD in fields, "PRICE_BASELINE_AGI_FIELD", SOURCE_AGI_FIELD) + for reserved in (SOURCE_RECID_FIELD, DESIGN_WEIGHT_FIELD): + _require(reserved not in fields, "PRICE_BASELINE_NON_MONEY_FIELD", reserved) + rules = [ + GrowthRule( + field, + FieldRole.MONEY, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.NOMINAL_PRICE_RESTATEMENT, + SignBranch.ANY, + PRICE_BASELINE_SERIES_NAME, + _CITATION, + ) + for field in fields + ] + rules.append( + GrowthRule( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_SOURCE_YEAR, + GrowthKind.NONE, + ) + ) + rules.append( + GrowthRule( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.DESIGN_WEIGHT_GROWTH, + SignBranch.ANY, + DESIGN_WEIGHT_IDENTITY_SERIES, + "declared identity: this baseline transitions no return mass", + ) + ) + return GrowthRecipe(recipe_id, tuple(rules)) + + +def require_price_baseline_contract( + compiled: CompiledPufGrowth, *, resource_bytes: bytes +) -> None: + """Refuse any contract that is not the adopted price baseline. + + :func:`adapt_projection_to_money_view` and + :func:`price_baseline_invariants` both call this before they record + anything, because both go on to state the source year, the exact rational + factor, the ``RECID`` identity and "this baseline transitions no return + mass" as facts about the run. A contract that named a different pair of + years, mixed a second factor series, aged rather than restated, split a + field across sign branches, or grew the design weight would leave every + other check in this module passing while those recorded statements were + false. The false receipt is the failure this refuses. + + What is checked is exactly what those receipts assert: + + * the ``developmental_public_resource`` authority, and no release + eligibility; + * every money rule at ``2015 -> 2024``, ``nominal_price_restatement``, + ``SignBranch.ANY``, on the one named CPI series, resolving to the + published level texts and to the exact float64 of ``313689/237017``; + * exactly one identifier rule, on ``RECID``, at the baseline source year; + * exactly one design-weight rule, on ``S006``, at the same two years, on + the declared identity series, whose resolved factor is exactly ``1.0`` + and whose two level texts are the same text — not two texts whose + float64 quotient happens to round to one; + * no rule in any other role; + * the exact resource bytes against the digest the contract names, and the + two source records against their declared series, years, period and text. + + The *field set* is deliberately not fixed: a contract over a subset of the + projection's money fields is still this baseline, and the adapter checks + the subset relation against the artifact separately. + + Args: + compiled: The contract a caller is about to run. + resource_bytes: The retained BLS body; checked against the contract's + own resource digest even if it came from the generic growth compiler. + + Raises: + PufGrowthRefusalError: With a ``PRICE_BASELINE_*`` reason naming the + first field that does not match. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _require( + compiled.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE, + "PRICE_BASELINE_AUTHORITY", + ) + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + verify_cpi_resource_bytes( + resource_bytes, + resource_sha256=compiled.factors.provenance["resource_sha256"], + ) + + rules = compiled.recipe.rules + money = tuple(rule for rule in rules if rule.role is FieldRole.MONEY) + identifiers = tuple(rule for rule in rules if rule.role is FieldRole.IDENTIFIER) + weights = tuple(rule for rule in rules if rule.role is FieldRole.DESIGN_WEIGHT) + for rule in rules: + _require( + rule.role + in (FieldRole.MONEY, FieldRole.IDENTIFIER, FieldRole.DESIGN_WEIGHT), + "PRICE_BASELINE_ROLE", + rule.field, + ) + _require(bool(money), "PRICE_BASELINE_NO_MONEY_FIELD") + _require( + SOURCE_AGI_FIELD in {rule.field for rule in money}, + "PRICE_BASELINE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + source_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_SOURCE_YEAR] + target_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_TARGET_YEAR] + for rule in money: + field = rule.field + _require( + rule.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_SOURCE_YEAR", + field, + ) + _require( + rule.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_TARGET_YEAR", + field, + ) + _require( + rule.growth_kind is GrowthKind.NOMINAL_PRICE_RESTATEMENT, + "PRICE_BASELINE_GROWTH_KIND", + field, + ) + # ``SignBranch.ANY`` is the field's only rule, which is what lets one + # positive factor be checked per row rather than per branch. + _require( + rule.sign_branch is SignBranch.ANY, "PRICE_BASELINE_SIGN_BRANCH", field + ) + _require( + rule.factor_series == PRICE_BASELINE_SERIES_NAME, + "PRICE_BASELINE_SERIES", + field, + ) + resolved = compiled.factor_for(field, SignBranch.ANY) + # The substantive claim first — this is the one float64 — and then the + # two level texts it must have come from. The order matters because + # more than one pair of texts divides to the same double, and this + # baseline is the published pair, not merely a pair that agrees. + _require( + resolved.ratio_bits == PRICE_BASELINE_FACTOR_BITS, + "PRICE_BASELINE_FACTOR", + field, + ) + _require( + resolved.source_level == source_level, "PRICE_BASELINE_SOURCE_LEVEL", field + ) + _require( + resolved.target_level == target_level, "PRICE_BASELINE_TARGET_LEVEL", field + ) + + # ``GrowthRecipe`` already requires exactly one identifier field, and + # ``GrowthRule`` already fixes a non-money, non-weight role to + # ``GrowthKind.NONE`` with equal years. What is open here is which field + # carries the role and which year it names. + _require( + len(identifiers) == 1 and identifiers[0].field == SOURCE_RECID_FIELD, + "PRICE_BASELINE_IDENTIFIER", + ) + identifier = identifiers[0] + _require( + identifier.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_IDENTIFIER_YEAR", + identifier.field, + ) + + # Likewise the design-weight role already fixes the kind and the sign + # branch. What is open is the field, the series, and whether the declared + # factor really is one. + _require( + len(weights) == 1 and weights[0].field == DESIGN_WEIGHT_FIELD, + "PRICE_BASELINE_DESIGN_WEIGHT", + ) + weight = weights[0] + _require( + weight.factor_series == DESIGN_WEIGHT_IDENTITY_SERIES, + "PRICE_BASELINE_DESIGN_WEIGHT_SERIES", + weight.field, + ) + _require( + weight.source_reference_year == PRICE_BASELINE_SOURCE_YEAR + and weight.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_DESIGN_WEIGHT_YEAR", + weight.field, + ) + resolved_weight = compiled.factor_for(weight.field, SignBranch.ANY) + # The substantive claim first: the declared weight factor is exactly one. + # Then the two level texts, which must be the same text and not merely two + # texts whose float64 quotient rounds to one. + _require( + resolved_weight.value == 1.0, + "PRICE_BASELINE_DESIGN_WEIGHT_FACTOR", + weight.field, + ) + _require( + resolved_weight.source_level == resolved_weight.target_level, + "PRICE_BASELINE_DESIGN_WEIGHT_LEVEL", + weight.field, + ) + + +def compile_price_baseline( + money_fields: Sequence[str], + *, + source_url: str, + resource_sha256: str, + retrieved_utc: str, + resource_bytes: bytes, + recipe_id: str = PRICE_BASELINE_RECIPE_ID, +) -> CompiledPufGrowth: + """Compile the price-baseline contract; never release-eligible. + + ``resource_bytes`` is required and is checked against ``resource_sha256`` + by :func:`price_baseline_factor_document`; this function cannot hand back + a contract pinned to resource metadata nobody verified. + """ + recipe = price_baseline_recipe(money_fields, recipe_id=recipe_id) + factors = GrowthFactorTable.from_bytes( + price_baseline_factor_document( + source_url=source_url, + resource_sha256=resource_sha256, + retrieved_utc=retrieved_utc, + resource_bytes=resource_bytes, + ) + ) + compiled = compile_puf_growth(recipe, factors) + # The authority gate already guarantees this. Asserting it here means the + # one function a caller reaches for cannot hand back an eligible contract + # even if the gate is later widened by mistake. + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + # The same gate the two entry points apply, so this function cannot hand + # back a contract they would decline. + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + return compiled + + +# -------------------------------------------------------------------------- +# The artifact-to-money-view adapter +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMoneyViewAdaptation: + """The money view the growth transform consumes, plus what stayed outside. + + Attributes: + money_view: One row per amount-known return, carrying ``RECID`` and + the declared money columns as int64. Never the design weight, and + never a disclosure-aggregate row. + recid: The kept rows' identifiers, in money-view order. + flpdyr: The kept rows' raw observation year, carried from the status + artifact and never restated. + demographic_status: The kept rows' demographic match flag. + design_weight: The kept rows' ``S006``, in the delivered integer + hundredths, unaltered and outside the money view. + excluded: The audit identity of the removed disclosure-aggregate rows. + receipt: What was joined, kept and removed. + """ + + money_view: pd.DataFrame + recid: np.ndarray + flpdyr: np.ndarray + demographic_status: np.ndarray + design_weight: np.ndarray + excluded: Mapping[str, object] + receipt: Mapping[str, object] + + +def _status_arrays(status: object, rows: int | None = None) -> Mapping[str, np.ndarray]: + """Every status column this adapter reads, present and the same length. + + The length check covers ``FLPDYR``, ``S006`` and the demographic status as + well as the two join keys, so a short or long side column is a refusal + rather than a silently misaligned slice later. + """ + try: + typed = status.typed + except AttributeError as error: + raise PufGrowthRefusalError("STATUS_ARTIFACT_TYPE") from error + # Presence first for every column, then length for every column, so a + # missing column is reported as missing rather than as a length mismatch. + for name in _STATUS_COLUMNS: + _require(name in typed, "STATUS_ARTIFACT_COLUMN", name) + if rows is not None: + for name in _STATUS_COLUMNS: + _require( + int(np.asarray(typed[name]).shape[0]) == rows, + "STATUS_COLUMN_ROWS", + name, + ) + return typed + + +def adapt_projection_to_money_view( + decoded: PufMonetarySourceProjection, + status: object, + compiled: CompiledPufGrowth, + *, + resource_bytes: bytes, +) -> PufMoneyViewAdaptation: + """Join the projection to the status artifact and drop only the aggregates. + + The join is by position **and** proved by value: the two artifacts' whole + ``RECID`` sequences and disclosure flags must be element-wise identical + before anything is selected, so a row's amounts and its status cannot come + from different returns. + + Exactly the rows outside the individual-return amount universe are + removed, and their identity is recorded. Every removed row must hold the + out-of-universe sentinel in every projected column, and no kept row may + hold it, so the sentinel cannot reach a numeric input in either direction. + + The contract is checked against the adopted baseline before anything is + recorded — see :func:`require_price_baseline_contract` — because the + receipt below states the source year, the design-weight identity and the + absence of a return-mass transition as facts about this run. + + Args: + decoded: A decoded thirteen-field projection. + status: The decoded ``puf_2015_raw_return_status`` artifact. + compiled: The compiled price-baseline contract. + resource_bytes: The retained BLS body authenticated by the entry gate. + + Returns: + The money view and everything deliberately kept out of it. + + Raises: + PufGrowthRefusalError: On any contract, join, universe or sentinel + violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require( + isinstance(decoded, PufMonetarySourceProjection), "PROJECTION_ARTIFACT_TYPE" + ) + projection_recid = np.asarray(decoded.typed[SOURCE_RECID_FIELD], dtype=" 0, "PROJECTION_NO_ROWS") + typed = _status_arrays(status, rows) + status_recid = np.asarray(typed[SOURCE_RECID_FIELD], dtype=" 0, "NO_AMOUNT_KNOWN_ROW") + _require( + removed == int(decoded.facts["row_class_rows"]["disclosure_aggregate"]), + "EXCLUDED_ROW_COUNT", + ) + + money_fields = compiled.money_fields + _require(set(money_fields) <= set(decoded.typed), "PROJECTION_MISSING_MONEY_FIELD") + _require(AGI_FIELD in money_fields, "PRICE_BASELINE_AGI_FIELD", AGI_FIELD) + columns: dict[str, np.ndarray] = {} + for field in money_fields: + values = np.asarray(decoded.typed[field], dtype=" int | None: + if greater not in frame.columns or lesser not in frame.columns: + return None + return int((frame[lesser] > frame[greater]).sum()) + + +def _bits(values: np.ndarray) -> np.ndarray: + """A float64 array's raw bit patterns, so ``-0.0`` is not ``0.0``.""" + return np.ascontiguousarray(values, dtype="float64").view(" Mapping[str, object]: + """Measure what one positive factor must leave true, and refuse otherwise. + + Three things happen here, in this order, and the order is deliberate. + + *The summaries.* Per column: dtype, finiteness, the sign set, the zero set, + no negative zero, and the exact integer sum of the input column times the + factor against the grown ``math.fsum``. Then the pairwise relations the + booklets leave open, counted before and after and required only to be + **equal**. These are the numbers a reader wants in the receipt, and each + names a specific symptom when it fires. + + *The exact row comparison.* Every kept row of every money column must be + bit-for-bit the contract's own resolved float64 factor applied to that + row's own delivered amount, and the two provenance columns must be + bit-for-bit their sources. This is the statement the summaries cannot + make: a coordinated rearrangement preserves every total, every sign, every + zero and every pairwise count, and is caught only here. It runs last so + that the narrower checks keep reporting the narrower diagnosis — and so + that each of them stays independently falsifiable. + + *The design weight.* Compared, if a caller supplies the weights it carried + forward, against the adaptation's own ``S006``. Not supplied, it is + reported as a declaration and **not** as a measurement. + + Args: + adaptation: The adaptation whose money view was grown. + grown: The result of :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` + on that money view, under this same contract. + compiled: The contract, checked against the adopted baseline. + resource_bytes: The retained BLS body authenticated by the entry gate. + design_weight_after: The post-growth design weights, if the caller ran + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + Omitted, ``design_weight_unchanged`` is ``None``, not ``True``. + relative_tolerance: Tolerance for the scaled-sum comparison only. + + Raises: + PufGrowthRefusalError: On any contract, identity, row, sum, pairwise or + weight violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require(type(grown) is GrownPufTable, "GROWN_TYPE") + _require(type(adaptation) is PufMoneyViewAdaptation, "ADAPTATION_TYPE") + # One contract, named the same way by all three objects. Without this the + # measurements below would describe a contract the table was not grown + # under, and every number in the receipt would be attributed to it. + contract_sha256 = compiled.sha256 + _require(grown.contract_sha256 == contract_sha256, "GROWN_CONTRACT_MISMATCH") + _require( + grown.receipt["contract_sha256"] == contract_sha256, + "GROWN_RECEIPT_CONTRACT_MISMATCH", + ) + _require( + adaptation.receipt["contract_sha256"] == contract_sha256, + "ADAPTATION_CONTRACT_MISMATCH", + ) + source = adaptation.money_view + table = grown.table + rows = len(source) + _require(len(table) == rows, "GROWN_ROWS") + _require( + bool( + np.array_equal( + table[SOURCE_RECID_FIELD].to_numpy(), + source[SOURCE_RECID_FIELD].to_numpy(), + ) + ), + "GROWN_RECID", + ) + factor = compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).value + columns: dict[str, object] = {} + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + _require( + resolved.ratio_bits + == compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).ratio_bits, + "MIXED_FACTOR", + field, + ) + original = source[field].to_numpy() + result = table[field].to_numpy() + _require(result.dtype == np.dtype("float64"), "GROWN_DTYPE", field) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", field) + _require( + bool((np.sign(result) == np.sign(original.astype("float64"))).all()), + "SIGN_CHANGED", + field, + ) + zeros = result == 0.0 + _require(not bool(np.signbit(result[zeros]).any()), "NEGATIVE_ZERO", field) + _require(bool(np.array_equal(zeros, original == 0)), "ZERO_SET_CHANGED", field) + exact_sum = int(sum(int(value) for value in original.tolist())) + gross_sum = int(sum(abs(int(value)) for value in original.tolist())) + grown_sum = math.fsum(result.tolist()) + expected = exact_sum * factor + # Scaled by the column's **gross** magnitude, not its net total. The + # float error one positive factor can accumulate is bounded by the + # magnitude that passed through it; a signed column whose positives + # and negatives nearly cancel has a tiny net and the same gross, and + # scaling by the net alone would refuse a correct result for it. + net_scale = max(abs(expected), 1.0) + scale = max(abs(gross_sum * factor), net_scale) + _require( + abs(grown_sum - expected) <= relative_tolerance * scale, + "SCALED_SUM_DISAGREEMENT", + field, + ) + columns[field] = { + "rows": rows, + "exact_source_sum": exact_sum, + "gross_source_sum_absolute": gross_sum, + "grown_fsum": grown_sum, + "expected_grown_sum": expected, + "scale_basis": "gross_absolute_source_sum_times_factor", + "relative_difference": ( + abs(grown_sum - expected) / scale if scale else 0.0 + ), + "relative_difference_net_scaled": ( + abs(grown_sum - expected) / net_scale if net_scale else 0.0 + ), + "negative": int((original < 0).sum()), + "positive": int((original > 0).sum()), + "zero": int((original == 0).sum()), + "ratio_bits": resolved.ratio_bits, + "grown_minimum": float(result.min()), + "grown_maximum": float(result.max()), + } + pairwise = {} + for name, greater, lesser in PAIRWISE_OBSERVATIONS: + before = _violations(source, greater, lesser) + after = _violations(table, greater, lesser) + # A contract over a subset of the projection's fields can leave one + # side of a relation out of the money view entirely. ``None == None`` + # would then pass, and the entry would read as an observation that was + # never made, so the absence is named instead of being counted. + _require((before is None) == (after is None), "PAIRWISE_COLUMN_ASYMMETRY", name) + observable = before is not None + if observable: + _require(before == after, "PAIRWISE_VIOLATIONS_MOVED", name) + pairwise[name] = { + "greater": greater, + "lesser": lesser, + "violations_before": before, + "violations_after": after, + "status": ( + "observed_not_enforced" + if observable + else "not_observable_column_absent" + ), + "absent_columns": sorted( + column for column in (greater, lesser) if column not in source.columns + ), + } + + # The exact row comparison. ``apply_puf_growth`` selects ``values != 0.0`` + # for a SignBranch.ANY rule and writes ``values[selected] * factor``, so + # the expected array is reproduced here by the same two operations in the + # same order and compared on raw bits. Nothing is recomputed from the + # levels or the rational: the contract's own resolved float64 is the one + # multiplier, which is what makes this a check of the transform rather + # than a second opinion about the factor. + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + original = source[field].to_numpy() + expected = original.astype("float64") + selected = expected != 0.0 + with np.errstate(over="ignore", under="ignore"): + expected[selected] = expected[selected] * resolved.value + result = table[field].to_numpy() + disagreeing = int(np.count_nonzero(_bits(result) != _bits(expected))) + _require(disagreeing == 0, "ROW_GROWTH_DISAGREEMENT", field) + + # The two columns ``apply_puf_growth`` copies rather than grows. A silent + # edit here would restate provenance while every money column still + # matched. + for column, origin in ( + (PROVENANCE_RECID_COLUMN, source[SOURCE_RECID_FIELD].to_numpy()), + ( + PROVENANCE_SOURCE_AGI_COLUMN, + source[SOURCE_AGI_FIELD].to_numpy().astype("float64"), + ), + ): + _require(column in table.columns, "PROVENANCE_COLUMN_MISSING", column) + carried = table[column].to_numpy() + if column == PROVENANCE_SOURCE_AGI_COLUMN: + _require( + bool((_bits(carried) == _bits(origin)).all()), + "PROVENANCE_DISAGREEMENT", + column, + ) + else: + _require( + bool(np.array_equal(carried, origin)), + "PROVENANCE_DISAGREEMENT", + column, + ) + + design_weight, design_weight_unchanged = _design_weight_observation( + adaptation, compiled, design_weight_after + ) + return MappingProxyType( + { + "rows": rows, + "factor_ratio_bits": compiled.factor_for( + compiled.money_fields[0], SignBranch.ANY + ).ratio_bits, + "factor_float64": factor, + "exact_rational": { + "numerator": PRICE_BASELINE_RATIO[0], + "denominator": PRICE_BASELINE_RATIO[1], + }, + "output_rounding": "none_unrounded_float64", + "columns": columns, + "pairwise_observations": pairwise, + "row_exact_growth": { + "rows": rows, + "fields": list(compiled.money_fields), + "comparison": ( + "bitwise float64 equality of every kept row against the " + "contract's own resolved factor applied to that row's " + "delivered amount, zeros left untouched" + ), + "disagreeing_rows": 0, + "provenance_columns_compared": list( + (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + ), + }, + "contract_sha256": contract_sha256, + "contract_named_by": { + "compiled": contract_sha256, + "grown_result": grown.contract_sha256, + "grown_receipt": grown.receipt["contract_sha256"], + "adaptation_receipt": adaptation.receipt["contract_sha256"], + }, + "design_weight": design_weight, + # ``None`` when no post-growth weight was supplied. This is never + # ``True`` unless the arrays were actually compared. + "design_weight_unchanged": design_weight_unchanged, + "release_eligible": compiled.release_eligible, + } + ) + + +def _design_weight_observation( + adaptation: PufMoneyViewAdaptation, + compiled: CompiledPufGrowth, + design_weight_after: Sequence[float] | np.ndarray | None, +) -> tuple[dict[str, object], bool | None]: + """What the design weight did, separating the declaration from a measurement. + + The declaration is the contract's: an identity series whose two levels are + both one, so the declared factor is exactly ``1.0``. That is a statement + about the recipe, and it is reported as one. + + The measurement needs a second array. ``adaptation.design_weight`` is the + status artifact's own ``S006`` sliced to the kept rows — the before side, + with its digest recorded so the comparison is auditable. The after side is + whatever the caller carried forward. Without it there is no observation to + report, and this returns ``None`` rather than manufacturing one. + """ + rule = compiled.design_weight_rule + declared = compiled.factor_for(rule.field, SignBranch.ANY) + before = np.asarray(adaptation.design_weight) + _require(before.ndim == 1, "DESIGN_WEIGHT_SHAPE", rule.field) + record: dict[str, object] = { + "field": rule.field, + "rows": int(before.shape[0]), + "declared_series": rule.factor_series, + "declared_source_level": declared.source_level, + "declared_target_level": declared.target_level, + "declared_factor_float64": declared.value, + "declared_return_mass_transition": "none", + "before_source": ( + "adaptation.design_weight: the status artifact's own S006, in " + "delivered integer hundredths, sliced to the kept rows" + ), + "before_sha256": hashlib.sha256( + np.ascontiguousarray(before, dtype=" None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufRawSourceRefusalError: + return PufRawSourceRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed definition +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SourcePin: + """The closed content identity of one delivered PUF file.""" + + name: str + codec: str + source_name: str + bytes: int + sha256: str + git_blob_sha1: str + header_record_canonical_sha256: str + delivered_header: tuple[str, ...] + data_records: int + + +@dataclass(frozen=True) +class PufRawSourceDefinition: + """A closed source declaration plus the route that produced it.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + main: SourcePin + demographic: SourcePin + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("ascii") + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _hex40(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 40 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _pin(document: Mapping[str, object], name: str) -> SourcePin: + entry = document["sources"][name] + header = entry["delivered_header"] + if not isinstance(header, list) or not header: + raise _refuse("DEFINITION_HEADER", name) + if len(header) != entry["delivered_header_width"]: + raise _refuse("DEFINITION_HEADER_WIDTH", name) + if len(set(header)) != len(header): + raise _refuse("DEFINITION_HEADER_DUPLICATE", name) + if _canonical_header_digest(header) != entry["header_record_canonical_sha256"]: + raise _refuse("DEFINITION_HEADER_DIGEST", name) + size = entry["bytes"] + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + raise _refuse("DEFINITION_SOURCE_BYTES", name) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("DEFINITION_SOURCE_OVER_CAP", name) + return SourcePin( + name=name, + codec=entry["codec"], + source_name=entry["source_name"], + bytes=size, + sha256=_hex64(entry["sha256"], f"{name}.sha256"), + git_blob_sha1=_hex40(entry["git_blob_sha1"], f"{name}.git_blob_sha1"), + header_record_canonical_sha256=_hex64( + entry["header_record_canonical_sha256"], f"{name}.header" + ), + delivered_header=tuple(header), + data_records=entry["data_records"], + ) + + +def _frozen(value: object) -> object: + """A deeply immutable view: mappings become proxies, sequences tuples. + + ``MappingProxyType`` only freezes the top level. ``definition.document`` + is reached through ordinary read-only attribute access, so handing out a + mutable nested container let a caller change a declared lexical width or + a CSV limit — changing what parses — while ``canonical``, ``sha256`` and + therefore ``params_text`` and the node key all stayed identical. Freezing + the whole tree closes that without any caller sandboxing: the document is + simply a value, as its canonical bytes always were. + """ + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _canonical_header_digest(header: Sequence[str]) -> str: + """The audit's header identity: sha256 of the canonical JSON header list.""" + + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _definition_from_document( + document: Mapping[str, object], *, route: str +) -> PufRawSourceDefinition: + if document.get("schema") != "microcosm.us.puf_2015_raw_source_definition": + raise _refuse("DEFINITION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("DEFINITION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("DEFINITION_ROUTE") + fields = document["fields"] + declared = tuple(field["name"] for field in fields) + if declared != _MAIN_FIELDS + _DEMOGRAPHIC_FIELDS: + raise _refuse("DEFINITION_FIELDS") + for field in fields: + width = field["lexical_width"] + if not isinstance(width, int) or isinstance(width, bool) or not 0 < width <= 64: + raise _refuse("DEFINITION_LEXICAL_WIDTH", field["name"]) + if field["grammar"] != "ascii_decimal_unsigned": + raise _refuse("DEFINITION_GRAMMAR", field["name"]) + if tuple(document["aggregates"]["recids"]) != PUF_AGGREGATE_RECIDS: + raise _refuse("DEFINITION_AGGREGATE_RECIDS") + canonical = canonical_json(dict(document)) + # The document is re-derived from the canonical bytes and then deeply + # frozen, so the two can never disagree and neither can be edited. + return PufRawSourceDefinition( + route=route, + document=_frozen(json.loads(canonical.decode("ascii"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + main=_pin(document, "main"), + demographic=_pin(document, "demographic"), + ) + + +def _thawed(value: object) -> object: + """The plain-container form of a frozen document, for canonical rendering.""" + + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_DEFINITION_PACKAGE).joinpath(_DEFINITION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufRawSourceDefinition | None = None + + +def packaged_definition() -> PufRawSourceDefinition: + """The one closed definition a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("ascii")) + _PACKAGED = _definition_from_document(document, route="packaged") + return _PACKAGED + + +def definition_document_json( + definition: PufRawSourceDefinition, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a definition's document. + + The definition's own document is deeply frozen. A caller that wants to + build a *different* document from this one — a fixture, say — starts here + and gets a copy it owns, rather than a handle on the original. + """ + + return json.loads(definition.canonical.decode("ascii")) + + +def fixture_definition(document: Mapping[str, object]) -> PufRawSourceDefinition: + """Build a definition on the explicit non-production test route. + + A fixture definition must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. There is no route where an omitted or optional expected hash admits + a file: a fixture carries its own real pins over its own invented bytes. + """ + + definition = _definition_from_document(document, route="test_fixture") + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged = packaged_definition() + genuine = {packaged.main.sha256, packaged.demographic.sha256} + if {definition.main.sha256, definition.demographic.sha256} & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return definition + + +# -------------------------------------------------------------------------- +# The pinned raw-byte codec +# -------------------------------------------------------------------------- + + +def _identity(status: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_size, + status.st_mtime_ns, + ) + + +def _git_blob_sha1(payload: bytes) -> str: + header = b"blob " + str(len(payload)).encode("ascii") + b"\0" + return hashlib.sha1(header + payload).hexdigest() + + +def load_pinned_puf_bytes( + path: Path, *, store: object | None = None, pin: SourcePin +) -> bytes: + """Read one pinned PUF delivery file, or refuse. + + The codec authenticates the bytes itself: exact length, SHA-256 and git + blob SHA-1, all three from the descriptor it opened. It never consults a + capture receipt or run summary, so a forged receipt cannot admit changed + bytes. + + ``O_NOFOLLOW`` protects the path *this function receives*. The graph + executor resolves source paths strictly before codec lookup, so under the + executor the codec sees an already-resolved path and a user's symlink is + followed upstream; on a direct call a symlink is refused here. + ``O_NONBLOCK`` means a FIFO cannot make the codec wait. + """ + + del store # identity is content; no content store is consulted + if not isinstance(pin, SourcePin): + raise _refuse("CODEC_PIN_TYPE") + if pin.bytes > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("CODEC_PIN_OVER_CAP", pin.name) + source = Path(path) + try: + descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except OSError as error: + raise _refuse("CODEC_OPEN", pin.name) from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise _refuse("CODEC_NOT_REGULAR_FILE", pin.name) + if before.st_size != pin.bytes: + raise _refuse("CODEC_SIZE", pin.name) + if _identity(source.lstat()) != _identity(before): + raise _refuse("CODEC_PATH_IDENTITY", pin.name) + digest = hashlib.sha256() + blob = hashlib.sha1(b"blob " + str(pin.bytes).encode("ascii") + b"\0") + chunks: list[bytes] = [] + total = 0 + limit = pin.bytes + 1 + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > limit: + raise _refuse("CODEC_GREW", pin.name) + digest.update(chunk) + blob.update(chunk) + chunks.append(chunk) + if total != pin.bytes: + raise _refuse("CODEC_LENGTH", pin.name) + if _identity(os.fstat(descriptor)) != _identity(before): + raise _refuse("CODEC_CHANGED_DURING_READ", pin.name) + if digest.hexdigest() != pin.sha256: + raise _refuse("CODEC_SHA256", pin.name) + if blob.hexdigest() != pin.git_blob_sha1: + raise _refuse("CODEC_GIT_BLOB_SHA1", pin.name) + finally: + os.close(descriptor) + return b"".join(chunks) + + +def register_us_puf_raw_source_codecs( + registry: SourceCodecRegistry, *, definition: PufRawSourceDefinition | None = None +) -> SourceCodecRegistry: + """Bind the two pinned byte codecs into ``registry`` through ``register_bytes``.""" + + resolved = packaged_definition() if definition is None else definition + registry.register_bytes( + resolved.main.codec, partial(load_pinned_puf_bytes, pin=resolved.main) + ) + registry.register_bytes( + resolved.demographic.codec, + partial(load_pinned_puf_bytes, pin=resolved.demographic), + ) + return registry + + +def puf_raw_source_codecs( + definition: PufRawSourceDefinition | None = None, +) -> SourceCodecRegistry: + """A private registry holding only this slice's two pinned byte codecs.""" + + return register_us_puf_raw_source_codecs( + SourceCodecRegistry(), definition=definition + ) + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufRawReturnStatus: + """The decoded slice: typed arrays, lexical tokens and aggregate facts.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + +def csv_acceptance_profile() -> dict[str, object]: + """The effective CSV acceptance state this decoder actually parses under. + + ``csv.field_size_limit`` is process-global state that silently changes + which deliveries parse: at a lowered limit a file that was admitted is + refused, and at a raised limit a file that was refused is admitted. It is + therefore part of the producer's identity, exactly as + :mod:`asec_income_observations` and :mod:`acs_housing_universe_source` + already bind it. This function only *reads* it — nothing in this module + ever sets it, so no other stage's acceptance is disturbed. + + The remaining entries are the reader semantics :func:`_reader` fixes + itself; the delimiter, quote character and doublequote flag come from the + closed definition and are checked against it at decode time. + """ + + return { + "engine": "stdlib_csv_reader", + "encoding": "utf-8-strict", + "newline": "", + "quoting": int(csv.QUOTE_MINIMAL), + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + } + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one. + + The packaged definition states ``field_size_limit``. Before this check it + was a decorative fact: the decoder relied on the process default happening + to match. Now the pinned value is the value the parse actually ran under, + or the decode refuses — a lowered limit is a refusal, never a silent + reparse, and a raised limit is a refusal, never a silent widening. + """ + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _token(value: str, *, width: int, label: str) -> str: + """An unsigned ASCII decimal token within its declared allocation.""" + + if not isinstance(value, str): + raise _refuse("TOKEN_TYPE", label) + raw = value.encode("ascii", errors="strict") if value.isascii() else None + if raw is None: + raise _refuse("TOKEN_NON_ASCII", label) + if b"\x00" in raw: + raise _refuse("TOKEN_EMBEDDED_NUL", label) + if not raw: + raise _refuse("TOKEN_EMPTY", label) + if len(raw) > width: + raise _refuse("TOKEN_OVER_WIDTH", label) + if any(byte not in _DIGITS for byte in raw): + raise _refuse("TOKEN_GRAMMAR", label) + return value + + +def _int64(token: str, label: str) -> int: + value = int(token) + if value > _INT64_MAX: + raise _refuse("TOKEN_INT64_OVERFLOW", label) + return value + + +def _widths(definition: PufRawSourceDefinition) -> dict[str, int]: + return { + field["name"]: field["lexical_width"] for field in definition.document["fields"] + } + + +def _domains(definition: PufRawSourceDefinition) -> dict[str, frozenset[int]]: + domains: dict[str, frozenset[int]] = {} + for field in definition.document["fields"]: + values = field.get("domain") + if values is not None: + domains[field["name"]] = frozenset(int(value) for value in values) + return domains + + +def _agerange_domains( + definition: PufRawSourceDefinition, +) -> dict[int, frozenset[int]]: + for field in definition.document["fields"]: + if field["name"] == "AGERANGE": + cases = field["conditional_domain"]["cases"] + return { + int(key): frozenset(int(value) for value in values) + for key, values in cases.items() + } + raise _refuse("DEFINITION_AGERANGE") + + +def _check_header( + reader, pin: SourcePin, *, profile: Mapping[str, object] +) -> list[str]: + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING", pin.name) from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE", pin.name) from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT", pin.name) + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH", pin.name) + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST", pin.name) + return header + + +def _records(reader, pin: SourcePin, *, profile: Mapping[str, object]): + width = len(pin.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE", pin.name) from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP", pin.name) + if len(record) != width: + raise _refuse("RECORD_WIDTH", pin.name) + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP", pin.name) + yield record + + +def decode_puf_raw_source( + main_bytes: bytes, + demographic_bytes: bytes, + definition: PufRawSourceDefinition, +) -> PufRawReturnStatus: + """Decode the two delivery payloads into the typed status slice. + + This is structure only. It checks headers, widths, token grammar, closed + domains and the exact lexical join against the definition it is handed, + and it authorizes nothing about where the bytes came from. + """ + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + widths = _widths(definition) + domains = _domains(definition) + agerange = _agerange_domains(definition) + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + + demographic = _decode_demographic( + demographic_bytes, definition, profile=profile, widths=widths + ) + main = _decode_main( + main_bytes, + definition, + profile=profile, + widths=widths, + domains=domains, + aggregates=aggregates, + ) + + rows = len(main["RECID_lexical"]) + orphans = set(demographic["by_key"]) - set(main["RECID_lexical"]) + if orphans: + raise _refuse("DEMOGRAPHIC_ORPHAN_KEYS", len(orphans)) + + typed: dict[str, np.ndarray] = { + name: np.empty(rows, dtype=_TYPED_DTYPES[name]) for name in _TYPED_COLUMNS + } + for name in _MAIN_FIELDS: + typed[name][:] = np.asarray(main[name], dtype=_TYPED_DTYPES[name]) + typed["disclosure_aggregate"][:] = np.asarray( + main["disclosure_aggregate"], dtype=_TYPED_DTYPES["disclosure_aggregate"] + ) + + lexical: dict[str, list[str]] = { + name: list( + main["RECID_lexical"] if name == "RECID" else main[f"{name}_lexical"] + ) + for name in _MAIN_FIELDS + } + for name in _DEMOGRAPHIC_FIELDS: + lexical[name] = [""] * rows + + status = np.zeros(rows, dtype=_TYPED_DTYPES["demographic_status"]) + for name in _DEMOGRAPHIC_FIELDS: + typed[name][:] = _ABSENT_SENTINEL + + aggregate_demographic_rows = 0 + for index, key in enumerate(main["RECID_lexical"]): + record = demographic["by_key"].get(key) + if record is None: + continue + status[index] = 1 + dsi = int(typed["DSI"][index]) + allowed = agerange.get(dsi) + if allowed is None: + raise _refuse("AGERANGE_CONDITION_UNKNOWN") + for name in _DEMOGRAPHIC_FIELDS: + token = record[name] + value = _int64(token, name) + permitted = allowed if name == "AGERANGE" else domains[name] + if value not in permitted: + raise _refuse("DEMOGRAPHIC_DOMAIN", name) + typed[name][index] = value + lexical[name][index] = token + if bool(typed["disclosure_aggregate"][index]): + aggregate_demographic_rows += 1 + typed["demographic_status"][:] = status + + del aggregate_demographic_rows # re-derived below from the arrays themselves + if demographic["records"] != int(status.sum()): + raise _refuse("DEMOGRAPHIC_RECORD_COUNT") + facts = _facts_from_arrays( + typed, + join_method=_join_method(definition), + untyped_main_columns=_untyped_main_columns(definition), + ) + _check_declared_expectations(facts, definition) + return PufRawReturnStatus( + typed=MappingProxyType({name: array for name, array in typed.items()}), + lexical=MappingProxyType( + {name: tuple(values) for name, values in lexical.items()} + ), + facts=MappingProxyType(facts), + ) + + +def _check_declared_expectations( + facts: Mapping[str, object], definition: PufRawSourceDefinition +) -> None: + """Cross-check the derived counts against the definition's declared ones. + + The packaged definition states how many records each delivery holds, how + many keys should match, how many main records should stay unmatched and + how many aggregate records to expect. Nothing compared them, so they were + documentation. The byte pins make them implied, but implied is not + checked: a delivery that hashed correctly and parsed to a different shape + would have gone unnoticed. + + This runs on the packaged route only. A fixture definition carries its own + invented counts over its own invented bytes, and inventing a fixture that + restates the genuine expectations would be the wrong kind of fidelity. + """ + + if definition.route != "packaged": + return + join = definition.document["join"] + for label, derived, declared in ( + ("rows", facts["rows"], definition.main.data_records), + ( + "demographic_records", + facts["demographic_records"], + definition.demographic.data_records, + ), + ("matched_keys", facts["matched_keys"], join["expected_matched_keys"]), + ( + "main_unmatched_records", + facts["main_unmatched_records"], + join["expected_main_unmatched_records"], + ), + ( + "aggregate_records", + facts["aggregate_records"], + definition.document["aggregates"]["expected_records"], + ), + ): + if derived != declared: + raise _refuse("DECLARED_EXPECTATION", label) + + +def _join_method(definition: PufRawSourceDefinition) -> str: + method = definition.document["join"]["method"] + if not isinstance(method, str) or not method: + raise _refuse("DEFINITION_JOIN_METHOD") + return method + + +def _untyped_main_columns(definition: PufRawSourceDefinition) -> int: + return len(definition.main.delivered_header) - len(_MAIN_FIELDS) + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + *, + join_method: str, + untyped_main_columns: int, +) -> dict[str, object]: + """Derive every reported fact from the typed arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count the arrays do not carry. Only + ``join_method`` and ``untyped_main_columns`` come from outside: both are + properties of the definition, not of the decoded rows. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + matched = int(np.count_nonzero(status == 1)) + return { + "rows": rows, + "main_records": rows, + "demographic_records": matched, + "matched_keys": matched, + "main_unmatched_records": rows - matched, + "demographic_orphan_records": 0, + "aggregate_records": int(np.count_nonzero(aggregate == 1)), + "aggregate_demographic_records": int( + np.count_nonzero((aggregate == 1) & (status == 1)) + ), + "join_status": "one_to_one_subset_compatible", + "join_method": join_method, + "code_frequencies": _code_frequencies(typed), + "period_semantics": "unresolved_raw_FLPDYR", + "weight_units": "S006_hundredths", + "amounts_decoded": False, + "untyped_main_columns": untyped_main_columns, + } + + +def _code_frequencies(typed: Mapping[str, np.ndarray]) -> dict[str, dict[str, int]]: + counted = ("FLPDYR", "FLPDMO", "MARS", "DSI", "demographic_status") + frequencies: dict[str, dict[str, int]] = {} + for name in (*counted, *_DEMOGRAPHIC_FIELDS): + values, counts = np.unique(typed[name], return_counts=True) + frequencies[name] = { + str(int(value)): int(count) + for value, count in zip(values.tolist(), counts.tolist(), strict=True) + } + return frequencies + + +def _decode_main( + payload: bytes, + definition: PufRawSourceDefinition, + *, + profile, + widths, + domains, + aggregates, +) -> dict[str, object]: + pin = definition.main + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in _MAIN_FIELDS} + columns: dict[str, list] = {name: [] for name in _MAIN_FIELDS} + lexical: dict[str, list[str]] = {f"{name}_lexical": [] for name in _MAIN_FIELDS} + keys: list[str] = [] + numeric_keys: set[int] = set() + seen: set[str] = set() + aggregate_flags: list[int] = [] + for record in _records(reader, pin, profile=profile): + values: dict[str, int] = {} + for name in _MAIN_FIELDS: + token = _token(record[index[name]], width=widths[name], label=name) + lexical[f"{name}_lexical"].append(token) + values[name] = _int64(token, name) + key = lexical["RECID_lexical"][-1] + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = values["RECID"] + if recid in numeric_keys: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + keys.append(key) + for name in ("FLPDYR", "FLPDMO", "MARS", "DSI"): + if values[name] not in domains[name]: + raise _refuse("MAIN_DOMAIN", name) + aggregate = recid in aggregates + if aggregate != (values["MARS"] == 0): + raise _refuse("AGGREGATE_MARS_DISAGREEMENT") + aggregate_flags.append(1 if aggregate else 0) + for name in _MAIN_FIELDS: + columns[name].append(values[name]) + if not keys: + raise _refuse("MAIN_NO_RECORDS") + result: dict[str, object] = dict(columns) + result.update(lexical) + result["disclosure_aggregate"] = aggregate_flags + return result + + +def _decode_demographic( + payload: bytes, definition: PufRawSourceDefinition, *, profile, widths +) -> dict[str, object]: + pin = definition.demographic + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in ("RECID", *_DEMOGRAPHIC_FIELDS)} + by_key: dict[str, dict[str, str]] = {} + numeric_keys: set[int] = set() + records = 0 + for record in _records(reader, pin, profile=profile): + key = _token(record[index["RECID"]], width=widths["RECID"], label="RECID") + if key in by_key: + raise _refuse("DEMOGRAPHIC_DUPLICATE_LEXICAL_KEY") + recid = _int64(key, "RECID") + if recid in numeric_keys: + raise _refuse("DEMOGRAPHIC_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + by_key[key] = { + name: _token(record[index[name]], width=widths[name], label=name) + for name in _DEMOGRAPHIC_FIELDS + } + records += 1 + return {"by_key": by_key, "records": records} + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def _payload_bound( + rows: int, definition: PufRawSourceDefinition +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact will allocate, and its per-column parts.""" + + widths = _widths(definition) + sizes = { + name: rows * np.dtype(_TYPED_DTYPES[name]).itemsize for name in _TYPED_COLUMNS + } + sizes.update({f"{name}_lexical": rows * widths[name] for name in _LEXICAL_COLUMNS}) + return sum(sizes.values()), sizes + + +def encode_return_status( + status: PufRawReturnStatus, definition: PufRawSourceDefinition +) -> bytes: + """Encode the slice into the bounded, self-describing artifact envelope. + + Magic line, a 4-byte big-endian header length matching the ACS rent + placement precedent, a canonical JSON header, then explicit little-endian + integer bodies and fixed-width NUL-padded ASCII lexical bodies, in the + declared order. Every body carries its own SHA-256 in the header. + """ + + rows = status.rows + total, sizes = _payload_bound(rows, definition) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + widths = _widths(definition) + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _TYPED_COLUMNS: + array = np.ascontiguousarray(status.typed[name], dtype=_TYPED_DTYPES[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": name, + "kind": "typed", + "dtype": _TYPED_DTYPES[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + for name in _LEXICAL_COLUMNS: + body = _fixed_ascii(status.lexical[name], widths[name], name) + if len(body) != sizes[f"{name}_lexical"]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": f"{name}_lexical", + "kind": "lexical_fixed_ascii_nul_padded", + "width": widths[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + header = { + "schema_version": 1, + "type": [RETURN_STATUS_TYPE.name, RETURN_STATUS_TYPE.schema_version], + "rows": rows, + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(status.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (RETURN_STATUS_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + """A mapping whose key set is exactly ``keys``. Nothing extra, nothing missing.""" + + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_definition( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> PufRawSourceDefinition | None: + """The definition this payload may be checked against, or ``None``. + + A caller may hand one in. Otherwise a payload that declares the *packaged* + route is checked against this module's own packaged definition, because + that one is closed and importable. A fixture-route payload has no + reconstructible definition, so it is checked structurally only: the + production and fixture boundaries stay exactly where they are. + """ + + route = header["definition_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_DEFINITION_ROUTE") + digest = _hex64(header["definition_sha256"], "envelope.definition_sha256") + if definition is not None: + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + if definition.route != route or definition.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_MISMATCH") + return definition + if route == "packaged": + packaged = packaged_definition() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _hex40(entry["git_blob_sha1"], f"sources.{name}.git_blob_sha1") + _hex64(entry["header_record_canonical_sha256"], f"sources.{name}.header") + size = _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("ENVELOPE_SOURCE_OVER_CAP", name) + if definition is None: + continue + pin = definition.main if name == "main" else definition.demographic + if ( + entry["sha256"] != pin.sha256 + or entry["git_blob_sha1"] != pin.git_blob_sha1 + or size != pin.bytes + or entry["header_record_canonical_sha256"] + != pin.header_record_canonical_sha256 + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], + rows: int, + definition: PufRawSourceDefinition | None, +) -> tuple[list[Mapping[str, object]], dict[str, int]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = [ + *_TYPED_COLUMNS, + *(f"{name}_lexical" for name in _LEXICAL_COLUMNS), + ] + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + widths = _widths(definition) if definition is not None else None + resolved: list[Mapping[str, object]] = [] + sizes: dict[str, int] = {} + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name in _TYPED_DTYPES: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # Exact dtype string: this closes width, signedness and endianness + # at once, so `` _LEXICAL_WIDTH_MAX: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + if widths is not None and width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + sizes[field] = width + _hex64(entry["sha256"], f"{name}.sha256") + size = _envelope_int(entry["bytes"], f"{name}.bytes") + if size != expected: + raise _refuse("ENVELOPE_COLUMN_SIZE", name) + total += size + if total > BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved, sizes + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + widths: Mapping[str, int], + definition: PufRawSourceDefinition | None, +) -> None: + """Re-prove the slice's own invariants from the arrays that were read back. + + Lexical and numeric spellings must agree, absence must stay absence rather + than a zero code, the aggregate flag must follow from ``RECID`` and agree + with ``MARS``, and the join keys must still be unique both ways. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + if int(typed["S006"].min(initial=0)) < 0: + raise _refuse("READBACK_S006_NEGATIVE") + + present = status == 1 + for name in _MAIN_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + for name in _DEMOGRAPHIC_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + if not present[position]: + # Absence is the sentinel and an empty spelling, never a zero. + if token or int(values[position]) != _ABSENT_SENTINEL: + raise _refuse("READBACK_ABSENCE_NOT_EMPTY", name) + continue + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + + keys = lexical["RECID"] + if len(set(keys)) != rows: + raise _refuse("READBACK_DUPLICATE_LEXICAL_KEY") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected_aggregate = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufRawReturnStatus: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema: exact key sets, an exact dtype string for + every typed column, an exact width and body size for every lexical column, + and a total body bound. Every reported fact is then **recomputed** from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a match count or a code frequency it does not carry. + + This stays structural. It is not source authority: it proves that a payload + is internally consistent and, when a definition is available, that it + matches that definition's widths, domains and source pins. It authorizes + nothing about where the delivery bytes came from — the codec does that. + """ + + if not isinstance(payload, bytes) or not payload.startswith(RETURN_STATUS_MAGIC): + raise _refuse("ENVELOPE_MAGIC") + start = len(RETURN_STATUS_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("ascii")) + except (UnicodeError, ValueError, RecursionError) as error: + # A deeply nested or malformed header is a refusal, not a stray + # ``RecursionError`` or ``UnicodeDecodeError`` out of the JSON reader. + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != 1 or header["type"] != [ + RETURN_STATUS_TYPE.name, + RETURN_STATUS_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_definition(header, definition) + _check_envelope_sources(header, resolved) + columns, widths = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, widths, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + if resolved is not None: + join_method = _join_method(resolved) + untyped_main_columns = _untyped_main_columns(resolved) + else: + join_method = facts["join_method"] + if not isinstance(join_method, str) or not join_method: + raise _refuse("ENVELOPE_FACTS", "join_method") + untyped_main_columns = _envelope_int( + facts["untyped_main_columns"], "facts.untyped_main_columns" + ) + recomputed = _facts_from_arrays( + typed, + join_method=join_method, + untyped_main_columns=untyped_main_columns, + ) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + if resolved is not None: + _check_declared_expectations(recomputed, resolved) + return PufRawReturnStatus( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ("microcosm.build.us_runtime.puf_raw_source",) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.raw_source/implementation/2\n" + + +class USPufRawSourceKernel(KernelBase): + """Read the two pinned deliveries and produce one typed status artifact.""" + + ref = "us.puf.raw_source.return_status@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = packaged_definition() if definition is None else definition + if self.definition.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_DEFINITION_ROUTE") + if self.definition.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the definition must be the packaged file's own canonical bytes. + # Compare the bytes, not only their digest, and re-prove that the + # document this definition carries is still those exact bytes. + packaged = packaged_definition() + if ( + self.definition.canonical != packaged.canonical + or self.definition.sha256 != packaged.sha256 + or canonical_json(_thawed(self.definition.document)) + != self.definition.canonical + ): + raise _refuse("KERNEL_DEFINITION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions **and** the effective CSV profile. + + The acceptance profile is process state the parse runs under, so it + belongs in the producer's identity: change it and this node's key + moves, which is a recompute or a ``require`` miss. It can no longer + reuse an admission made under a different acceptance. + """ + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + != (definition.main.source_name, definition.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("return_status", RETURN_STATUS_TYPE),) + or set(context.params) != {"definition"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + demographic_bytes = load_source_bytes( + definition.demographic.codec, + context.sources[definition.demographic.source_name], + registry=self.source_codecs, + ) + status = decode_puf_raw_source(main_bytes, demographic_bytes, definition) + payload = encode_return_status(status, definition) + return KernelResult( + artifacts={"return_status": payload}, + receipt={ + "puf_raw_return_status": { + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "main_sha256": definition.main.sha256, + "demographic_sha256": definition.demographic.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + **{ + key: value + for key, value in status.facts.items() + if key != "code_frequencies" + }, + } + }, + ) + + +class USPufRawReturnStatusSummaryKernel(KernelBase): + """A downstream consumer: decode the typed artifact and report counts.""" + + ref = "us.puf.raw_source.return_status_summary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + status = decode_return_status(value.payload) + return KernelResult( + receipt={ + "puf_raw_return_status_summary": { + "producer_key": value.producer_key, + "rows": status.rows, + "matched_keys": status.facts["matched_keys"], + "main_unmatched_records": status.facts["main_unmatched_records"], + "aggregate_records": status.facts["aggregate_records"], + "period_semantics": status.facts["period_semantics"], + "weight_units": status.facts["weight_units"], + "amounts_decoded": status.facts["amounts_decoded"], + } + } + ) + + +def us_puf_raw_source_refs( + definition: PufRawSourceDefinition | None = None, +) -> tuple[SourceRef, ...]: + """The two ``SourceRef`` declarations this slice's node reads.""" + + resolved = packaged_definition() if definition is None else definition + return ( + SourceRef( + resolved.main.source_name, + resolved.main.codec, + description="2015 PUF main delivery, pinned by content.", + ), + SourceRef( + resolved.demographic.source_name, + resolved.demographic.codec, + description="2015 PUF demographic delivery, pinned by content.", + ), + ) + + +def us_puf_raw_source_node( + *, + population: str, + definition: PufRawSourceDefinition | None = None, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved = packaged_definition() if definition is None else definition + return Node( + f"{stage}.return_status", + USPufRawSourceKernel.ref, + population=population, + sources=(resolved.main.source_name, resolved.demographic.source_name), + params={"definition": resolved.params_text}, + artifact_outputs=(ArtifactOutput("return_status", RETURN_STATUS_TYPE),), + description=( + "Type the 2015 PUF return status slice from the pinned delivery " + "bytes. Owns no population columns." + ), + ) + + +def us_puf_raw_return_status_summary_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare an ordinary consumer of the typed artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.return_status_summary", + USPufRawReturnStatusSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + "return_status", producer, "return_status", RETURN_STATUS_TYPE + ), + ), + description="Read the typed PUF status artifact and report its counts.", + ) + + +def register_us_puf_raw_source_kernels( + registry: KernelRegistry, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufRawSourceKernel(definition=definition, source_codecs=source_codecs) + ) + registry.register(USPufRawReturnStatusSummaryKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py new file mode 100644 index 000000000..5992b7c1d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py @@ -0,0 +1,448 @@ +"""Measured ASEC at-interview coverage recodes omitted by frozen H5 inputs. + +The pooled ``census_cps`` inputs carry ``NOW_GRP`` and ``NOW_MRK`` for every +income year, but their 2022 and 2023 person tables omit seven other recodes +consumed by :func:`derive_us_cps_carried_inputs`. Those omissions otherwise +turn measured coverage into structural false values for two pooled vintages. + +This module restores the seven fields from the same SHA-pinned official ASEC +person members already used for ``ED_VAL`` and ``PAW_TYP``. Restoration is an +exact ``(source_year, PERIDNUM)`` join with redundant Census identity checks; +it does not predict, infer, or substitute any coverage value. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from collections.abc import Mapping +from pathlib import Path +from typing import BinaryIO + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + AsecEducationArchive, + fetch_asec_education_assistance_source, +) + +__all__ = [ + "ASEC_REPORTED_COVERAGE_INCOME_YEARS", + "ASEC_REPORTED_COVERAGE_RAW_COLUMNS", + "ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS", + "fill_asec_reported_coverage_source", + "load_asec_reported_coverage_sources", +] + +# ``NOW_GRP`` and ``NOW_MRK`` are deliberately absent: they already survive +# in every frozen pooled source and remain native rather than sidecar-restored. +ASEC_REPORTED_COVERAGE_RAW_COLUMNS: tuple[str, ...] = ( + "NOW_NONM", + "NOW_MCAID", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", +) +ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS: tuple[str, ...] = ( + "PH_SEQ", + "P_SEQ", + "A_LINENO", + "PERIDNUM", + *ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) +ASEC_REPORTED_COVERAGE_INCOME_YEARS = ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS +_AUDIT_WEIGHT_COLUMN = "A_FNLWGT" +_VALID_CODES = frozenset({1, 2}) + + +def _sha256_stream(stream: BinaryIO, *, chunk_size: int) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _verify_member_path( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> None: + if not path.is_file(): + raise FileNotFoundError(path) + if path.stat().st_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if digest != pins.member_sha256: + raise ValueError( + f"ASEC {pins.survey_year} person member SHA-256 mismatch: " + f"expected {pins.member_sha256}, got {digest}." + ) + + +def _verified_zip_member( + archive: zipfile.ZipFile, + pins: AsecEducationArchive, +) -> zipfile.ZipInfo: + members = [info for info in archive.infolist() if info.filename == pins.member] + if len(members) != 1: + raise ValueError( + f"ASEC {pins.survey_year} archive must contain exactly one " + f"{pins.member!r} member; found {len(members)}." + ) + info = members[0] + if info.file_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {info.file_size}." + ) + actual_crc = f"{info.CRC:08x}" + if actual_crc != pins.member_crc32: + raise ValueError( + f"ASEC {pins.survey_year} person member CRC32 mismatch: expected " + f"{pins.member_crc32}, got {actual_crc}." + ) + return info + + +def _read_one_source( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> pd.DataFrame: + usecols = [*ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS, _AUDIT_WEIGHT_COLUMN] + if zipfile.is_zipfile(path): + if path.stat().st_size != pins.zip_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} archive byte length mismatch: expected " + f"{pins.zip_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + archive_digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if archive_digest != pins.zip_sha256: + raise ValueError( + f"ASEC {pins.survey_year} archive SHA-256 mismatch: expected " + f"{pins.zip_sha256}, got {archive_digest}." + ) + with zipfile.ZipFile(path) as archive: + info = _verified_zip_member(archive, pins) + with archive.open(info) as member: + member_digest, member_size = _sha256_stream( + member, + chunk_size=chunk_size, + ) + if ( + member_size != pins.member_size_bytes + or member_digest != pins.member_sha256 + ): + raise ValueError( + f"ASEC {pins.survey_year} person member identity mismatch." + ) + with archive.open(info) as member: + try: + return pd.read_csv( + member, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is " + f"missing required column(s): {error}." + ) from error + + _verify_member_path(path, pins, chunk_size=chunk_size) + try: + return pd.read_csv( + path, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is missing " + f"required column(s): {error}." + ) from error + + +def _fixed_width_peridnum(values: pd.Series, *, label: str) -> pd.Series: + if values.isna().any(): + rows = values.index[values.isna()].tolist()[:5] + raise ValueError(f"{label} PERIDNUM is missing at row(s): {rows}.") + decoded = values.map( + lambda value: ( + value.decode() + if isinstance(value, (bytes, bytearray, np.bytes_)) + else value + ) + ) + valid = decoded.map(lambda value: isinstance(value, str)) + valid &= decoded.astype("string").str.fullmatch(r"[0-9]{22}", na=False) + if not valid.all(): + rows = decoded.index[~valid].tolist()[:5] + raise ValueError( + f"{label} PERIDNUM must be an exact 22-digit string at row(s): {rows}." + ) + return decoded.astype(str) + + +def _validated_source_year(values: pd.Series, *, label: str) -> pd.Series: + numeric = pd.to_numeric(values, errors="coerce") + valid = numeric.notna() & np.isfinite(numeric) & numeric.eq(np.floor(numeric)) + if not valid.all(): + rows = values.index[~valid].tolist()[:5] + raise ValueError(f"{label} source_year is invalid at row(s): {rows}.") + return numeric.astype(np.int64) + + +def _validated_codes(values: pd.Series, *, label: str) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy(dtype=np.float64) + boolean = values.map(lambda value: isinstance(value, (bool, np.bool_))).to_numpy() + valid = np.isfinite(numeric) & np.isin(numeric, sorted(_VALID_CODES)) & ~boolean + if not valid.all(): + rows = np.flatnonzero(~valid)[:5].tolist() + raise ValueError( + f"{label} must be complete integers in {sorted(_VALID_CODES)} at " + f"row(s): {rows}." + ) + return numeric.astype(np.int64) + + +def load_asec_reported_coverage_sources( + paths: Mapping[int, str | Path] | None = None, + *, + income_years: tuple[int, ...] = ASEC_REPORTED_COVERAGE_INCOME_YEARS, + chunk_size: int = 8 * 1024 * 1024, +) -> pd.DataFrame: + """Load the seven measured coverage recodes from pinned ASEC members.""" + + unknown = sorted(set(paths or ()) - set(ASEC_EDUCATION_ASSISTANCE_ARCHIVES)) + if unknown: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year(s) " + f"{unknown}; pinned income years: " + f"{list(ASEC_REPORTED_COVERAGE_INCOME_YEARS)}." + ) + if not income_years: + empty = pd.DataFrame( + columns=["source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS] + ) + empty.attrs["source_audit"] = {} + return empty + + parts: list[pd.DataFrame] = [] + audits: dict[int, dict[str, object]] = {} + for income_year in income_years: + pins = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(income_year) + if pins is None: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year " + f"{income_year}." + ) + provided = None if paths is None else paths.get(income_year) + path = ( + Path(provided).expanduser() + if provided is not None + else fetch_asec_education_assistance_source(income_year) + ) + raw = _read_one_source(path, pins, chunk_size=chunk_size) + if len(raw) != pins.rows: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source row count " + f"mismatch: expected {pins.rows}, got {len(raw)}." + ) + raw["PERIDNUM"] = _fixed_width_peridnum( + raw["PERIDNUM"], + label=f"ASEC {pins.survey_year} reported-coverage source", + ) + if raw["PERIDNUM"].duplicated(keep=False).any(): + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source PERIDNUM " + "must be unique." + ) + weights = pd.to_numeric(raw[_AUDIT_WEIGHT_COLUMN], errors="coerce").to_numpy( + dtype=np.float64 + ) + if ( + not np.isfinite(weights).all() + or (weights < 0.0).any() + or float(weights.sum()) <= 0.0 + ): + raise ValueError( + f"ASEC {pins.survey_year} A_FNLWGT must be finite and " + "nonnegative with positive total mass." + ) + + column_audits: dict[str, dict[str, float | int]] = {} + scaled_weights = weights / 100.0 + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + values = _validated_codes( + raw[column], + label=f"ASEC {pins.survey_year} {column}", + ) + raw[column] = values + yes = values == 1 + column_audits[column] = { + "no_rows": int(np.count_nonzero(values == 2)), + "yes_rows": int(np.count_nonzero(yes)), + "weighted_yes_share": float( + scaled_weights[yes].sum() / scaled_weights.sum() + ), + } + audits[income_year] = { + "rows": int(len(raw)), + "columns": column_audits, + } + part = raw.loc[:, list(ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS)].copy() + part.insert(0, "source_year", np.int64(income_year)) + parts.append(part) + + result = pd.concat(parts, ignore_index=True) + result.attrs["source_audit"] = audits + return result + + +def fill_asec_reported_coverage_source( + person: pd.DataFrame, + source: pd.DataFrame, +) -> pd.DataFrame: + """Restore all seven recodes by exact ``(source_year, PERIDNUM)`` join.""" + + required_person = ("source_year", "PERIDNUM") + missing_person = [column for column in required_person if column not in person] + if missing_person: + raise ValueError( + "ASEC reported-coverage repair requires person column(s): " + f"{missing_person}." + ) + required_source = ("source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS) + missing_source = [column for column in required_source if column not in source] + if missing_source: + raise ValueError( + f"ASEC reported-coverage sidecar missing column(s): {missing_source}." + ) + + donor = source.copy(deep=True) + donor["source_year"] = _validated_source_year( + donor["source_year"], label="ASEC reported-coverage sidecar" + ) + donor["PERIDNUM"] = _fixed_width_peridnum( + donor["PERIDNUM"], label="ASEC reported-coverage sidecar" + ) + duplicate = donor.duplicated(["source_year", "PERIDNUM"], keep=False) + if duplicate.any(): + keys = donor.loc[duplicate, ["source_year", "PERIDNUM"]].head(5) + raise ValueError( + "ASEC reported-coverage sidecar (source_year, PERIDNUM) keys must " + f"be unique; duplicate key(s): {keys.to_dict(orient='records')}." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + donor[column] = _validated_codes( + donor[column], label=f"ASEC reported-coverage sidecar {column}" + ) + + result = person.copy(deep=True) + person_years = _validated_source_year( + result["source_year"], label="ASEC reported-coverage person" + ) + result["PERIDNUM"] = _fixed_width_peridnum( + result["PERIDNUM"], label="ASEC reported-coverage frame" + ) + needed_years = sorted(int(year) for year in person_years.unique()) + covered_years = set(int(year) for year in donor["source_year"].unique()) + uncovered = [year for year in needed_years if year not in covered_years] + if uncovered: + raise ValueError( + "ASEC reported-coverage sidecar does not cover pooled income " + f"year(s): {uncovered}." + ) + + for year in needed_years: + year_mask = person_years.eq(year).to_numpy() + year_donor = donor.loc[donor["source_year"].eq(year)].set_index("PERIDNUM") + keys = result.loc[year_mask, "PERIDNUM"] + missing_keys = keys[~keys.isin(year_donor.index)].drop_duplicates() + if not missing_keys.empty: + raise ValueError( + "ASEC reported-coverage sidecar does not cover frame PERIDNUM " + f"key(s) for income year {year}: {missing_keys.tolist()[:5]}." + ) + aligned = year_donor.reindex(keys.to_numpy()) + aligned.index = result.index[year_mask] + + identity_pairs = ( + ( + "PH_SEQ", + "source_household_id" if "source_household_id" in result else "PH_SEQ", + ), + ("P_SEQ", "P_SEQ"), + ("A_LINENO", "A_LINENO"), + ) + for donor_column, frame_column in identity_pairs: + if frame_column not in result: + continue + observed = pd.to_numeric( + result.loc[year_mask, frame_column], errors="coerce" + ).to_numpy(dtype=np.float64) + expected = pd.to_numeric(aligned[donor_column], errors="coerce").to_numpy( + dtype=np.float64 + ) + mismatch = ( + ~np.isfinite(observed) | ~np.isfinite(expected) | (observed != expected) + ) + if mismatch.any(): + rows = result.index[year_mask].to_numpy()[mismatch][:5].tolist() + raise ValueError( + "ASEC reported-coverage redundant identity mismatch for " + f"{frame_column} against sidecar {donor_column} in income " + f"year {year} at row(s): {rows}." + ) + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + expected = aligned[column].to_numpy(dtype=np.int64) + if column in result: + current = result.loc[year_mask, column] + present = current.notna().to_numpy() + numeric = pd.to_numeric(current, errors="coerce").to_numpy( + dtype=np.float64 + ) + invalid = present & ( + ~np.isfinite(numeric) + | ~np.isin(numeric, sorted(_VALID_CODES)) + | current.map( + lambda value: isinstance(value, (bool, np.bool_)) + ).to_numpy() + ) + mismatch = present & ~invalid & (numeric != expected) + if invalid.any() or mismatch.any(): + bad = invalid | mismatch + rows = result.index[year_mask].to_numpy()[bad][:5].tolist() + raise ValueError( + f"ASEC reported-coverage existing {column} disagrees " + f"with the pinned sidecar in income year {year} at " + f"row(s): {rows}." + ) + else: + result[column] = np.nan + result.loc[year_mask, column] = expected + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + result[column] = pd.to_numeric(result[column], errors="raise").astype("int64") + result.attrs["reported_coverage_source_audit"] = source.attrs.get( + "source_audit", {} + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py new file mode 100644 index 000000000..1f38f18b1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py @@ -0,0 +1,31 @@ +"""Bind source CSV readers to the real stdlib builtin, including caller aliases.""" + +import _csv +import csv +from types import BuiltinFunctionType + +_NATIVE_CSV = _csv +_CSV = csv +_READER = _csv.reader + + +def csv_reader_bound(module) -> bool: + """Refuse live rebinding, including wrappers installed before this import.""" + return ( + type(_READER) is BuiltinFunctionType + and _READER.__module__ == "_csv" + and _READER.__name__ == "reader" + and _READER.__self__ is _NATIVE_CSV + and module is _CSV + and getattr(module, "reader", None) is _READER + and getattr(_NATIVE_CSV, "reader", None) is _READER + ) + + +def capture_csv_reader(module): + """Return the checked builtin itself, never a later mutable alias lookup.""" + return _READER if csv_reader_bound(module) else None + + +if not csv_reader_bound(csv): + raise ValueError("SOURCE_CSV_READER_CHANGED") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py new file mode 100644 index 000000000..5ac898f64 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py @@ -0,0 +1,217 @@ +"""Explicit S0101-only activation for the selected ACS/ASEC development path. + +A reviewed declaration pins two source responses. The caller's review and +guarded acquisition supply trust; this decoder is not independent attestation. +There is deliberately no default genuine declaration or mixed-inventory reader. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.graph.canonical import canonical_json + +from . import demographic_calibration_graph as demographic +from . import national_age_activation as age +from .survey_age_sources import _digest, read_survey_age_response, survey_age_requests + +PROFILE = "selected_survey_s0101_development_v1" +_CONVENTION = ( + "ACS 2024 observed interview ages and ASEC 2025 interview ages with income " + "year 2024; completed years used verbatim without temporal aging; calibration " + "to the published 2024 all-resident distribution does not establish that " + "the selected survey population covers that complete universe" +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_ACTIVATION_" + reason) + + +@dataclass(frozen=True, slots=True) +class SurveyAgeActivation: + """Exact source pins whose approval is a separate caller responsibility.""" + + metadata_sha256: str + data_sha256: str + + def __post_init__(self) -> None: + _digest(self.metadata_sha256) + _digest(self.data_sha256) + + +def activation_binding(declaration: SurveyAgeActivation) -> dict: + """Return the complete, detached semantic declaration, without source I/O.""" + _require(type(declaration) is SurveyAgeActivation, "DECLARATION") + declaration.__post_init__() + return { + "profile": PROFILE, + "requests": list(survey_age_requests()), + "metadata_sha256": declaration.metadata_sha256, + "data_sha256": declaration.data_sha256, + "period": "2024", + "entity": "household", + "universe": age.NATIONAL_AGE_ACTIVATION.universe, + "age_convention_id": "observed_interview_age_completed_years", + "age_convention": _CONVENTION, + "bands": [ + { + "variable": b.variable, + "label": b.label, + "low": b.low, + "high": b.high, + "column": b.column, + } + for b in age.NATIONAL_AGE_ACTIVATION.bands + ], + "published_total": "S0101_C01_001_consistency_only", + "consumes_se": False, + "release_eligible": False, + } + + +def declaration_from_binding(binding: object) -> SurveyAgeActivation: + """Validate value-level profile semantics; grant no source authorization.""" + _require(type(binding) is dict, "DECLARATION") + _require("metadata_sha256" in binding and "data_sha256" in binding, "DECLARATION") + declaration = SurveyAgeActivation( + binding["metadata_sha256"], binding["data_sha256"] + ) + _require( + canonical_json(binding) == canonical_json(activation_binding(declaration)), + "DECLARATION", + ) + return declaration + + +def activation_digest(declaration: SurveyAgeActivation) -> str: + """Bind source pins and all selected-survey conventions in one identifier.""" + return hashlib.sha256(canonical_json(activation_binding(declaration))).hexdigest() + + +def validate_survey_age_registry( + registry: TargetRegistry, binding: dict +) -> TargetRegistry: + """Check numerical profile metadata, not the truth of caller-supplied values.""" + declaration = declaration_from_binding(binding) + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + bands = age.NATIONAL_AGE_ACTIVATION.bands + _require(len(frozen) == len(bands), "REGISTRY") + digest = activation_digest(declaration) + for spec, band in zip(frozen, bands, strict=True): + _require( + spec.name == band.variable + and spec.measure == band.column + and spec.metadata["table"] == "S0101" + and spec.metadata["reference_sha256"] == digest + and spec.metadata["evidence_scope"] == "source_documented", + "REGISTRY", + ) + return frozen + + +def activate_survey_age_targets( + source_dir: str | Path, *, declaration: SurveyAgeActivation +) -> TargetRegistry: + """Derive exactly 18 targets from the two explicitly pinned local responses.""" + binding = activation_binding(declaration) + _, metadata = read_survey_age_response( + source_dir, "metadata", declaration.metadata_sha256 + ) + _, data = read_survey_age_response(source_dir, "data", declaration.data_sha256) + # Reuse the established publisher/count/MOE interpretation without invoking + # its old inventory loader. This object supplies only pure field semantics. + semantics = replace(age.NATIONAL_AGE_ACTIVATION, age_convention=_CONVENTION) + row = age._national_row(data, semantics) + _require( + all( + name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_") + for name in row + ), + "RESPONSE_SCOPE", + ) + _require( + type(metadata) is dict and type(metadata.get("variables")) is dict, "METADATA" + ) + _require( + all( + type(name) is str + and (name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_")) + for name in metadata["variables"] + ), + "METADATA_SCOPE", + ) + + def cell(variable): + _require( + all(variable + suffix in row for suffix in ("E", "EA", "M", "MA")), "CELL" + ) + return { + "estimate": age.classify_acs_value( + row[variable + "E"], row[variable + "EA"] + ), + "moe": age.classify_acs_value(row[variable + "M"], row[variable + "MA"]), + } + + digest = activation_digest(declaration) + specs = [] + data_url = survey_age_requests()[1]["url"] + for band in semantics.bands: + published = age._published_variable(metadata, band.variable + "E", semantics) + _require( + published.get("label") == band.label + and published.get("predicateType") == "int", + "PUBLISHED_DEFINITION", + ) + derived = cell(band.variable) + specs.append( + TargetSpec( + name=band.variable, + entity="household", + measure=band.column, + value=float(age._count_estimate(derived, band.variable)), + period="2024", + se=age._standard_error(derived, band.variable, confidence_level=0.9), + source=f"{data_url} (S0101 {band.variable}E)", + family="acs.S0101", + notes=f"activation={digest}; label={band.label}; age_convention={_CONVENTION}; published 90% MOE divided by {age._MOE_90_TO_SE}, not consumed by current loss", + metadata={ + "table": "S0101", + "reference_sha256": digest, + "geography": "0100000US", + "universe": "population", + "role": "calibration", + "evidence_scope": "source_documented", + }, + ) + ) + total = age._published_variable(metadata, "S0101_C01_001E", semantics) + _require( + total.get("label") == "Estimate!!Total!!Total population" + and total.get("predicateType") == "int", + "PUBLISHED_TOTAL_DEFINITION", + ) + _require( + sum(int(s.value) for s in specs) + == age._count_estimate(cell("S0101_C01_001"), "S0101_C01_001"), + "PARTITION", + ) + return validate_survey_age_registry(TargetRegistry(specs, country="us"), binding) + + +def verify_survey_age_targets( + source_dir: str | Path, + *, + declaration: SurveyAgeActivation, + registry: TargetRegistry, +) -> None: + """Rederive the pinned target values; no stored registry substitutes for this.""" + actual = activate_survey_age_targets(source_dir, declaration=declaration) + _require( + demographic._registry_json(actual) == demographic._registry_json(registry), + "REGISTRY", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py new file mode 100644 index 000000000..49a6510b2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -0,0 +1,773 @@ +"""Country admission for two explicit seven-node survey age profiles. + +Every call reconstructs the accepted source preparation. The numerical kernels +are values-only; source, full-population, group/row-cap and cache checks live +here. The original entry point remains invented-only; the named development +entry point derives its targets from an explicitly pinned S0101-only capture. +Both permit only the first IMPORTANCE-to-CALIBRATED transition and no release. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass, replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.frame import Frame +from microcosm.graph import ( + CompiledGraph, + KernelRegistry, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.canonical import canonical_json +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _project_context, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + seed, + weights_key, +) +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) +from microcosm.graph.store import _verified_meta + +from . import graph_combined_clone as clone +from . import graph_survey_age_artifact as ages +from . import graph_survey_budget as transport +from . import graph_survey_calibration as numerical +from . import graph_survey_population as source_graph +from . import survey_age_activation as age_activation +from . import survey_calibration_diagnostics as diagnostic_check +from . import survey_origin_budget as budgets +from . import survey_population_replay as replay + +COUNT_NODE = "survey.age_count_matrix" + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_RUN_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _node_identity(node): + """Immutable snapshot excluding execution time and cache-hit bookkeeping.""" + return canonical_json( + { + "key": node.key, + "ref": node.kernel_ref, + "implementation": node.kernel_impl_hash, + "capabilities": _capabilities_projection(node.capabilities), + "typed_artifacts": node.typed_artifacts, + "seed": node.seed, + "frame_key": node.frame_key, + "weight_key": node.weight_key, + "artifacts": [ + [entity, column, key] + for (entity, column), key in sorted(node.artifacts.items()) + ], + "opaque_artifacts": node.opaque_artifacts, + "receipt": node.receipt, + "legacy_capabilities": node.legacy_capabilities, + } + ) + + +def _measurement(population): + node = ages.survey_age_count_artifact_node( + population=population.version, node_id=COUNT_NODE + ) + context = _project_context( + node, + population, + key="0" * 64, + sources={}, + tolerances={}, + numerics={}, + ) + measured = ages.SurveyAgeCountArtifactKernel().run(context) + payload = measured.artifacts["counts"] + values = ages.decode_survey_age_counts(payload) + _require( + tuple(values.household_ids) + == tuple(population.frame.table("household").household_id), + "MEASUREMENT_HOUSEHOLD_ORDER", + ) + return payload, json.loads(canonical_json(measured.receipt)) + + +def _measure(population): + return _measurement(population)[0] + + +def _load_diagnostics(store, key): + """Bound the diagnostic payload buffer before reading it. + + The existing store verifier still authenticates metadata and streams its + payload hashes. This does not claim a bound on that verifier's metadata + decoding or total I/O. Only the selected diagnostic payload is buffered here. + """ + path = store.object_path(key) + payload_path = path / "payload.bin" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + fd = os.open(payload_path, flags) + try: + before = os.fstat(fd) + limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, + "DIAGNOSTIC_STORE_LIMIT", + ) + metadata = _verified_meta(path, expected_kind="bytes") + _require( + set(metadata["payloads"]) == {"payload.bin"}, "DIAGNOSTIC_STORE_ROSTER" + ) + record = metadata["payloads"]["payload.bin"] + _require(record["size"] == before.st_size, "DIAGNOSTIC_STORE_SIZE") + with os.fdopen(fd, "rb", closefd=False) as stream: + payload = stream.read(limit + 1) + after = os.fstat(fd) + current = os.stat(payload_path, follow_symlinks=False) + + def seal(s): + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + + _require( + seal(before) == seal(after) == seal(current) + and stat.S_ISREG(current.st_mode) + and len(payload) == record["size"] <= limit + and _sha(payload) == record["sha256"], + "DIAGNOSTIC_STORE_CHANGED", + ) + return payload + finally: + os.close(fd) + + +def _expected_nodes( + compiled: CompiledGraph, + kernels: KernelRegistry, + keys: dict[str, str], + implementations: dict[str, str], + prefix_identities: dict[str, bytes], + frame: Frame, +): + """Detach declaration expectations before any graph kernel/store execution.""" + states = {} + cells = tuple((e, str(c)) for e in frame.entities for c in frame.table(e)) + for node in compiled.graph.nodes: + if node.id in prefix_identities: + state = json.loads(prefix_identities[node.id]) + state["artifacts"] = {(e, c): k for e, c, k in state["artifacts"]} + # The accepted prefix runner already independently sealed this + # receipt. No receipt from the new seven-node run is trusted here. + states[node.id] = state + continue + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + outputs = ( + cells if structural else tuple((o.entity, o.column) for o in node.outputs) + ) + states[node.id] = { + "key": key, + "ref": node.kernel, + "implementation": implementations[node.id], + "capabilities": json.loads( + canonical_json( + _capabilities_projection(kernels.get(node.kernel).capabilities) + ) + ), + "typed_artifacts": json.loads( + canonical_json(typed_contracts(compiled, node, keys, kernels)) + ), + "seed": seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, node.weights.entity) + if node.weights + else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in outputs}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + } + return states + + +def _complete_receipts( + compiled: CompiledGraph, + states: dict[str, dict[str, object]], + authored: dict[str, dict[str, object]], + observed: dict[str, Population], +): + """Normalize independently authored receipts through the executor's rules.""" + previous = {} + for node_id in compiled.order: + state = states[node_id] + if node_id in authored: + node = compiled.graph.node(node_id) + receipt = json.loads(canonical_json(authored[node_id])) + receipt["capabilities"] = dict(state["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=previous) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = mass_record_receipt(observed[node_id].mass_ledger[-1]) + receipt.update(weight_cap_receipt(observed[node_id], node)) + state["receipt"] = json.loads(canonical_json(receipt)) + previous[node_id] = SimpleNamespace(receipt=state["receipt"]) + + +def _check_manifest(manifest, compiled, states): + _require( + manifest.country == compiled.graph.country and manifest.decisions == (), + "MANIFEST_COUNTRY_DECISIONS", + ) + _require( + set(manifest.populations) == set(compiled.versions.values()) + and set(manifest.mass_ledgers) == set(compiled.versions.values()), + "MANIFEST_POPULATION_VERSIONS", + ) + # Detached expectations use JSON arrays. NodeReceipt freezes those arrays + # as tuples, including nested capabilities and typed artifact descriptors. + # Match its representation without weakening the canonical value check. + frozen_states = { + node_id: { + **state, + "receipt": _freeze_json(state["receipt"]), + "typed_artifacts": _freeze_json(state["typed_artifacts"]), + } + for node_id, state in states.items() + } + source_graph._check_node_states(manifest, frozen_states) + for node_id, state in states.items(): + expected = {**state, "legacy_capabilities": False} + expected["artifacts"] = [ + [e, c, key] for (e, c), key in sorted(state["artifacts"].items()) + ] + _require( + _node_identity(manifest.node(node_id)) == canonical_json(expected), + "MANIFEST_CANONICAL_VALUES", + ) + + +@dataclass(frozen=True) +class SurveyAgeCalibrationRun: + """Values validated at return; retained source handles still require rechecks.""" + + manifest: object + compiled: object + budget: object + successor: object + diagnostics: dict + counts_sha256: str + numeric_bounds_sha256: str + release_eligible: bool = False + + +def run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + target_registry, + epochs, + learning_rate, + resume="auto", +): + """Reconstruct sources and admit a complete invented calibration graph. + + The source and store paths are the caller's explicitly approved local + inputs. Cache reuse never bypasses fresh source preparation or the complete + receiving-population and sampling-reference checks. This v1 does not admit + genuine targets, donor-detail descendants, pruning or repeated calibration. + """ + # Refuse unsupported targets/options before any source I/O. + calibration = numerical.survey_age_calibration_node( + target_registry, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + ) + + +def run_survey_age_development( + source_dir, + *, + age_source_dir, + activation, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume="auto", +): + """Run development calibration after deriving targets from exact source pins. + + The caller separately reviews the declaration and explicitly admits these + local source paths. There is no default capture, caller-supplied registry, + mixed-inventory fallback, production attestation or release permission. + """ + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + binding = age_activation.activation_binding(activation) + registry = age_activation.activate_survey_age_targets( + age_source_dir, declaration=activation + ) + calibration = numerical.survey_age_development_node( + registry, + activation_binding=binding, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + age_source_dir=age_source_dir, + activation=activation, + ) + + +def _run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume, + calibration, + age_source_dir=None, + activation=None, +): + """Shared population, budget, replay, diagnostic and final-return checks.""" + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + activation_identity = None + calibration_binding = numerical.calibration_activation_binding(calibration.params) + _require( + (calibration_binding is None) == (activation is None), "ACTIVATION_REQUIRED" + ) + if activation is not None: + activation_identity = canonical_json( + age_activation.activation_binding(activation) + ) + _require( + canonical_json(calibration_binding) == activation_identity, + "ACTIVATION_BINDING", + ) + prefix = source_graph.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + clones=True, + return_values=True, + ) + initial = prefix.clone_population + _require(initial is not None, "COMPLETE_CLONE_REQUIRED") + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=prefix.allocated_population, + clone_population=initial, + ) + budget_view = budget.checked_view() + budget_payload = budget_view.payload + numeric_payload = transport.numeric_survey_budget_payload(budget_payload) + numeric_bounds = numerical.decode_numeric_survey_bounds(numeric_payload) + count_payload, count_receipt = _measurement(initial) + _require( + numeric_bounds.grouped.household_ids + == tuple(initial.frame.table("household").household_id), + "BUDGET_HOUSEHOLD_ORDER", + ) + prefix_identities = { + name: _node_identity(row) for name, row in prefix.manifest.nodes.items() + } + initial_identity = budgets._population_identity(initial) + allocation_identity = budgets._population_identity(prefix.allocated_population) + prefix_budget = budget + prefix_budget_entry = budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + kernels = prefix.kernels + kernels.register(transport.SurveySamplingBudgetKernel(budget_payload)) + kernels.register(ages.SurveyAgeCountArtifactKernel()) + kernels.register(numerical.SurveyAgeCalibrationKernel()) + graph = replace( + prefix.compiled.graph, + nodes=( + *prefix.compiled.graph.nodes, + transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), + ages.survey_age_count_artifact_node( + population=initial.version, node_id=COUNT_NODE + ), + calibration, + ), + ) + compiled = compile_graph(graph) + _require(len(compiled.order) == 7, "EXACT_GRAPH") + _require( + clone.COMBINED_CLONE_CLAIM_NODE in compiled.predecessors[transport.BUDGET_NODE], + "OWNERSHIP_CLAIM_EDGE", + ) + _paths, source_keys = _source_paths_and_keys(compiled, prefix.sources, prefix.store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = _expected_nodes( + compiled, kernels, keys, implementations, prefix_identities, initial.frame + ) + observed, snapshots = {}, {} + admitted = None + successor_entry = None + successor_payload = None + receiving_initial = None + receiving_budget_entry = None + + def observe(node_id, population): + nonlocal admitted, budget, receiving_initial, receiving_budget_entry + nonlocal successor_entry, successor_payload + _require(node_id not in observed, "DUPLICATE_OBSERVATION") + if node_id == numerical.CALIBRATION_NODE: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + # Independent of the numerical kernel, on both execution and load. + numerical.check_numeric_survey_weights( + numeric_bounds, population.frame.weights_for("household").values + ) + admitted = budgets.admit_survey_weight_only_population( + budget, + previous=receiving_initial, + current=population, + ) + successor_entry = budgets._entry(admitted, budgets.SamplingOriginSuccessor) + successor_payload = admitted.payload + elif node_id in {transport.BUDGET_NODE, COUNT_NODE}: + # Count may precede the ownership claim; it is a read-only artifact. + # A named replay check does not pretend it shares the claim owners. + replay.same_replayed_frame(initial.frame, population.frame) + _require( + population.frame.weights_for("household").values.tobytes() + == numeric_bounds.incoming.tobytes(), + "RECEIVING_INCOMING_WEIGHTS", + ) + if node_id == transport.BUDGET_NODE: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + _require( + budgets._population_identity(population) + == snapshots[clone.COMBINED_CLONE_CLAIM_NODE], + "COMPLETE_CLAIMED_CLONE", + ) + else: + _require(_measure(population) == count_payload, "REMEASURED_COUNTS") + elif node_id == source_graph.ALLOCATION_NODE: + replay.same_replayed_population(prefix.allocated_population, population) + elif node_id == clone.COMBINED_CLONE_CLAIM_NODE: + replay.same_replayed_population(initial, population) + _require(source_graph.ALLOCATION_NODE in observed, "ALLOCATION_REQUIRED") + # Actual source reconstruction rebinds an issued budget to this run's + # receiving objects. Candidate bytes grant no authority by themselves. + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=observed[source_graph.ALLOCATION_NODE], + clone_population=population, + candidate=budget_payload, + ) + receiving_budget_entry = budgets._entry( + budget, budgets.SamplingOriginBudget + ) + receiving_initial = population + elif node_id == source_graph.CREATE_NODE: + replay.same_replayed_frame( + prefix.manifest.population(node_id), population.frame + ) + elif node_id == clone.COMBINED_CLONE_NODE: + actual_allocation = observed[source_graph.ALLOCATION_NODE] + source_graph._verify_cloned_frame( + actual_allocation.frame, + population.frame, + actual_allocation.design_weights["household"], + ) + replay.same_replayed_frame(initial.frame, population.frame) + else: + _require(False, "UNEXPECTED_NODE") + observed[node_id] = population + snapshots[node_id] = budgets._population_identity(population) + + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require( + tuple(observed) == compiled.order and admitted is not None, + "OBSERVATION_COVERAGE", + ) + _require(set(manifest.nodes) == set(compiled.order), "MANIFEST_NODES") + for node in graph.nodes: + actual = manifest.node(node.id) + _require( + actual.key == keys[node.id] + and actual.kernel_ref == node.kernel + and actual.kernel_impl_hash == implementations[node.id] + and actual.typed_artifacts == expected_nodes[node.id]["typed_artifacts"] + and actual.seed == seed(keys[node.id]) + and actual.legacy_capabilities is False, + "MANIFEST_NODE_IDENTITY", + ) + if node.id in prefix_identities: + _require( + _node_identity(actual) == prefix_identities[node.id], + "PREFIX_RECEIPT_REPLAY", + ) + for node_id, name, type_, payload, capabilities in ( + ( + transport.BUDGET_NODE, + "budget", + budgets.BUDGET_TYPE, + budget_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + transport.BUDGET_NODE, + "numeric_bounds", + numerical.BOUNDS_TYPE, + numeric_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + COUNT_NODE, + "counts", + ages.COUNTS_TYPE, + count_payload, + ages.SurveyAgeCountArtifactKernel.capabilities, + ), + ): + source_graph._final_artifact( + manifest, + prefix.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=capabilities, + ) + receipt = manifest.node(numerical.CALIBRATION_NODE) + diagnostic_payload = _load_diagnostics( + prefix.store, + expected_nodes[numerical.CALIBRATION_NODE]["opaque_artifacts"]["diagnostics"], + ) + current = observed[numerical.CALIBRATION_NODE] + anchors = { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "counts_sha256": _sha(count_payload), + "accepted_weight_sha256": _sha( + current.frame.weights_for("household").values.tobytes() + ), + "constraint_digest": numeric_bounds.grouped.digest, + "weight_anchor": calibration.params["weight_anchor"], + "cap_enforcement": calibration.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(numeric_bounds.incoming == 0)), + } + _require( + _sha(diagnostic_payload) == receipt.receipt.get("diagnostics_sha256"), + "DIAGNOSTICS_DIGEST", + ) + _require( + all(receipt.receipt.get(k) == v for k, v in anchors.items()), "RECEIPT_ANCHORS" + ) + registry = numerical.demographic._registry_from_json(calibration.params["registry"]) + diagnostics = diagnostic_check.validate_survey_calibration_diagnostics( + diagnostic_payload, + counts_payload=count_payload, + bounds_payload=numeric_payload, + weights=current.frame.weights_for("household").values, + registry=registry, + epochs=epochs, + learning_rate=learning_rate, + anchors=anchors, + activation_binding=calibration_binding, + ) + diagnostic_identity = canonical_json(diagnostics) + _complete_receipts( + compiled, + expected_nodes, + { + transport.BUDGET_NODE: { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "constraint_digest": numeric_bounds.grouped.digest, + "group_count": numeric_bounds.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + COUNT_NODE: count_receipt, + numerical.CALIBRATION_NODE: { + **anchors, + "diagnostics_sha256": _sha(diagnostic_payload), + "scope": numerical.calibration_receipt_scope(calibration.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + }, + observed, + ) + _check_manifest(manifest, compiled, expected_nodes) + result = SurveyAgeCalibrationRun( + manifest, + compiled, + budget, + admitted, + diagnostics, + _sha(count_payload), + _sha(numeric_payload), + ) + final_numbers = numerical.decode_numeric_survey_bounds(numeric_payload) + # A population version can be shared by several nonstructural nodes. Check + # the actual last receiving state for every version, including the prefix. + terminal_nodes = tuple({compiled.versions[n]: n for n in compiled.order}.values()) + terminal_frames = { + node_id: manifest.population(compiled.versions[node_id]) + for node_id in terminal_nodes + } + terminal_ledgers = { + node_id: manifest.mass_ledger(compiled.versions[node_id]) + for node_id in terminal_nodes + } + # Finish storage and key/implementation I/O before final source/target + # revalidation and complete-population checks. No store read follows. + final_keys, final_implementations = _all_node_keys(compiled, kernels, source_keys) + _require( + final_keys == keys and final_implementations == implementations, + "FINAL_IMPLEMENTATIONS", + ) + _require(_measure(initial) == count_payload, "FINAL_REMEASUREMENT") + _require( + _measure(receiving_initial) == count_payload, "FINAL_RECEIVING_MEASUREMENT" + ) + # Retain both authorities; the original source/planned budget is not waived + # when a candidate-identical handle is issued for the actual receiving run. + budgets.verify_survey_origin_budget(prefix_budget) + budgets.verify_survey_weight_only_successor(admitted) + if activation is not None: + # This is the last external evidence read. The existing complete + # population, manifest and retained budget seals all follow it. + age_activation.verify_survey_age_targets( + age_source_dir, declaration=activation, registry=registry + ) + _require(budget.payload == budget_payload, "FINAL_BUDGET_BYTES") + _require( + budgets._population_identity(initial) == initial_identity + and budgets._population_identity(prefix.allocated_population) + == allocation_identity, + "FINAL_PREFIX_POPULATIONS", + ) + for node_id, population in observed.items(): + _require( + budgets._population_identity(population) == snapshots[node_id], + "FINAL_POPULATION_STATE", + ) + # A nonstructural node can precede the claim within the same version; + # its own snapshot is checked above. Terminal version state is checked + # on the declared weight transition and complete ownership claim. + if node_id in terminal_nodes: + replay.same_replayed_frame(population.frame, terminal_frames[node_id]) + _require( + terminal_ledgers[node_id] == population.mass_ledger, + "FINAL_MASS_LEDGER", + ) + numerical.check_numeric_survey_weights( + final_numbers, + current.frame.weights_for("household").values, + ) + if activation is not None: + _require( + canonical_json(age_activation.activation_binding(activation)) + == activation_identity + and calibration.params.get("activation") == activation_identity.decode(), + "FINAL_ACTIVATION_BINDING", + ) + # No decoding, store access, source borrow or implementation helper follows + # this seal. It covers all seven receipts/descriptors, including scope and + # release flags, using expectations detached before execution. + _check_manifest(manifest, compiled, expected_nodes) + _require( + canonical_json(result.diagnostics) == diagnostic_identity, + "FINAL_DIAGNOSTICS_VALUES", + ) + # The verifications above perform I/O. Finish with the retained issuance + # entries, so the second verification cannot invalidate the first handle. + _require( + budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + is prefix_budget_entry + and budgets._entry(budget, budgets.SamplingOriginBudget) + is receiving_budget_entry + and prefix_budget.payload == budget.payload == budget_payload, + "FINAL_BUDGET_ISSUANCE", + ) + _require( + budgets._entry(admitted, budgets.SamplingOriginSuccessor) is successor_entry + and admitted.payload == successor_payload, + "FINAL_SUCCESSOR_ISSUANCE", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py new file mode 100644 index 000000000..d4314af93 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py @@ -0,0 +1,181 @@ +"""Two explicit S0101-only source captures; no network or target activation. + +Callers supply already acquired response bytes. Reading an explicit capture +never discovers files or falls back to a mixed national reference inventory. +""" + +from __future__ import annotations + +import hashlib +import os +import stat +from pathlib import Path +from urllib.parse import urlencode + +from microcosm.graph.canonical import canonical_json + +from .cd_reference_sources import strict_json + +MAX_RESPONSE_BYTES = 8 * 1024**2 +MAX_DESCRIPTOR_BYTES = 4096 +_DESCRIPTOR_FIELDS = frozenset( + { + "table", + "year", + "dataset", + "geography", + "kind", + "url", + "sha256", + "size_bytes", + "path", + } +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_SOURCE_" + reason) + + +def survey_age_requests() -> tuple[dict[str, object], dict[str, object]]: + """Return detached descriptors of exactly two supported national requests.""" + base = "https://api.census.gov/data/2024/acs/acs1/subject" + common = { + "table": "S0101", + "year": 2024, + "dataset": "acs/acs1/subject", + "geography": "us:*", + } + return ( + {**common, "kind": "metadata", "url": f"{base}/groups/S0101.json"}, + { + **common, + "kind": "data", + "url": base + "?" + urlencode({"get": "group(S0101)", "for": "us:*"}), + }, + ) + + +def _request(value: object) -> dict[str, object]: + _require(type(value) is dict, "REQUEST") + for request in survey_age_requests(): + if set(value) == set(request) and all( + type(value[k]) is type(v) and value[k] == v for k, v in request.items() + ): + return request + _require(False, "REQUEST") + + +def _digest(value: object) -> str: + _require( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value), + "PIN", + ) + return value + + +def _path(value: str | Path) -> Path: + path = Path(value) + _require(".." not in path.parts, "PATH") + path = path.absolute() + for member in (*reversed(path.parents), path): + _require(not member.is_symlink(), "SYMLINK") + return path + + +def _read(path: Path, limit: int) -> bytes: + path = _path(path) + try: + # A FIFO must reach the regular-file check instead of waiting for a + # writer before fstat can reject it. Nonblocking leaves regular reads unchanged. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE") + _require(0 < before.st_size <= limit, "SIZE") + payload = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + _require( + len(payload) == before.st_size + and (before.st_size, before.st_mtime_ns, before.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "CHANGED_FILE", + ) + return payload + except OSError: + raise ValueError("SURVEY_AGE_SOURCE_FILE") from None + + +def _write(path: Path, payload: bytes, limit: int) -> None: + path = _path(path) + if path.exists(): + _require(_read(path, limit) == payload, "IMMUTABLE") + return + path.parent.mkdir(parents=True, exist_ok=True) + _path(path) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_survey_age_response( + root: str | Path, request: dict, payload: bytes +) -> dict: + """Store one caller-acquired response immutably; grant no target permission.""" + request = _request(request) + _require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, "SIZE") + strict_json(payload) + root = _path(root) + digest = hashlib.sha256(payload).hexdigest() + descriptor = { + **request, + "sha256": digest, + "size_bytes": len(payload), + "path": f"raw/{digest}.json", + } + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{key}.json" + encoded = canonical_json(descriptor) + # Refuse replacement before writing even an unreferenced new raw file. + if _path(descriptor_path).exists(): + _require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, "IMMUTABLE") + _write(root / descriptor["path"], payload, MAX_RESPONSE_BYTES) + _write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES) + return descriptor + + +def read_survey_age_response( + root: str | Path, kind: str, expected_sha256: str +) -> tuple[bytes, object]: + """Read the fixed named response against a separately reviewed content pin.""" + _require(type(kind) is str and kind in {"metadata", "data"}, "KIND") + expected_sha256 = _digest(expected_sha256) + request = next(row for row in survey_age_requests() if row["kind"] == kind) + root = _path(root) + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor = strict_json( + _read(root / "requests" / f"{key}.json", MAX_DESCRIPTOR_BYTES) + ) + _require( + type(descriptor) is dict and set(descriptor) == _DESCRIPTOR_FIELDS, "DESCRIPTOR" + ) + _request({key: descriptor[key] for key in request}) + _require( + descriptor["sha256"] == expected_sha256 + and descriptor["path"] == f"raw/{expected_sha256}.json", + "PIN", + ) + _require( + type(descriptor["size_bytes"]) is int + and 0 < descriptor["size_bytes"] <= MAX_RESPONSE_BYTES, + "SIZE", + ) + payload = _read(root / f"raw/{expected_sha256}.json", MAX_RESPONSE_BYTES) + _require( + len(payload) == descriptor["size_bytes"] + and hashlib.sha256(payload).hexdigest() == expected_sha256, + "RESPONSE_PIN", + ) + return payload, strict_json(payload) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py new file mode 100644 index 000000000..28caa0f09 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -0,0 +1,196 @@ +"""Recompute supported age diagnostics without rerunning an optimizer. + +This is a value checker, not source authority. The caller supplies independently +verified ordered counts, sampling bounds and receiving weights. Optimizer history +has a bounded schema but cannot be reconstructed from those final values. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.calibrate import diagnostics_payload +from microcosm.calibrate.score import score_targets +from microcosm.frame import WeightKind, Weights +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_calibration as numeric + +MAX_DIAGNOSTIC_BYTES = 8 * 1024**2 +HISTORY_FIELDS = ( + "initial_loss", + "loss_trajectory", + "options.matrix_format", + "options.grouped_upper_bounds.last_corrected_group_count", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_DIAGNOSTICS_" + reason) + + +def _history(document, epochs, group_count): + options = document.get("options") + _require(type(options) is dict, "OPTIONS") + grouped = options.get("grouped_upper_bounds") + _require(type(grouped) is dict, "GROUPED_OPTIONS") + corrected = grouped.get("last_corrected_group_count") + _require(type(corrected) is int and 0 <= corrected <= group_count, "HISTORY_COUNT") + layout = options.get("matrix_format") + _require( + type(layout) is str and layout in {"dense", "sparse_csr"}, "HISTORY_LAYOUT" + ) + trajectory = document.get("loss_trajectory") + _require(type(trajectory) is list and len(trajectory) == epochs, "HISTORY_LENGTH") + _require( + all(type(v) is float and np.isfinite(v) and 0 <= v <= 10 for v in trajectory), + "HISTORY_LOSS", + ) + _require( + type(document.get("initial_loss")) is float + and document["initial_loss"] == trajectory[0], + "HISTORY_INITIAL", + ) + return corrected, layout, trajectory + + +def validate_survey_calibration_diagnostics( + payload, + *, + counts_payload, + bounds_payload, + weights, + registry, + epochs, + learning_rate, + anchors, + activation_binding=None, +): + """Return defensive schema-6 values plus an explicit verification annotation. + + Every field except the four named history fields is reconstructed using + score_targets and the package's diagnostics encoder. Only warning-free + schema 6 with target-loss attribution is supported in this first profile. + No convergence, optimality, target eligibility or release claim is granted. + """ + _require( + type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, + "PAYLOAD_LIMIT", + ) + try: + document = json.loads(payload) + except (ValueError, UnicodeError, RecursionError): + raise ValueError("SURVEY_DIAGNOSTICS_JSON") from None + _require(type(document) is dict, "DOCUMENT") + _require(canonical_json(document) == payload, "CANONICAL") + # Reuse the actual declaration's option/registry restrictions. This is a + # declaration and numeric work Frame, never a population/source substitute. + factory = numeric.survey_age_calibration_node + profile_options = {} + if activation_binding is not None: + factory = numeric.survey_age_development_node + profile_options["activation_binding"] = activation_binding + factory( + registry, + **profile_options, + base="diagnostic-values", + budget_node="bounds", + count_node="counts", + epochs=epochs, + learning_rate=learning_rate, + ) + bounds = numeric.decode_numeric_survey_bounds(bounds_payload) + counts = numeric.ages.decode_survey_age_counts(counts_payload) + _require( + counts.population == bounds.population + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "ORDERED_POPULATION", + ) + numeric.check_numeric_survey_weights(bounds, weights) + accepted_weights = weights.tobytes() + corrected, layout, trajectory = _history( + document, epochs, bounds.grouped.group_count + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + work = numeric.demographic.kernels_module._frame_from_context( + SimpleNamespace( + tables={"household": table}, + weights={ + "household": Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + }, + ), + "household", + ) + targets = registry.to_target_set() + initial = score_targets( + work, targets, weights=bounds.incoming, target_loss_cap=10.0 + ) + final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) + _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") + options = { + "grouped_preserve_zeros": { + "enabled": True, + "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), + "ordered_zero_mask_sha256": hashlib.sha256( + np.asarray(bounds.incoming == 0, dtype=np.uint8).tobytes() + ).hexdigest(), + }, + "grouped_upper_bounds": bounds.grouped.diagnostics(weights, corrected), + "method": "adam", + "epochs": epochs, + "learning_rate": learning_rate, + "mass": "free", + "mass_reason": None, + "max_weight_ratio": None, + "target_records": None, + "l1_lambda": 0.0, + "l1_penalty": "mean_initial_weight_ratio_abs", + "l2_lambda": 0.0, + "l2_anchor": "initial", + "l2_anchor_weights_supplied": False, + "l2_penalty": "mean_initial_pre_gate_weight_ratio_squared", + "seed": 0, + "target_loss_weights": final.options["target_loss_weights"], + "target_loss_scales": final.options["target_loss_scales"], + "warm_start_weights": {"enabled": False, "kind": None}, + "matrix_format": layout, + } + combined = replace( + final, + diagnostics=tuple( + replace(after, initial_estimate=before.final_estimate) + for before, after in zip( + initial.diagnostics, final.diagnostics, strict=True + ) + ), + options=options, + loss_trajectory=np.asarray(trajectory, dtype=np.float64), + ) + expected = diagnostics_payload(combined, target_registry=registry, build=anchors) + _require( + expected.get("schema_version") == 6 + and expected.get("diagnostic_warnings") == [] + and "target_loss_basis" in expected, + "UNSUPPORTED_DIAGNOSTIC_SCHEMA", + ) + _require(canonical_json(expected) == payload, "RECOMPUTED_VALUES") + # Make the distinction visible to downstream callers without altering the + # stored schema-6 evidence or its digest. + document["verification"] = { + "protocol": "microcosm.us.survey-age-diagnostics-verification.v1", + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "recomputed": "all artifact fields except the listed optimizer history", + "optimizer_history_schema_checked_only": list(HISTORY_FIELDS), + "optimizer_rerun": False, + "release_eligible": False, + } + numeric.check_numeric_survey_weights(bounds, weights) + _require(weights.tobytes() == accepted_weights, "FINAL_WEIGHT_BYTES") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py new file mode 100644 index 000000000..933e86851 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py @@ -0,0 +1,259 @@ +"""Plan one draw over complete supplied survey records, before native Frames. + +This is declaration/arithmetic only. Closed source owners must authenticate the +complete inputs and bind the result to the selected population. This module +reads no source, creates no Frame and never totals publisher weights. Original +anchors remain separate from the one-time share/inclusion multiplier. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction + +from microcosm.build.survey_domain_sample import select_domain_households + +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.us.survey-catalogue-selection.v1" +_BATCH_HOUSEHOLDS = 10_000 +_BATCH_PEOPLE = 100_000 + + +class CatalogueSelectionError(ValueError): + """Static refusal, without raw source identities or observations.""" + + +def _require(condition, code): + if not condition: + raise CatalogueSelectionError(code) + + +@dataclass(frozen=True, slots=True) +class SelectedHousehold: + key: domains.HouseholdKey + domain: domains.Domain + statistical_unit: str + original_design_weight: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_multiplier(self) -> Fraction: + return self.share / self.inclusion_probability + + +@dataclass(frozen=True, slots=True) +class ExcludedHousehold: + key: domains.HouseholdKey + reason: str + + +@dataclass(frozen=True, slots=True) +class SelectionCell: + source: domains.Source + domain: domains.Domain + statistical_unit: str + share: Fraction + eligible_households: int + selected_households: int + inclusion_probability: Fraction | None + + +@dataclass(frozen=True, slots=True) +class CatalogueSelectionPlan: + selected: tuple[SelectedHousehold, ...] + excluded: tuple[ExcludedHousehold, ...] + cells: tuple[SelectionCell, ...] + supplied_households: int + fraction: Fraction + seed: int + + @property + def source_authenticated(self) -> bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + +def _sampling_key(key): + # Keep the original H_SEQ token in the result, but order ASEC keys + # numerically. All validated values fit five digits; padded strings allow + # both survey channels to use the shared selector's one literal-ID axis. + return ( + key.native_id + if key.source is domains.Source.ACS + else f"{int(key.native_id):05d}" + ) + + +def _ordered_key(key): + return key.source.value, _sampling_key(key) + + +def _original_anchor(decision): + row = decision.original + if row.key.source is domains.Source.ASEC: + return Fraction(int(row.hsup_wgt), 100) + if decision.statistical_unit == "person": + return Fraction(int(row.persons[0].pwgtp)) + return Fraction(int(row.wgtp)) + + +def plan_catalogue_selection(*, acs_households, asec_households, fraction, seed): + """Classify all supplied records and draw once in each positive-share cell. + + ``N`` counts complete eligible households in the supplied catalogue, never + selected Frame rows. Unknown classifications refuse before any draw, even + if a tentative share is zero. Known structural exclusions have a separate + ledger. Original zero ASEC anchors remain eligible; an all-zero selected + cell refuses without redrawing. No common-total normalization is performed. + + Full source completeness, immutable source authority, and original files + are obligations of the calling source orchestration, not this value API. + """ + _require( + type(acs_households) is tuple and type(asec_households) is tuple, + "CATALOGUE_TUPLES", + ) + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(_BATCH_HOUSEHOLDS) is int + and 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS) + and type(_BATCH_PEOPLE) is int + and 0 < _BATCH_PEOPLE <= min(100_000, domains.MAX_TOTAL_MEMBERS), + "BATCH_LIMITS", + ) + rows, exclusions = [], [] + household_keys, person_keys = set(), set() + batch, batch_people = [], 0 + + def consume(): + for decision in domains.classify_households(tuple(batch)): + _require( + decision.status is not domains.Status.REVIEW_REQUIRED, + "SOURCE_CLASSIFICATION_REVIEW_REQUIRED", + ) + if decision.status is domains.Status.EXCLUDED: + _require(decision.share == 0, "EXCLUSION_SHARE") + exclusions.append( + ExcludedHousehold(decision.original.key, decision.reason) + ) + else: + _require( + decision.status is domains.Status.ELIGIBLE + and decision.share is not None + and decision.share > 0 + and decision.domain is not None, + "ELIGIBLE_CLASSIFICATION", + ) + # Retain only the compact household decision, not another full + # population of per-person decisions and observation objects. + rows.append( + ( + decision.original.key, + decision.domain, + decision.statistical_unit, + _original_anchor(decision), + decision.share, + ) + ) + + for source, expected, catalogue in ( + (domains.Source.ACS, domains.AcsHousehold, acs_households), + (domains.Source.ASEC, domains.AsecHousehold, asec_households), + ): + for row in catalogue: + _require(type(row) is expected, "CATALOGUE_SOURCE_TYPE") + canonical = domains._key(row.key) + _require( + canonical[0] is source + and row.key.source_year == 2024 + and row.key.survey_year + == (2024 if source is domains.Source.ACS else 2025), + "CATALOGUE_PERIOD_SOURCE", + ) + _require(canonical not in household_keys, "GLOBAL_HOUSEHOLD_COLLISION") + household_keys.add(canonical) + _require(type(row.persons) is tuple, "MEMBER_TUPLE") + _require( + len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE), + "HOUSEHOLD_MEMBER_BOUND", + ) + if source is domains.Source.ASEC: + for person in row.persons: + _require(type(person) is domains.AsecPerson, "PERSON_TYPE") + identity = (2024, person.peridnum) + _require(identity not in person_keys, "GLOBAL_PERSON_COLLISION") + person_keys.add(identity) + if batch and ( + len(batch) == _BATCH_HOUSEHOLDS + or batch_people + len(row.persons) > _BATCH_PEOPLE + ): + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += len(row.persons) + if batch: + consume() + + declarations = { + (item.domain.value, source.value): (item.statistical_unit, share) + for item in domains.declaration() + for source, share in ( + (domains.Source.ACS, item.acs_share), + (domains.Source.ASEC, item.asec_share), + ) + if share > 0 + } + selection = select_domain_households( + row_ids=tuple(_sampling_key(row[0]) for row in rows), + source_channels=tuple(row[0].source.value for row in rows), + domain_keys=tuple(row[1].value for row in rows), + cells=tuple(declarations), + fraction=fraction, + seed=seed, + ) + chosen, cells = [], [] + for cell in selection: + unit, share = declarations[cell.domain, cell.source] + selected = [rows[position] for position in cell.positions] + _require( + not selected or any(row[3] > 0 for row in selected), + "SELECTED_CELL_HAS_NO_POSITIVE_ANCHOR", + ) + for key, domain, observed_unit, anchor, observed_share in selected: + _require( + (observed_unit, observed_share) == (unit, share), "DECLARATION_DRIFT" + ) + chosen.append( + SelectedHousehold( + key, domain, unit, anchor, share, cell.inclusion_probability + ) + ) + cells.append( + SelectionCell( + domains.Source(cell.source), + domains.Domain(cell.domain), + unit, + share, + cell.eligible_households, + len(selected), + cell.inclusion_probability, + ) + ) + return CatalogueSelectionPlan( + tuple(sorted(chosen, key=lambda row: _ordered_key(row.key))), + tuple(sorted(exclusions, key=lambda row: _ordered_key(row.key))), + tuple(cells), + len(household_keys), + fraction, + seed, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py new file mode 100644 index 000000000..fde1388e8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py @@ -0,0 +1,96 @@ +"""Pure observed-age alias; no source, population or release authority. + +The caller authenticates the integer A_AGE observations. This rule changes no +observation date, top code or raw value and reads no source/weight/domain label. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +RULE = "microcosm.us.observed-age-normalization.v1" +AGE_CONVENTION = "observed_interview_age_completed_years" +MAX_ROWS = 2_000_000 +MAX_EXACT_FLOAT64_INTEGER = 2**53 + + +def rule_document() -> dict: + return { + "protocol": RULE, + "input": "A_AGE", + "output": "age", + "dtype": "float64", + "relation": "numeric_identity", + "age_convention": AGE_CONVENTION, + "existing_nonmissing": "must_equal_observed_age", + "temporal_adjustment": False, + "top_code_replacement": False, + } + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_OBSERVED_AGE_" + reason) + + +def normalize_observed_age( + raw_age: pd.Series, incumbent_age: pd.Series | None = None +) -> pd.Series: + """Return a defensive common age, accepting only verified numeric identity. + + Missing common cells may acquire the already observed value; missing or + malformed raw ages refuse. The float bound is a representation limit, not + a scientifically permitted maximum age. Measurement keeps its own range. + """ + _require(isinstance(raw_age, pd.Series), "RAW_SERIES") + _require(0 < len(raw_age) <= MAX_ROWS, "ROW_BOUND") + _require( + pd.api.types.is_integer_dtype(raw_age.dtype) + and not pd.api.types.is_bool_dtype(raw_age.dtype) + and not raw_age.isna().any(), + "RAW_INTEGER_REQUIRED", + ) + _require( + ((raw_age >= 0) & (raw_age <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "RAW_REPRESENTATION", + ) + values = raw_age.to_numpy(dtype=np.float64, copy=True) + if incumbent_age is not None: + _require(isinstance(incumbent_age, pd.Series), "INCUMBENT_SERIES") + _require( + incumbent_age.index.identical(raw_age.index) + and len(incumbent_age) == len(raw_age), + "ORDERED_INDEX", + ) + present = incumbent_age.notna().to_numpy(dtype=bool) + if present.any(): + _require( + pd.api.types.is_numeric_dtype(incumbent_age.dtype) + and not pd.api.types.is_bool_dtype(incumbent_age.dtype) + and not pd.api.types.is_complex_dtype(incumbent_age.dtype), + "INCUMBENT_NUMERIC_REQUIRED", + ) + known = incumbent_age[present] + _require( + ((known >= 0) & (known <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "INCUMBENT_REPRESENTATION", + ) + _require((known == raw_age[present]).all(), "INCUMBENT_CONFLICT") + if pd.api.types.is_integer_dtype(incumbent_age.dtype): + _require( + np.array_equal( + known.to_numpy(dtype=np.int64), + raw_age[present].to_numpy(dtype=np.int64), + ), + "INCUMBENT_CONFLICT", + ) + incumbent = incumbent_age.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + np.isfinite(incumbent[present]).all() + and np.array_equal(incumbent[present], values[present]), + "INCUMBENT_CONFLICT", + ) + # Preserve the original float bits of every known incumbent cell. + values[present] = incumbent[present] + return pd.Series(values, index=raw_age.index.copy(deep=True), name="age") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py new file mode 100644 index 000000000..ef4783e07 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -0,0 +1,842 @@ +"""Source-qualified, sampling-aware budgets for the complete first survey clone. + +Original publisher d is never replaced. The sampling reference b=d/p and +incoming a=s*b are different quantities. These development coefficients are +provisional; this module grants neither graph execution nor release authority. +Decoded bytes cannot issue a budget or a weight-only successor. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import sys +import weakref +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.calibrate import group_bounds +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ( + ArtifactType, + KernelResult, + Node, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + storage_equal, +) + +from . import graph_combined_clone as clone +from . import graph_survey_population as graph +from . import survey_population_preparation as source +from .graph_sources import frame_column_declarations +from .support_provenance import support_clone_index_column, support_source_id_column + +BUDGET_PROTOCOL = "microcosm.us.sampling-origin-budget.v1" +SUCCESSOR_PROTOCOL = "microcosm.us.sampling-origin-weight-only-successor.v1" +BUDGET_TYPE = ArtifactType("microcosm.us.sampling_origin_budget", 1) +SUCCESSOR_TYPE = ArtifactType("microcosm.us.sampling_origin_weight_only_successor", 1) +MAX_PAYLOAD_BYTES = 64 * 1024**2 +MAX_GROUPS = 1_000_000 +MAX_SCALAR_CHARS = 4096 +PRESCRIPTION = ( + "development-provisional-8b-4a-v1", + "allocation=float(exact d*s/p)", + "reference=float(exact d/p)", + "bounds=min(8.0*reference,4.0*actual_incoming)", +) +_ISSUED = {} + + +class SurveyOriginBudgetError(ValueError): + """Static refusal without source observations or identifiers.""" + + +def _require(condition, reason): + if not condition: + raise SurveyOriginBudgetError(reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _json(value): + return graph._bounded_json(value, MAX_PAYLOAD_BYTES) + + +def _pair(value): + _require(type(value) is Fraction, "EXACT_FRACTION_REQUIRED") + _require( + value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, + "FRACTION_BOUND", + ) + return [value.numerator, value.denominator] + + +def _reference(d, p, s, actual_incoming): + """Pure exact arithmetic, not source authority; no float-product shortcut.""" + for value in (d, p, s): + _pair(value) + _require( + d >= 0 and 0 < p <= 1 and s in (Fraction(1), Fraction(1, 2)), "REFERENCE_DOMAIN" + ) + _require(type(actual_incoming) is float, "INCOMING_FLOAT64_REQUIRED") + b, a = d / p, d * s / p + expected_a, reference = float(a), float(b) + _require( + math.isfinite(actual_incoming) + and actual_incoming.hex() == expected_a.hex() + and math.isfinite(reference), + "ALLOCATION_OPERATION_ORDER", + ) + design_bound, incoming_bound = 8.0 * reference, 4.0 * actual_incoming + _require( + all(math.isfinite(v) and v >= 0 for v in (design_bound, incoming_bound)), + "REFERENCE_OVERFLOW", + ) + upper = min(design_bound, incoming_bound) + _require(upper == incoming_bound, "CURRENT_SHARE_BOUND_RELATION") + return { + "d": _pair(d), + "p": _pair(p), + "s": _pair(s), + "b": _pair(b), + "a": _pair(a), + "original_float64_hex": float(d).hex(), + "reference_float64_hex": reference.hex(), + "actual_incoming_float64_hex": actual_incoming.hex(), + "design_bound_float64_hex": design_bound.hex(), + "incoming_bound_float64_hex": incoming_bound.hex(), + "upper_float64_hex": upper.hex(), + } + + +def _modules(): + return ( + sys.modules[__name__], + graph, + clone, + group_bounds, + sys.modules[Population.__module__], + sys.modules[Frame.__module__], + sys.modules[frame_column_declarations.__module__], + sys.modules[support_source_id_column.__module__], + sys.modules[clone.clone_us_frame_for_puf_support.__module__], + ) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = source._function_seal( + function + ) + result["contract"] = ( + BUDGET_PROTOCOL, + SUCCESSOR_PROTOCOL, + (BUDGET_TYPE.name, BUDGET_TYPE.schema_version), + (SUCCESSOR_TYPE.name, SUCCESSOR_TYPE.schema_version), + MAX_PAYLOAD_BYTES, + MAX_GROUPS, + MAX_SCALAR_CHARS, + PRESCRIPTION, + np.__version__, + np.ndarray, + np.float64, + ) + return result + + +def _code_bytes(): + return {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()} + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return _BYTES + + +def _population_identity(population): + _require(type(population) is Population, "LIVE_POPULATION_REQUIRED") + return ( + source._frame_identity(population.frame), + _physical_nonweight_identity(population.frame), + population.version, + tuple(sorted(population.owners.items())), + tuple(population.weight_kind.items()), + _json([asdict(record) for record in population.mass_ledger]), + tuple( + (e, str(v.dtype), v.shape, v.tobytes()) + for e, v in population.design_weights.items() + ), + ) + + +def _physical_nonweight_identity(frame): + """Detach actual storage, including masked numeric slots, one column at a time. + + Axis/schema/dtype identity is bound separately by the full Frame identity. + This is an in-process population seal, not a cache canonicalization boundary. + """ + digest = hashlib.sha256() + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + selected = np.ones(len(series), dtype=np.bool_) + for part in _storage_parts(series, selected): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _initial(view, allocated, expanded): + _require( + type(allocated) is Population and type(expanded) is Population, + "LIVE_POPULATION_REQUIRED", + ) + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") + design = view.frame.weights_for("household").values.copy() + _weights, _context, allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + graph._same_frame(expected, allocated.frame) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(allocated, design) + graph._check_design_anchors(expanded, copied_design) + allocation_ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + cells = tuple((e, str(c)) for e in view.frame.entities for c in view.frame.table(e)) + graph._check_population_state( + allocated, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=allocation_ledger, + ) + owners = dict.fromkeys(cells, clone.COMBINED_CLONE_NODE) + owners.update( + { + (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE + for o in clone_nodes[1].outputs + } + ) + graph._check_population_state( + expanded, + version=clone.COMBINED_CLONE_NODE, + owners=owners, + kind=WeightKind.IMPORTANCE, + ledger=( + *allocation_ledger, + _mass_record( + expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + ), + ), + ) + return instructions, allocation + + +def _document(view, allocated, expanded, instructions, allocation, producer): + before, after = ( + allocated.frame.table("household"), + expanded.frame.table("household"), + ) + source_column = support_source_id_column("household") + role_column = support_clone_index_column("household") + positions = {int(value): i for i, value in enumerate(before[source_column])} + _require(len(positions) == len(instructions), "ROOT_SOURCE_ID_COLLISION") + members = [[] for _ in instructions] + ids, groups = [], [] + for row in after[["household_id", source_column, role_column]].itertuples( + index=False, name=None + ): + hh_id, source_id, role = (int(value) for value in row) + _require(source_id in positions and role in (0, 1), "CLONE_ORIGIN_ROLE") + group = positions[source_id] + members[group].append((hh_id, role)) + ids.append(hh_id) + groups.append(group) + _require( + len(ids) == 2 * len(instructions) and len(set(ids)) == len(ids), + "CLONE_CARDINALITY", + ) + incoming = allocated.frame.weights_for("household").values + clone_weights = expanded.frame.weights_for("household").values + by_id = {value: i for i, value in enumerate(ids)} + bounds = [] + selected_by_key = {row.key: row for row in view.selection_plan.selected} + + def records(): + for i, (instruction, group_members) in enumerate( + zip(instructions, members, strict=True) + ): + _require( + len(group_members) == 2 and {m[1] for m in group_members} == {0, 1}, + "COMPLETE_CLONE_ROLES", + ) + actual = float(incoming[i]) + _require( + math.fsum(float(clone_weights[by_id[m[0]]]) for m in group_members) + == actual, + "CLONE_INCOMING_CONSERVATION", + ) + reference = _reference( + instruction.original_anchor, + instruction.inclusion_probability, + instruction.share, + actual, + ) + bounds.append(float.fromhex(reference["upper_float64_hex"])) + key = instruction.key + _require( + type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, + "NATIVE_KEY_BOUND", + ) + yield { + "source": key.source.value, + "source_year": key.source_year, + "survey_year": key.survey_year, + "raw_native_id": key.native_id, + "selected_receiving_household_id": instruction.selected_receiving_household_id, + "combined_household_id": instruction.household_id, + "statistical_unit": selected_by_key[instruction.key].statistical_unit, + "original_design_float64_bytes": allocated.design_weights["household"][ + i : i + 1 + ] + .tobytes() + .hex(), + **reference, + "members": [[hh_id, role] for hh_id, role in group_members], + "incoming_clone_float64_bytes": [ + clone_weights[by_id[hh_id] : by_id[hh_id] + 1].tobytes().hex() + for hh_id, _role in group_members + ], + } + + header = { + "protocol": BUDGET_PROTOCOL, + "prescription": PRESCRIPTION, + "coefficient_status": "provisional_development_not_scientific_approval", + "current_share_simplification": "min(8b,4a)=4a; 8b is redundant for current shares", + "preparation_sha256": _sha(view.payload), + "source_producer": view.receipt["producer"], + "source_native": view.receipt["native"], + "source_catalogues": view.receipt["catalogues"], + "selection_sha256": _sha(_json(view.receipt["selection"])), + "allocation_sha256": _sha(allocation), + "producer": producer, + "allocated_frame_sha256": source._frame_identity(allocated.frame), + "clone_frame_sha256": source._frame_identity(expanded.frame), + "household_ids": ids, + "group_indices": groups, + "group_count": len(instructions), + "release_eligible": False, + } + # Stream one bounded origin record at a time; never materialize an unbounded + # list of origin dictionaries before the transport cap is checked. + payload = bytearray() + + def append(piece): + _require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), "TRANSPORT_LIMIT") + payload.extend(piece) + + append(b"{") + keys = sorted((*header, "origins")) + for index, key in enumerate(keys): + if index: + append(b",") + append(_json(key) + b":") + if key == "origins": + append(b"[") + for position, record in enumerate(records()): + if position: + append(b",") + append(_json(record)) + append(b"]") + else: + append(_json(header[key])) + append(b"}") + constraint = group_bounds.GroupedUpperBounds(ids, groups, bounds) + constraint.check(clone_weights, positive=False) + return bytes(payload), constraint + + +@dataclass(frozen=True) +class _BudgetState: + preparation: object + preparation_entry: tuple + initial_view: source.CheckedSurveyPopulationView + allocated: Population + expanded: Population + preparation_payload: bytes + allocated_identity: tuple + expanded_identity: tuple + + +def _preparation_entry(preparation, payload, expected=None): + """Check retained issuance identity without repeating source-file reads.""" + entry = source._ISSUED.get(id(preparation)) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation + and entry is not None + and (expected is None or entry is expected) + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1] == payload, + "FINAL_PREPARATION_ISSUANCE", + ) + return entry + + +def _final_budget_state(state): + """Seal retained owners after helper/file I/O, with no recursive borrow. + + The owner's pure check still hashes retained Frames. It does not parse or + reread source files, issue replacement capsules or grant decoded authority. + """ + _producer() + entry = _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + source._pure_final(entry[2]) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "FINAL_POPULATION_SEAL", + ) + _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") + _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + + +def _validate_budget(state, payload): + _producer() + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "INITIAL_POPULATION_CHANGED", + ) + # This private retained view is only a reconstruction input. The single + # fresh source borrow below must authenticate it after all helper/file I/O. + view = state.initial_view + _require(view.payload == state.preparation_payload, "SOURCE_PREPARATION_CHANGED") + instructions, allocation = _initial(view, state.allocated, state.expanded) + expected, constraint = _document( + view, state.allocated, state.expanded, instructions, allocation, _producer() + ) + _require(expected == payload, "BUDGET_RECONSTRUCTION") + final = source.AuthenticatedSurveyPopulationPreparation.checked_view( + state.preparation + ) + _require( + final.payload == view.payload + and final.frame is view.frame + and final.selection_plan is view.selection_plan, + "FINAL_SOURCE_SEAL", + ) + _initial(final, state.allocated, state.expanded) + return constraint + + +def _entry(value, cls): + entry = _ISSUED.get(id(value)) + _require( + type(value) is cls + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +def _issue(cls, payload, state): + value = object.__new__(cls) + object.__setattr__(value, "payload", payload) + identifier = id(value) + + def forget(reference): + current = _ISSUED.get(identifier) + if current is not None and current[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(value, forget) + _ISSUED[identifier] = (reference, payload, state) + return value + + +@dataclass(frozen=True) +class SamplingOriginBudget: + payload: bytes + + def __post_init__(self): + raise SurveyOriginBudgetError("NO_PUBLIC_BUDGET_CONSTRUCTOR") + + def checked_view(self): + entry = _entry(self, SamplingOriginBudget) + document, digest = json.loads(entry[1]), _sha(entry[1]) + constraint = _validate_budget(entry[2], entry[1]) + result = CheckedSamplingOriginBudget( + entry[1], + digest, + document, + constraint, + entry[2].preparation, + entry[2].allocated, + entry[2].expanded, + ) + _final_budget_state(entry[2]) + _require(_entry(self, SamplingOriginBudget) is entry, "FINAL_BUDGET_SEAL") + return result + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginBudget: + """Defensive values and a numerical constraint, not source authority.""" + + payload: bytes + digest: str + document: dict + grouped_bounds: group_bounds.GroupedUpperBounds + preparation: object + allocated_population: Population + initial_population: Population + + +def freeze_survey_origin_budget( + preparation, *, allocated_population, clone_population, candidate=None +): + _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") + _require( + candidate is None or len(candidate) <= MAX_PAYLOAD_BYTES, "CANDIDATE_LIMIT" + ) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "ISSUED_PREPARATION_REQUIRED", + ) + producer = _producer() + view = source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) + preparation_entry = _preparation_entry(preparation, view.payload) + instructions, allocation = _initial(view, allocated_population, clone_population) + state = _BudgetState( + preparation, + preparation_entry, + view, + allocated_population, + clone_population, + view.payload, + _population_identity(allocated_population), + _population_identity(clone_population), + ) + payload, _constraint = _document( + view, allocated_population, clone_population, instructions, allocation, producer + ) + final = source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) + _require( + final.payload == view.payload and final.frame is view.frame, "FINAL_SOURCE_SEAL" + ) + _initial(final, allocated_population, clone_population) + _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") + result = _issue(SamplingOriginBudget, payload, state) + entry = _entry(result, SamplingOriginBudget) + _final_budget_state(state) + _require(_entry(result, SamplingOriginBudget) is entry, "FINAL_BUDGET_SEAL") + return result + + +def verify_survey_origin_budget(budget): + _require(type(budget) is SamplingOriginBudget, "BUDGET_TYPE") + budget.checked_view() + return budget + + +def _same_nonweight(before, after): + _require( + before.schema == after.schema + and before.entities == after.entities + and before.links == after.links == () + and before.metadata == after.metadata + and before.mass_log == after.mass_log + and before.weighted_entities == after.weighted_entities == ("household",), + "SUCCESSOR_NONWEIGHT_CONTEXT", + ) + try: + for entity in before.entities: + pd.testing.assert_frame_equal( + before.table(entity), + after.table(entity), + check_exact=True, + check_flags=True, + ) + for column in before.table(entity): + _require( + storage_equal( + before.table(entity)[column], after.table(entity)[column] + ), + "SUCCESSOR_NONWEIGHT_STORAGE", + ) + pd.testing.assert_series_equal(before.strata, after.strata, check_exact=True) + _require( + storage_equal(before.strata, after.strata), "SUCCESSOR_NONWEIGHT_STORAGE" + ) + except (AssertionError, ValueError, TypeError): + raise SurveyOriginBudgetError("SUCCESSOR_NONWEIGHT_TABLE") from None + + +@dataclass(frozen=True) +class _SuccessorState: + budget: SamplingOriginBudget + budget_entry: tuple + previous: Population + current: Population + previous_binding: object + previous_identity: tuple + current_identity: tuple + + +def _successor_document(state, budget_view): + previous, current = state.previous, state.current + _require( + type(previous) is Population and type(current) is Population, + "LIVE_POPULATION_REQUIRED", + ) + initial = _entry(state.budget, SamplingOriginBudget)[2].expanded + _require(state.previous_binding is None, "UNSUPPORTED_REFINEMENT") + _require(previous is initial, "INITIAL_PREVIOUS_REQUIRED") + _same_nonweight(initial.frame, previous.frame) + _same_nonweight(initial.frame, current.frame) + graph._check_design_anchors(previous, initial.design_weights["household"]) + graph._check_design_anchors(current, initial.design_weights["household"]) + _require( + current.frame.weights_for("household").kind is WeightKind.CALIBRATED + and current.version != previous.version, + "SUCCESSOR_CALIBRATED_VERSION", + ) + _require( + previous.frame.weights_for("household").kind is WeightKind.IMPORTANCE, + "UNSUPPORTED_REFINEMENT", + ) + node = Node( + current.version, + "us.sampling_origin.weight_only_admission@1", + base=previous.version, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + ) + ledger = ( + *previous.mass_ledger, + _mass_record(previous.frame, current.frame, node, KernelResult(), "free"), + ) + owners = dict.fromkeys(previous.owners, current.version) + graph._check_population_state( + current, + version=current.version, + owners=owners, + kind=WeightKind.CALIBRATED, + ledger=ledger, + ) + constraint = budget_view.grouped_bounds + constraint.check(previous.frame.weights_for("household").values, positive=False) + totals = constraint.check( + current.frame.weights_for("household").values, positive=False + ) + # Explicit necessary per-row reference bound, in addition to group U. + rows = budget_view.document["origins"] + for position, group in enumerate(constraint.group_indices): + _require( + float(current.frame.weights_for("household").values[position]) + <= float.fromhex(rows[int(group)]["design_bound_float64_hex"]), + "ROW_REFERENCE_BOUND", + ) + return { + "protocol": SUCCESSOR_PROTOCOL, + "budget_sha256": budget_view.digest, + "previous_binding_sha256": None, + "previous_version": previous.version, + "current_version": current.version, + "previous_frame_sha256": source._frame_identity(previous.frame), + "current_frame_sha256": source._frame_identity(current.frame), + "mass_ledger": [asdict(record) for record in current.mass_ledger], + "group_total_float64_hex": [float(v).hex() for v in totals], + "constraint_digest": constraint.digest, + "release_eligible": False, + } + + +def _state_unchanged(state): + _require( + _population_identity(state.previous) == state.previous_identity + and _population_identity(state.current) == state.current_identity, + "FINAL_SUCCESSOR_POPULATION_SEAL", + ) + + +def _checked_successor_document(state, budget_view): + """Only the actual initial IMPORTANCE to CALIBRATED transition is supported.""" + _state_unchanged(state) + payload = _json(_successor_document(state, budget_view)) + _state_unchanged(state) + return payload + + +def _final_successor_state(state): + """Check actual retained upstream handles after the final document helper.""" + _require( + _entry(state.budget, SamplingOriginBudget) is state.budget_entry, + "FINAL_UPSTREAM_BUDGET_SEAL", + ) + budget_state = state.budget_entry[2] + _final_budget_state(budget_state) + _state_unchanged(state) + _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") + _preparation_entry( + budget_state.preparation, + budget_state.preparation_payload, + budget_state.preparation_entry, + ) + _require( + _entry(state.budget, SamplingOriginBudget) is state.budget_entry, + "FINAL_UPSTREAM_BUDGET_SEAL", + ) + + +@dataclass(frozen=True) +class SamplingOriginSuccessor: + payload: bytes + + def __post_init__(self): + raise SurveyOriginBudgetError("NO_PUBLIC_SUCCESSOR_CONSTRUCTOR") + + def checked_view(self): + entry = _entry(self, SamplingOriginSuccessor) + document, digest = json.loads(entry[1]), _sha(entry[1]) + state = entry[2] + view = state.budget.checked_view() + expected = _checked_successor_document(state, view) + _require(expected == entry[1], "SUCCESSOR_RECONSTRUCTION") + final = state.budget.checked_view() + _require( + _checked_successor_document(state, final) == entry[1], + "FINAL_SUCCESSOR_SEAL", + ) + result = CheckedSamplingOriginSuccessor( + entry[1], + digest, + document, + state.budget, + state.previous, + state.current, + state.previous_binding, + ) + _final_successor_state(state) + _require(_entry(self, SamplingOriginSuccessor) is entry, "FINAL_SUCCESSOR_SEAL") + return result + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginSuccessor: + """A checked value borrow; authority remains in the issued handles.""" + + payload: bytes + digest: str + document: dict + budget: SamplingOriginBudget + previous: Population + current: Population + previous_binding: object + + +def admit_survey_weight_only_population( + budget, *, previous, current, previous_binding=None +): + # The actual graph permits only strictly forward weight-kind transitions. + # Keeping this keyword fail-closed makes deferred refinement explicit. + _require(previous_binding is None, "UNSUPPORTED_REFINEMENT") + _require(type(previous) is Population, "LIVE_POPULATION_REQUIRED") + _require( + previous.frame.weights_for("household").kind is WeightKind.IMPORTANCE, + "UNSUPPORTED_REFINEMENT", + ) + _require(type(budget) is SamplingOriginBudget, "BUDGET_TYPE") + view = budget.checked_view() + state = _SuccessorState( + budget, + _entry(budget, SamplingOriginBudget), + previous, + current, + previous_binding, + _population_identity(previous), + _population_identity(current), + ) + payload = _checked_successor_document(state, view) + final = budget.checked_view() + _require( + _checked_successor_document(state, final) == payload, "FINAL_SUCCESSOR_SEAL" + ) + result = _issue(SamplingOriginSuccessor, payload, state) + entry = _entry(result, SamplingOriginSuccessor) + _final_successor_state(state) + _require(_entry(result, SamplingOriginSuccessor) is entry, "FINAL_SUCCESSOR_SEAL") + return result + + +def verify_survey_weight_only_successor(binding): + _require(type(binding) is SamplingOriginSuccessor, "SUCCESSOR_TYPE") + binding.to_bytes() + return binding + + +_BYTES = _code_bytes() +_LIVE = _live() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py new file mode 100644 index 000000000..584a63e8f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_domains.py @@ -0,0 +1,577 @@ +"""Pure US development-domain declarations over complete, supplied literal rows. + +This module reads no source, constructs no Frame and applies no weights. Its +consistency checks cannot authenticate original membership, publisher weights, +source bytes or a receiving population. Every result requires those bindings. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from fractions import Fraction + +PROTOCOL = "microcosm.us.survey-population-domains.v1" +DECLARATION = "experiments/us-survey-allocation-declaration-v1-20260907.md" +MAX_TOKEN_CHARS = 128 +MAX_MEMBERS = 20 +MAX_HOUSEHOLDS = 100_000 +MAX_TOTAL_MEMBERS = 1_000_000 + + +class DomainInputError(ValueError): + """Static consistency refusal; never include raw identities or observations.""" + + +class Source(Enum): + ACS = "acs" + ASEC = "asec" + + +class Qualifier(Enum): + TRUE = "true" + FALSE = "false" + UNKNOWN = "unknown" + + +class Status(Enum): + ELIGIBLE = "eligible" + EXCLUDED = "excluded" + REVIEW_REQUIRED = "review_required" + + +class Domain(Enum): + SHARED_HOUSING = "shared_housing" + RESIDUAL_HOUSING = "residual_housing" + INSTITUTIONAL_GQ = "institutional_gq" + NONINSTITUTIONAL_GQ = "noninstitutional_gq" + + +@dataclass(frozen=True, slots=True) +class HouseholdKey: + source: Source + source_year: int + survey_year: int + native_id: str + + +@dataclass(frozen=True, slots=True) +class AcsPerson: + sporder: str + age: str + esr: str + esr_state: str + mil: str + mil_state: str + pwgtp: str + household_key: HouseholdKey + + +@dataclass(frozen=True, slots=True) +class AsecPerson: + peridnum: str + a_lineno: str + age: str + prpertyp: str + prpertyp_state: str + household_key: HouseholdKey + + +@dataclass(frozen=True, slots=True) +class AcsHousehold: + key: HouseholdKey + typehugq: str + np: str + wgtp: str + persons: tuple[AcsPerson, ...] + + +@dataclass(frozen=True, slots=True) +class AsecHousehold: + key: HouseholdKey + h_hhtype: str + hrhtype: str + h_livqrt: str + h_numper: str + hsup_wgt: str + persons: tuple[AsecPerson, ...] + + +@dataclass(frozen=True, slots=True) +class Observation: + field: str + literal: str + state: str + + +@dataclass(frozen=True, slots=True) +class PersonDecision: + original: AcsPerson | AsecPerson + qualifier: Qualifier + reason: str + observations: tuple[Observation, ...] + diagnostics: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class Annotation: + source: str + chosen_rule: str + alternative: str + status: str + + +@dataclass(frozen=True, slots=True) +class DomainShare: + domain: Domain + statistical_unit: str + acs_share: Fraction + asec_share: Fraction + + +@dataclass(frozen=True, slots=True) +class HouseholdDecision: + original: AcsHousehold | AsecHousehold + status: Status + domain: Domain | None + share: Fraction | None + reason: str + statistical_unit: str | None + physical_housing_units: int | None + publisher_weight_is_zero: bool | None + people: tuple[PersonDecision, ...] + observations: tuple[Observation, ...] + annotations: tuple[Annotation, ...] + + @property + def source_authenticated(self) -> bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + @property + def required_bindings(self) -> tuple[str, ...]: + return ( + "original_source_bytes", + "complete_original_membership", + "original_publisher_weight_authority", + "source_period_and_residence_scope", + "full_receiving_population_and_producer_edges", + "actual_selection_probability", + ) + + @property + def analysis_year(self) -> int: + return 2024 + + @property + def income_reference(self) -> str: + if self.original.key.source is Source.ASEC: + return f"calendar_year_{self.original.key.source_year}" + return "previous_12_months_at_each_2024_interview" + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise DomainInputError(code) + + +def _token(value: str) -> str: + _require(type(value) is str and len(value) <= MAX_TOKEN_CHARS, "LITERAL_TYPE_SIZE") + return value + + +def _unsigned(raw: str, digits: int, maximum: int) -> int | None: + _token(raw) + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None: + return None + value = int(raw) + return value if value <= maximum else None + + +def _key(key: HouseholdKey) -> tuple[Source, int, int, str | int]: + _require(type(key) is HouseholdKey and type(key.source) is Source, "HOUSEHOLD_KEY") + _require( + type(key.source_year) is int and type(key.survey_year) is int, "COHORT_TYPE" + ) + _token(key.native_id) + if key.source is Source.ACS: + _require((key.source_year, key.survey_year) == (2024, 2024), "COHORT_IDENTITY") + _require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", key.native_id) is not None + and int(key.native_id[-7:]) > 0, + "HOUSEHOLD_KEY", + ) + native = key.native_id + else: + _require( + key.source_year in (2022, 2023, 2024) + and key.survey_year == key.source_year + 1, + "COHORT_IDENTITY", + ) + native = _unsigned(key.native_id, MAX_TOKEN_CHARS, 99999) + _require(native is not None and native > 0, "HOUSEHOLD_KEY") + return key.source, key.source_year, key.survey_year, native + + +def declaration() -> tuple[DomainShare, ...]: + """The prospectively fixed coefficients, without applying them to weights.""" + return ( + DomainShare( + Domain.SHARED_HOUSING, + "occupied_housing_unit", + Fraction(1, 2), + Fraction(1, 2), + ), + DomainShare( + Domain.RESIDUAL_HOUSING, "occupied_housing_unit", Fraction(1), Fraction(0) + ), + DomainShare(Domain.INSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + DomainShare(Domain.NONINSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + ) + + +def reduce_qualifiers(values: tuple[Qualifier, ...]) -> Qualifier: + """Existential reduction over known roster members, not membership proof.""" + _require( + type(values) is tuple and 0 < len(values) <= MAX_MEMBERS, "QUALIFIER_ROSTER" + ) + _require(all(type(v) is Qualifier for v in values), "QUALIFIER_TYPE") + if Qualifier.TRUE in values: + return Qualifier.TRUE + if Qualifier.UNKNOWN in values: + return Qualifier.UNKNOWN + return Qualifier.FALSE + + +def _acs_state(age: str, raw: str, minimum: int, codes: tuple[str, ...]) -> str: + # Exact vocabulary of acs_person_coverage_columns._field_state. No owner import. + value = _unsigned(age, 2, 99) + if value is None: + return "age_unresolved" + if value < minimum: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _asec_state(raw: str) -> str: + # Exact vocabulary of asec_person_coverage_source._state. + if raw in ("1", "2", "3"): + return "observed_code" + if raw == "": + return "blank_unresolved" + if raw in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", raw) and raw != "-0": + return "out_of_range" + return "malformed_token" + + +def _person(person: AcsPerson | AsecPerson) -> PersonDecision: + is_acs = type(person) is AcsPerson + age = _unsigned(person.age, 2 if is_acs else 19, 99 if is_acs else 2**63 - 1) + observations = [ + Observation( + "AGEP" if is_acs else "A_AGE", + person.age, + "observed_integer" if age is not None else "age_unresolved", + ) + ] + diagnostics = [] + if is_acs: + _token(person.pwgtp) + for raw, state in ( + (person.esr, person.esr_state), + (person.mil, person.mil_state), + ): + _token(raw) + _token(state) + _require( + person.esr_state + == _acs_state(person.age, person.esr, 16, ("1", "2", "3", "4", "5", "6")), + "FIELD_STATE_MISMATCH", + ) + _require( + person.mil_state + == _acs_state(person.age, person.mil, 17, ("1", "2", "3", "4")), + "FIELD_STATE_MISMATCH", + ) + observations.extend( + ( + Observation("ESR", person.esr, person.esr_state), + Observation("MIL", person.mil, person.mil_state), + ) + ) + true = person.esr in ("1", "2", "3", "6") + false = person.esr in ("4", "5") + if ( + person.mil_state == "observed_code" + and person.esr_state == "observed_code" + and ( + (person.mil == "1" and true) + or (person.mil in ("2", "3", "4") and false) + ) + ): + diagnostics.append("mil_esr_disagreement") + if person.mil_state != "observed_code": + diagnostics.append("mil_" + person.mil_state) + contradiction = False + else: + _token(person.prpertyp) + _token(person.prpertyp_state) + _require( + person.prpertyp_state == _asec_state(person.prpertyp), + "FIELD_STATE_MISMATCH", + ) + observations.append( + Observation("PRPERTYP", person.prpertyp, person.prpertyp_state) + ) + true, false = person.prpertyp == "2", person.prpertyp == "3" + contradiction = age is not None and age >= 16 and person.prpertyp == "1" + if age is None: + qualifier, reason = Qualifier.UNKNOWN, "source_age_unresolved" + elif age < 16: + qualifier, reason = Qualifier.FALSE, "source_age_below_16" + elif contradiction: + qualifier, reason = Qualifier.UNKNOWN, "adult_age_with_child_type" + elif true: + qualifier, reason = Qualifier.TRUE, "source_civilian_proxy" + elif false: + qualifier, reason = Qualifier.FALSE, "source_armed_forces_proxy" + else: + qualifier, reason = Qualifier.UNKNOWN, "required_person_status_unresolved" + return PersonDecision( + person, qualifier, reason, tuple(observations), tuple(diagnostics) + ) + + +def _members(row: AcsHousehold | AsecHousehold) -> tuple[PersonDecision, ...]: + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + household = _key(row.key) + seen_keys, seen_lines = set(), set() + expected = AcsPerson if type(row) is AcsHousehold else AsecPerson + for person in row.persons: + _require(type(person) is expected, "MEMBER_TYPE") + _require(_key(person.household_key) == household, "MEMBER_HOUSEHOLD_KEY") + raw_line = person.sporder if expected is AcsPerson else person.a_lineno + line = _unsigned( + raw_line, + 2 if expected is AcsPerson else 19, + 20 if expected is AcsPerson else 2**63 - 1, + ) + _require( + line is not None and line > 0 and line not in seen_lines, "MEMBER_LINE_KEY" + ) + seen_lines.add(line) + person_key = line if expected is AcsPerson else _token(person.peridnum) + if expected is AsecPerson: + _require( + re.fullmatch(r"[0-9]{22}", person_key) is not None, "MEMBER_PERSON_KEY" + ) + _require(person_key not in seen_keys, "MEMBER_DUPLICATE_KEY") + seen_keys.add(person_key) + return tuple(_person(person) for person in row.persons) + + +def _field( + name: str, raw: str, digits: int, maximum: int, minimum: int = 0 +) -> tuple[int | None, Observation]: + value = _unsigned(raw, digits, maximum) + known = value is not None and value >= minimum + state = ( + "valid" + if known + else "missing" + if raw == "" + else "malformed" + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None + else "unlabelled" + ) + return value if known else None, Observation(name, raw, state) + + +def classify_household(row: AcsHousehold | AsecHousehold) -> HouseholdDecision: + """Classify supplied complete rows; unresolved results authorize no allocation. + + Key/tuple/count inconsistencies raise. Unknown required observations retain + their literals and produce REVIEW_REQUIRED, never residual by default. + Source and population authority cannot be supplied as a boolean argument. + """ + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + is_acs = type(row) is AcsHousehold + _require(key[0] is (Source.ACS if is_acs else Source.ASEC), "HOUSEHOLD_SOURCE") + people = _members(row) + observations = [] + zero = None + + def field(name, raw, digits, maximum, minimum=0): + value, observation = _field(name, raw, digits, maximum, minimum) + observations.append(observation) + return value + + def result(status, reason, domain=None, share=None, unit=None, housing_units=None): + return HouseholdDecision( + row, + status, + domain, + share, + reason, + unit, + housing_units, + zero, + people, + tuple(observations), + ( + Annotation( + DECLARATION, + "source_age_16_ESR_only_or_PRPERTYP", + "joint_MIL_ESR_or_other_age_threshold", + "prespecified_development_choice", + ), + Annotation( + DECLARATION, + "fixed_domain_shares_and_structural_exclusions", + "estimated_overlap_or_optimized_shares", + "not_empirically_validated", + ), + Annotation( + DECLARATION, + reason, + "no_silent_residual_or_donor_admission", + status.value, + ), + ), + ) + + count = field( + "NP" if is_acs else "H_NUMPER", + row.np if is_acs else row.h_numper, + 2, + 20 if is_acs else 16, + ) + if count is not None: + _require(count == len(row.persons), "REPORTED_MEMBERSHIP_COUNT") + if is_acs: + housing = field("TYPEHUGQ", row.typehugq, 1, 3, 1) + weight = field("WGTP", row.wgtp, 4, 9999) + if count is None or housing is None: + return result(Status.REVIEW_REQUIRED, "acs_household_condition_unresolved") + if row.key.native_id[4:6] != ("HU" if housing == 1 else "GQ"): + return result(Status.REVIEW_REQUIRED, "acs_serialno_housing_conflict") + if housing == 1 and count == 0: + return result( + Status.EXCLUDED, "acs_vacancy", share=Fraction(0), housing_units=0 + ) + if housing in (2, 3): + if count != 1 or weight != 0: + return result(Status.REVIEW_REQUIRED, "acs_gq_placeholder_conflict") + person_weight = field("PWGTP", row.persons[0].pwgtp, 4, 9999, 1) + if person_weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + return result( + Status.ELIGIBLE, + "acs_gq_allocation", + Domain.INSTITUTIONAL_GQ if housing == 2 else Domain.NONINSTITUTIONAL_GQ, + Fraction(1), + "person", + 0, + ) + if weight is None or weight <= 0: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + else: + interview = field("H_HHTYPE", row.h_hhtype, 1, 3, 1) + household_type = field("HRHTYPE", row.hrhtype, 2, 10) + living = field("H_LIVQRT", row.h_livqrt, 2, 12, 1) + weight = field("HSUP_WGT", row.hsup_wgt, MAX_TOKEN_CHARS, 999999999) + if count is None: + return result(Status.REVIEW_REQUIRED, "asec_membership_count_unresolved") + if row.key.source_year != 2024: + return result(Status.EXCLUDED, "asec_older_cohort", share=Fraction(0)) + if interview in (2, 3): + return result(Status.EXCLUDED, "asec_noninterview", share=Fraction(0)) + if interview is None or living is None or household_type is None: + return result(Status.REVIEW_REQUIRED, "asec_household_condition_unresolved") + if living == 11: + return result( + Status.REVIEW_REQUIRED, + "asec_student_quarters_source_review", + share=Fraction(0), + housing_units=0, + ) + if living >= 8: + return result( + Status.EXCLUDED, "asec_nonhousing", share=Fraction(0), housing_units=0 + ) + if household_type not in range(1, 9) or count == 0: + return result(Status.REVIEW_REQUIRED, "asec_housing_membership_conflict") + if weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = weight == 0 + predicate = reduce_qualifiers(tuple(person.qualifier for person in people)) + if predicate is Qualifier.UNKNOWN: + return result(Status.REVIEW_REQUIRED, "household_qualifier_unresolved") + if predicate is Qualifier.TRUE: + return result( + Status.ELIGIBLE, + "chosen_shared_housing", + Domain.SHARED_HOUSING, + Fraction(1, 2), + "occupied_housing_unit", + 1, + ) + return result( + Status.ELIGIBLE if is_acs else Status.EXCLUDED, + "acs_known_residual_housing" if is_acs else "asec_residual_zero_allocation", + Domain.RESIDUAL_HOUSING, + Fraction(1) if is_acs else Fraction(0), + "occupied_housing_unit", + 1, + ) + + +def classify_households( + rows: tuple[AcsHousehold | AsecHousehold, ...], +) -> tuple[HouseholdDecision, ...]: + """Classify a bounded batch, rejecting household and cohort-person collisions.""" + _require(type(rows) is tuple and 0 < len(rows) <= MAX_HOUSEHOLDS, "HOUSEHOLD_BATCH") + keys, members = set(), 0 + for row in rows: + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + _require(key not in keys, "DUPLICATE_HOUSEHOLD_KEY") + keys.add(key) + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + members += len(row.persons) + _require(members <= MAX_TOTAL_MEMBERS, "BATCH_MEMBER_BOUND") + person_keys = set() + for row in rows: + if type(row) is AsecHousehold: + for person in row.persons: + _require(type(person) is AsecPerson, "MEMBER_TYPE") + literal = _token(person.peridnum) + _require( + re.fullmatch(r"[0-9]{22}", literal) is not None, + "MEMBER_PERSON_KEY", + ) + person_key = (row.key.source_year, literal) + _require(person_key not in person_keys, "DUPLICATE_COHORT_PERSON_KEY") + person_keys.add(person_key) + return tuple(classify_household(row) for row in rows) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py new file mode 100644 index 000000000..2840e376f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -0,0 +1,1338 @@ +"""Authenticate complete catalogues, select once, and stack original anchors. + +The selection is a development declaration. Source issuers retain their own +exact-key/unknown-inclusion receipts; this successor binds the separately +demonstrated catalogue sampling relation. No allocation, calibration or donor +operation runs here. Source catalogues/parents still have full-source costs. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import json +import math +import os +import stat +import sys +import weakref +from contextlib import contextmanager +from dataclasses import InitVar, dataclass, fields +from enum import Enum +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd +from microcosm.build import survey_domain_sample +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import population as graph_population +from microcosm.graph.population import dtype_for_token, storage_equal + +from . import acs_native_coverage_binding as acs_native +from . import acs_population_catalogue as acs_catalogue +from . import asec_2024_native_population as asec_native +from . import asec_current_money as current_money +from . import asec_population_catalogue as asec_catalogue +from . import graph_context, graph_sources, spine_assembly +from . import survey_catalogue_selection as selection +from . import survey_observed_age as observed_age +from . import survey_population_domains as domains +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.survey-population-preparation.v2" +REQUEST_PROTOCOL = "microcosm.us.survey-population-request.v1" +SOURCE_CODEC = "us-survey-population-source-v1" +MAX_REQUEST_BYTES = 4096 +MAX_PAYLOAD_BYTES = 64 * 1024**2 +_MAX_SCALAR_BYTES = 1024**2 +_SOURCE_ROSTER = ( + "selection-request.json", + "acs/csv_hus.zip", + "acs/csv_pus.zip", + "asec/parent.h5", + "asec/household-attachment.h5", + "asec/person-income-attachment.h5", + "asec/pppub23.csv", + "asec/pppub24.csv", + "asec/pppub25.csv", + "asec/hhpub25.csv", +) +_TOKEN = object() +_ISSUED = {} + + +class SurveyPopulationPreparationError(ValueError): + """Static refusal; messages contain no source observations or paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationPreparationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _check_scalars(value, depth=0): + _require(depth <= 64, "VALUE_DEPTH") + if type(value) is str: + _require(len(value) <= _MAX_SCALAR_BYTES, "SCALAR_LIMIT") + elif type(value) in (tuple, list): + for item in value: + _check_scalars(item, depth + 1) + elif type(value) is dict: + for key, item in value.items(): + _check_scalars(key, depth + 1) + _check_scalars(item, depth + 1) + + +def _chunks(value): + # A single primitive is bounded before JSON quoting. The encoder never + # builds an unbounded complete JSON string before enforcing the transport. + _check_scalars(value) + yield from json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).iterencode(value) + + +def _encode(value, maximum=MAX_PAYLOAD_BYTES): + result = bytearray() + for chunk in _chunks(value): + encoded = chunk.encode("utf-8") + _require(len(result) + len(encoded) <= maximum, "PAYLOAD_LIMIT") + result.extend(encoded) + return bytes(result) + + +def _digest(value): + digest = hashlib.sha256() + for chunk in _chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _value(value): + if isinstance(value, Enum): + return value.value + if type(value) is Fraction: + return [value.numerator, value.denominator] + if hasattr(type(value), "__dataclass_fields__"): + return { + field.name: _value(getattr(value, field.name)) for field in fields(value) + } + if type(value) in (tuple, list): + return [_value(item) for item in value] + if type(value) is dict: + return {key: _value(item) for key, item in value.items()} + if isinstance(value, np.generic): + return value.item() + _require(value is None or type(value) in (str, int, bool, float), "VALUE_TYPE") + return value + + +def _plan_document(plan): + _require(type(plan) is selection.CatalogueSelectionPlan, "SELECTION_TYPE") + result = { + "fraction": _value(plan.fraction), + "seed": plan.seed, + "supplied_households": plan.supplied_households, + } + budget = [0] + for name in ("selected", "excluded", "cells"): + result[name] = [] + for row in getattr(plan, name): + _bounded_append(result[name], _value(row), budget) + return result + + +def _root(value, *, allow_missing=False): + _require(isinstance(value, (str, Path)), "PATH_TYPE") + path = Path(value).absolute() + for component in (path, *path.parents): + if allow_missing: + try: + component.lstat() + except FileNotFoundError: + continue + _require(not stat.S_ISLNK(component.lstat().st_mode), "SOURCE_SYMLINK") + return path + + +def _stat_identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def _regular_reader(path, maximum): + """Never follow a replaced final symlink or block on a substituted FIFO.""" + _root(path.parent) + before = path.lstat() + _require( + stat.S_ISREG(before.st_mode) and 0 <= before.st_size <= maximum, + "SOURCE_REGULAR_FILE", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + opened = os.fstat(descriptor) + _require( + stat.S_ISREG(opened.st_mode) + and _stat_identity(opened) == _stat_identity(before), + "SOURCE_DESCRIPTOR_CHANGED", + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + yield stream, opened + after = os.fstat(descriptor) + current = path.lstat() + _require( + stat.S_ISREG(current.st_mode) + and _stat_identity(opened) + == _stat_identity(after) + == _stat_identity(current), + "SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _request(source_dir): + root = _root(source_dir) + path = root / "selection-request.json" + info = path.lstat() + _require( + stat.S_ISREG(info.st_mode) and info.st_size <= MAX_REQUEST_BYTES, "REQUEST_FILE" + ) + with _regular_reader(path, MAX_REQUEST_BYTES) as (stream, _opened): + payload = stream.read(MAX_REQUEST_BYTES + 1) + _require(len(payload) <= MAX_REQUEST_BYTES, "REQUEST_LIMIT") + document = json.loads(payload) + _require( + type(document) is dict + and set(document) == {"protocol", "declaration", "fraction", "seed"}, + "REQUEST_FIELDS", + ) + _require( + document["protocol"] == REQUEST_PROTOCOL + and document["declaration"] == domains.DECLARATION, + "REQUEST_DECLARATION", + ) + pair = document["fraction"] + _require( + type(pair) is list + and len(pair) == 2 + and all(type(v) is int for v in pair) + and pair[1] > 0, + "REQUEST_FRACTION", + ) + fraction = Fraction(*pair) + _require( + 0 < fraction <= 1 and [fraction.numerator, fraction.denominator] == pair, + "REQUEST_FRACTION", + ) + seed = document["seed"] + _require(type(seed) is int and 0 <= seed < 2**64, "REQUEST_SEED") + _require(_encode(document, MAX_REQUEST_BYTES) == payload, "REQUEST_CANONICAL") + return root, document, payload, fraction, seed + + +def read_survey_population_request(source_dir): + """Decode bounded request arguments, without granting source authority.""" + try: + _path, _document, _payload, fraction, seed = _request(source_dir) + return fraction, seed + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("REQUEST_REFUSED") from None + + +def _file_limits(): + # Limits bound only transport. Actual closed owners independently verify + # their pinned source bytes and stricter member/row/receipt contracts. + return { + **{name: asec_native._MAX_CHECKPOINT_BYTES for name in _SOURCE_ROSTER}, + **{ + f"asec/pppub{year}.csv": asec_native.coverage_owner.literal.MAX_MEMBER_BYTES + for year in (23, 24, 25) + }, + "asec/hhpub25.csv": asec_native.anchor_owner._MAX_MEMBER_BYTES, + "selection-request.json": MAX_REQUEST_BYTES, + "acs/csv_hus.zip": acs_catalogue._ARCHIVE_BYTES, + "acs/csv_pus.zip": acs_catalogue._ARCHIVE_BYTES, + } + + +def _source_files(root): + _root(root) + _require(stat.S_ISDIR(root.lstat().st_mode), "SOURCE_ROOT") + found = [] + for directory, expected in ( + (root, {"acs", "asec", "selection-request.json"}), + (root / "acs", {"csv_hus.zip", "csv_pus.zip"}), + ( + root / "asec", + {Path(n).name for n in _SOURCE_ROSTER if n.startswith("asec/")}, + ), + ): + _require(stat.S_ISDIR(directory.lstat().st_mode), "SOURCE_DIRECTORY") + children = tuple(directory.iterdir()) + _require( + {p.name for p in children} == expected and len(children) == len(expected), + "SOURCE_ROSTER", + ) + for path in children: + mode = path.lstat().st_mode + _require(not stat.S_ISLNK(mode), "SOURCE_SYMLINK") + if path.name in {"acs", "asec"} and path.parent == root: + _require(stat.S_ISDIR(mode), "SOURCE_DIRECTORY") + else: + _require(stat.S_ISREG(mode), "SOURCE_REGULAR_FILE") + found.append(path.relative_to(root).as_posix()) + _require(set(found) == set(_SOURCE_ROSTER), "SOURCE_ROSTER") + limits = _file_limits() + _require( + sum( + (root / name).lstat().st_size + for name in ("acs/csv_hus.zip", "acs/csv_pus.zip") + ) + <= acs_catalogue._ARCHIVE_BYTES, + "ARCHIVE_LIMIT", + ) + result = [] + for name in _SOURCE_ROSTER: + path = root / name + digest, size = hashlib.sha256(), 0 + with _regular_reader(path, limits[name]) as (stream, opened): + while block := stream.read(1024**2): + size += len(block) + _require(size <= limits[name], "SOURCE_FILE_LIMIT") + digest.update(block) + _require(size == opened.st_size, "SOURCE_CHANGED") + result.append([name, size, digest.hexdigest()]) + return tuple(tuple(row) for row in result) + + +def _file_stats(root): + _root(root) + result = [] + # Directory ctime/mtime also bind the exact admitted roster across the last + # producer I/O, including a newly added unlisted entry in either source arm. + for directory in (root, root / "acs", root / "asec"): + info = directory.lstat() + _require(stat.S_ISDIR(info.st_mode), "SOURCE_DIRECTORY") + result.append(_stat_identity(info)) + for name in _SOURCE_ROSTER: + path = root / name + _root(path.parent) + info = path.lstat() + _require(stat.S_ISREG(info.st_mode), "SOURCE_REGULAR_FILE") + result.append(_stat_identity(info)) + return tuple(result) + + +def _modules(): + modules = ( + sys.modules[__name__], + domains, + selection, + survey_domain_sample, + spine_assembly, + graph_sources, + graph_context, + graph_population, + observed_age, + acs_catalogue, + asec_catalogue, + acs_native, + acs_native.housing, + acs_native.coverage, + acs_native.coverage.literal, + sys.modules[acs_native.housing.build_acs_pums_unit_frame.__module__], + sys.modules[acs_native.housing.map_acs_native_inputs.__module__], + *asec_native._modules(), + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _runtime_marker(value, depth=0): + """Detach simple callable configuration, retaining opaque dependency identity.""" + _require(depth <= 16, "PRODUCER_CONFIGURATION_DEPTH") + if type(value) in (type(None), bool, int, float, str, bytes): + return type(value), value + if type(value) in (tuple, list): + return type(value), tuple(_runtime_marker(v, depth + 1) for v in value) + if type(value) is dict: + return tuple( + (_runtime_marker(k, depth + 1), _runtime_marker(v, depth + 1)) + for k, v in value.items() + ) + if isinstance(value, FunctionType): + return value, value.__code__ + return type(value), id(value) + + +def _function_seal(function, depth=0): + # contextmanager keeps executable code in both __wrapped__ and a closure; + # the public wrapper's __code__ alone cannot bind that implementation. + _require(depth <= 16, "PRODUCER_WRAPPER_DEPTH") + wrapped = getattr(function, "__wrapped__", None) + return ( + function, + function.__code__, + _runtime_marker(function.__defaults__), + _runtime_marker(function.__kwdefaults__), + tuple(_runtime_marker(c.cell_contents) for c in function.__closure__ or ()), + _function_seal(wrapped, depth + 1) + if isinstance(wrapped, FunctionType) + else _runtime_marker(wrapped), + ) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = _function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = _function_seal( + function + ) + result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) + result["csv"] = (csv.reader, _csv.reader) + result["contract"] = ( + PROTOCOL, + REQUEST_PROTOCOL, + SOURCE_CODEC, + MAX_REQUEST_BYTES, + MAX_PAYLOAD_BYTES, + _MAX_SCALAR_BYTES, + _SOURCE_ROSTER, + domains.DECLARATION, + domains.MAX_MEMBERS, + domains.MAX_HOUSEHOLDS, + domains.MAX_TOTAL_MEMBERS, + selection._BATCH_HOUSEHOLDS, + selection._BATCH_PEOPLE, + observed_age.RULE, + observed_age.AGE_CONVENTION, + observed_age.MAX_ROWS, + observed_age.MAX_EXACT_FLOAT64_INTEGER, + ) + return result + + +def _code_bytes(): + return { + module.__name__: _sha(Path(module.__file__).read_bytes()) + for module in _modules() + } + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return { + "implementation": _BYTES, + "acs": acs_catalogue._producer(), + "asec": _sha(_encode(asec_catalogue._implementation())), + "numpy": np.__version__, + "rng": "numpy.random.PCG64/SeedSequence", + "declaration": domains.DECLARATION, + } + + +def _authority(): + return _encode( + { + "asec": asec_catalogue._source_authority().hex(), + "acs": acs_native.housing._ARCHIVE_PINS, + } + ) + + +def _cell(value): + if ( + value is None + or value is pd.NA + or (isinstance(value, (float, np.floating)) and math.isnan(value)) + ): + return None + if isinstance(value, np.generic): + value = value.item() + _require(type(value) in (str, int, bool, float), "FRAME_CELL_TYPE") + if type(value) is float: + _require(math.isfinite(value), "FRAME_NONFINITE") + return ["float", value.hex()] + return value + + +def _frame_identity(frame): + _require( + isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, + "FRAME_TYPE", + ) + digest = hashlib.sha256() + + def update(value): + digest.update(_encode(value)) + digest.update(b"\n") + + update([list(frame.entities), list(frame.weighted_entities)]) + for entity in frame.entities: + table = frame.table(entity) + _require( + type(table) is pd.DataFrame and not table.columns.has_duplicates, + "FRAME_TABLE", + ) + update( + [ + entity, + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ] + ) + for value in table.index: + update(_cell(value)) + for column in table: + series = table[column] + dtype = series.dtype + update( + [ + column, + str(dtype), + getattr(dtype, "storage", None), + str(getattr(dtype, "na_value", "")), + ] + ) + for value in series: + update(_cell(value)) + update( + [ + str(frame.strata.dtype), + type(frame.strata.index).__name__, + str(frame.strata.index.dtype), + list(frame.strata.index.names), + getattr(frame.strata.dtype, "storage", None), + frame.strata.name, + ] + ) + for value in frame.strata.index: + update(_cell(value)) + for value in frame.strata: + update(_cell(value)) + for entity in frame.weighted_entities: + weights = frame.weights_for(entity) + update( + [ + entity, + weights.kind.value, + str(weights.values.dtype), + list(weights.values.shape), + ] + ) + digest.update(weights.values.tobytes()) + digest.update(graph_context.encode_us_frame_context(frame)) + return digest.hexdigest() + + +def _copy_source(frame): + result = Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + { + e: Weights(frame.weights_for(e).values.copy(), frame.weights_for(e).kind) + for e in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + # Only physical string storage changes on owned copies, before the stack's + # exact shared-dtype check. Actual values and missing masks stay identical. + for entity in result.entities: + for column in result.table(entity): + original = result.table(entity)[column] + if isinstance(original.dtype, pd.StringDtype): + changed = original.astype(dtype_for_token("string")) + _same_values(original, changed) + result.table(entity)[column] = changed + return result + + +def _same_values(left, right): + _require(len(left) == len(right), "ROW_COUNT") + _require(list(map(_cell, left)) == list(map(_cell, right)), "SOURCE_VALUE_CHANGED") + + +def _normalized_source_copy(frame: Frame) -> Frame: + result = _copy_source(frame) + _require("A_AGE" in frame.person, "OBSERVED_AGE_REQUIRED") + result.person["age"] = observed_age.normalize_observed_age( + frame.person["A_AGE"], frame.person.get("age") + ) + _verify_normalized_copy(frame, result) + return result + + +def _verify_normalized_copy(original: Frame, normalized: Frame): + """Only the recorded age alias and existing string storage may differ.""" + _require( + original.schema == normalized.schema + and original.entities == normalized.entities + and original.links == normalized.links == () + and original.metadata == normalized.metadata + and original.mass_log == normalized.mass_log + and original.weighted_entities == normalized.weighted_entities, + "NORMALIZED_FRAME_CONTEXT", + ) + for entity in original.entities: + before, after = original.table(entity), normalized.table(entity) + expected_columns = list(before) + original_axis = after.columns + if entity == "person" and "age" not in before: + expected_columns.append("age") + original_axis = after.columns[:-1] + _require( + list(after) == expected_columns + and before.columns.identical(original_axis) + and before.index.identical(after.index), + "NORMALIZED_ORDERED_COLUMNS", + ) + for column in before: + if entity == "person" and column == "age": + continue # Independently reconstructed from raw observations below. + left, right = before[column], after[column] + if isinstance(left.dtype, pd.StringDtype): + _require( + right.dtype == dtype_for_token("string"), + "NORMALIZED_STRING_DTYPE", + ) + _same_values(left, right) + else: + _require(storage_equal(left, right), "NORMALIZED_SOURCE_STORAGE") + _require( + original.strata.index.identical(normalized.strata.index) + and original.strata.name == normalized.strata.name + and storage_equal(original.strata, normalized.strata), + "NORMALIZED_STRATA", + ) + for entity in original.weighted_entities: + left, right = original.weights_for(entity), normalized.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "NORMALIZED_WEIGHTS", + ) + expected = observed_age.normalize_observed_age( + original.person["A_AGE"], original.person.get("age") + ) + _require( + storage_equal(expected, normalized.person["age"]), "NORMALIZED_AGE_IDENTITY" + ) + + +def _bounded_append(rows, row, budget): + size = len(_encode(row)) + 1 + _require(budget[0] + size <= MAX_PAYLOAD_BYTES, "ORIGIN_LIMIT") + budget[0] += size + rows.append(row) + + +def _origins(frame, sources, selected, native_receipts): + selected_by_key = {(r.key.source.value, r.key.native_id): r for r in selected} + native_hh, native_people = {}, {} + for row in native_receipts["asec"]["households"]: + native_hh[("asec", row["household_id"])] = row["source"]["H_SEQ"] + asec_rows = {row["person_id"]: row for row in native_receipts["asec"]["persons"]} + for channel, source in sources.items(): + for row in source.table("household").itertuples(index=False): + if channel == "acs": + native_hh[(channel, int(row.household_id))] = row.SERIALNO + for row in source.person.itertuples(index=False): + if channel == "acs": + person_key, line = str(row.SPORDER), str(row.SPORDER) + else: + original = asec_rows[int(row.person_id)] + person_key, line = original["PERIDNUM"], str(original["A_LINENO"]) + native_people[(channel, int(row.person_id))] = ( + person_key, + line, + int(row.person_household_id), + ) + maps, entities, households, people, budget = {}, {}, [], [], [0] + household_positions = {} + for channel, source in sources.items(): + ids = source.table("household").household_id + positions = {int(value): index for index, value in enumerate(ids)} + _require(len(positions) == len(ids), "SOURCE_ID_COLLISION") + household_positions[channel] = positions + for entity in frame.entities: + table = frame.table(entity) + ids = US_SCHEMA.entity_id_column(entity) + channels, previous = ( + support_channel_column(entity), + spine_source_id_column(entity), + ) + provenance = set(spine_assembly._support_metadata_columns(entity)) + expected_columns = provenance | { + column for source in sources.values() for column in source.table(entity) + } + _require(set(table) == expected_columns, "UNDECLARED_STACK_COLUMN") + records, mapping = [], {} + for new_id, channel, source_id in table[[ids, channels, previous]].itertuples( + index=False, name=None + ): + key = (channel, int(source_id)) + _require(key not in mapping, "SOURCE_ID_COLLISION") + mapping[key] = int(new_id) + _bounded_append(records, [int(new_id), channel, int(source_id)], budget) + expected = { + (channel, int(i)) + for channel, source in sources.items() + for i in source.table(entity)[ids] + } + _require(set(mapping) == expected, "COMPLETE_ENTITY_ORIGIN") + maps[entity], entities[entity] = mapping, records + for entity in frame.entities: + table = frame.table(entity) + id_column = US_SCHEMA.entity_id_column(entity) + for channel, source in sources.items(): + arm = table.loc[table[support_channel_column(entity)].eq(channel)] + old_ids = arm[spine_source_id_column(entity)].to_numpy() + original = ( + source.table(entity).set_index(id_column, drop=False).loc[old_ids] + ) + for column in original: + if column == id_column: + expected = [ + maps[entity][(channel, int(v))] for v in original[column] + ] + elif entity == "person" and column in { + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + }: + group = next( + e + for e in US_SCHEMA.group_entities + if US_SCHEMA.membership_column(e) == column + ) + expected = [ + maps[group][(channel, int(v))] for v in original[column] + ] + else: + expected = original[column] + _same_values(expected, arm[column]) + for column in ( + set(arm) + - set(original) + - set(spine_assembly._support_metadata_columns(entity)) + ): + _require(arm[column].isna().all(), "ABSENT_SOURCE_VALUE_INVENTED") + if entity == "person": + source_positions = { + int(value): index + for index, value in enumerate(source.person.person_id) + } + expected_strata = source.strata.iloc[ + [source_positions[int(value)] for value in old_ids] + ] + receiving_positions = np.flatnonzero( + table[support_channel_column(entity)].eq(channel).to_numpy() + ) + _same_values(expected_strata, frame.strata.iloc[receiving_positions]) + household_table = frame.table("household") + for position, (new_id, channel, source_id) in enumerate(entities["household"]): + raw = native_hh[(channel, source_id)] + chosen = selected_by_key.pop((channel, raw), None) + _require(chosen is not None, "SELECTED_SOURCE_IDENTITY") + original = chosen.original_design_weight + source = sources[channel] + source_position = household_positions[channel][source_id] + _require( + source.weights_for("household").kind is WeightKind.DESIGN + and frame.weights_for("household").kind is WeightKind.DESIGN, + "ORIGINAL_WEIGHT_KIND", + ) + expected = np.array([float(original)], dtype=np.float64).tobytes() + _require( + source.weights_for("household") + .values[source_position : source_position + 1] + .tobytes() + == expected + == frame.weights_for("household").values[position : position + 1].tobytes(), + "ORIGINAL_ANCHOR_CHANGED", + ) + _bounded_append( + households, + { + "household_id": new_id, + "source": channel, + "source_year": 2024, + "survey_year": 2024 if channel == "acs" else 2025, + "raw_native_id": raw, + "selected_receiving_household_id": source_id, + "original_anchor": _value(original), + }, + budget, + ) + _require( + not selected_by_key and len(households) == len(household_table), + "SELECTED_HOUSEHOLD_ROSTER", + ) + for new_id, channel, source_id in entities["person"]: + person_key, line, household_id = native_people[(channel, source_id)] + _bounded_append( + people, + [ + new_id, + channel, + 2024, + 2024 if channel == "acs" else 2025, + native_hh[(channel, household_id)], + person_key, + line, + source_id, + household_id, + maps["household"][(channel, household_id)], + ], + budget, + ) + return { + "households": households, + "entities": entities, + "persons": { + "columns": [ + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + "selected_receiving_household_id", + "household_id", + ], + "rows": people, + }, + } + + +@dataclass(frozen=True) +class _State: + root: Path + files: tuple + file_stats: tuple + producer: bytes + authority: bytes + catalogues: tuple + native: tuple + attached: tuple + source_frames: tuple + source_frame_seals: tuple + frame: Frame + identity: str + context: bytes + plan: selection.CatalogueSelectionPlan + plan_sha256: str + nested: tuple + + +def _attached(state): + return tuple(value.payload for value in (*state.catalogues, *state.native)) + + +def _nested_seals(catalogues, native): + """Pure final borrowed-object seals after all source/producer file checks. + + These inspect the actual retained issued owners, never construct issuer + authority from decoded records. The ancestor Frames still impose their + existing full-parent verification cost. + """ + acs = acs_native._owned(native[0]) + acs_cat = acs_catalogue._lookup(catalogues[0]) + values = [ + id(acs), + id(acs.prepared), + id(acs.prepared.source), + id(acs.literal), + acs.prepared.receipt_json, + acs.prepared.source.receipt_json, + acs.prepared.source.projection_json, + acs.literal.payload, + _encode( + [ + [str(path) for _, path in sorted(paths.items())] + for paths in acs.snapshots + ] + ), + _digest((acs_cat.records, acs_cat.vacancies)), + ] + for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): + entry = module._ISSUED.get(id(value)) + _require( + entry is not None and entry[0]() is value and entry[1] == value.payload, + "ANCESTOR_ISSUANCE_CHANGED", + ) + state = entry[2] + attached = asec_native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + values.extend( + ( + id(state), + id(state.parent), + id(attached[0]), + *attached[1:], + asec_native._frame_identity(state.parent.frame), + ) + ) + if module is asec_catalogue: + values.append( + asec_catalogue._records_identity(state.households, state.ledger) + ) + return tuple(values) + + +def _pure_final(state): + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require( + _nested_seals(state.catalogues, state.native) == state.nested, + "NESTED_EVIDENCE_CHANGED", + ) + _require(_digest(_plan_document(state.plan)) == state.plan_sha256, "PLAN_CHANGED") + _require( + _frame_identity(state.frame) == state.identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "PREPARED_FRAME_CHANGED", + ) + _require( + tuple( + ( + acs_native._frame_sha256(frame) + if i == 0 + else asec_native._frame_identity(frame) + ) + for i, frame in enumerate(state.source_frames) + ) + == state.source_frame_seals, + "SOURCE_FRAME_CHANGED", + ) + + +def _validate(state): + _pure_final(state) + acs_catalogue.verify_acs_source_catalogue(state.catalogues[0]) + asec_catalogue.verify_asec_source_catalogue(state.catalogues[1]) + acs_native.verify_acs_native_coverage(state.native[0], state.source_frames[0]) + state.native[1].validate() + _require(_source_files(state.root) == state.files, "SOURCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + _require(_file_stats(state.root) == state.file_stats, "SOURCE_STAT_CHANGED") + _pure_final(state) + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedSurveyPopulationPreparation: + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedSurveyPopulationPreparation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from None + + def validate(self): + self._checked() + + def checked_view(self): + """Borrow one checked bundle; the view itself grants no authority.""" + _reference, payload, state = self._checked() + return CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + + @property + def frame(self): + return self._checked()[2].frame + + @property + def context(self): + return self._checked()[2].context + + @property + def selection_plan(self): + return self._checked()[2].plan + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +@dataclass(frozen=True, slots=True) +class CheckedSurveyPopulationView: + """A checked borrow's values, not an issued source or reusable certificate.""" + + payload: bytes + context: bytes + frame: Frame + selection_plan: selection.CatalogueSelectionPlan + receipt: dict + + +def verify_survey_population_preparation(value): + _require( + type(value) is AuthenticatedSurveyPopulationPreparation, "PREPARATION_TYPE" + ) + value._checked() + return value + + +def verify_materialized_survey_population(preparation, frame): + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _validate(entry[2]) + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _require(preparation.payload == entry[1], "UNISSUED_OR_CHANGED") + + +def prepare_authenticated_survey_population( + source_dir, *, snapshot_root, fraction, seed, candidate=None +): + """Reconstruct actual source authority before accepting candidate bytes.""" + try: + _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") + _require( + candidate is None or len(candidate) <= MAX_PAYLOAD_BYTES, "CANDIDATE_LIMIT" + ) + root, request, request_bytes, requested_fraction, requested_seed = _request( + source_dir + ) + _require( + type(fraction) is Fraction + and fraction == requested_fraction + and type(seed) is int + and seed == requested_seed, + "REQUEST_ARGUMENTS", + ) + snapshots = _root(snapshot_root, allow_missing=True) + _require( + not snapshots.is_relative_to(root) and not root.is_relative_to(snapshots), + "SNAPSHOT_LOCATION", + ) + snapshots.mkdir(parents=True, exist_ok=True) + authority, files_before, producer = ( + _authority(), + _source_files(root), + _encode(_producer()), + ) + _require( + files_before[0] + == ("selection-request.json", len(request_bytes), _sha(request_bytes)), + "REQUEST_CHANGED", + ) + file_stats = _file_stats(root) + kwargs = dict( + parent_path=root / "asec/parent.h5", + household_attachment_path=root / "asec/household-attachment.h5", + person_income_attachment_path=root / "asec/person-income-attachment.h5", + person_member_paths={ + 2022: root / "asec/pppub23.csv", + 2023: root / "asec/pppub24.csv", + 2024: root / "asec/pppub25.csv", + }, + household_member_path=root / "asec/hhpub25.csv", + ) + acs = acs_catalogue.issue_acs_source_catalogue( + root / "acs", snapshot_root=snapshots + ) + asec = asec_catalogue.issue_asec_source_catalogue(**kwargs) + acs_rows, asec_rows = acs.households, asec.households + plan = selection.plan_catalogue_selection( + acs_households=acs_rows, + asec_households=asec_rows, + fraction=fraction, + seed=seed, + ) + acs_keys = tuple( + r.key.native_id for r in plan.selected if r.key.source is domains.Source.ACS + ) + asec_keys = tuple( + (2024, int(r.key.native_id)) + for r in plan.selected + if r.key.source is domains.Source.ASEC + ) + _require( + acs_keys and asec_keys and len(set(asec_keys)) == len(asec_keys), + "SELECTED_SOURCE_SUPPORT", + ) + actual_acs = acs_native.issue_acs_native_coverage( + root / "acs", snapshot_root=snapshots, serialnos=acs_keys + ) + actual_asec = asec_native.load_authenticated_asec_2024_native_population( + **kwargs, selected_households=asec_keys + ) + native_frames = actual_acs.frame, actual_asec.frame + source_seals = ( + acs_native._frame_sha256(native_frames[0]), + asec_native._frame_identity(native_frames[1]), + ) + source_copies = { + channel: _normalized_source_copy(frame) + for channel, frame in zip(("acs", "asec"), native_frames, strict=True) + } + stacked = spine_assembly.stack_survey_spines(source_copies) + frame = stacked.frame + transitions = graph_sources._canonical_assembly( + frame, tuple(source_copies.values()) + ) + context = graph_context.encode_us_frame_context(frame) + native_receipts = {"acs": actual_acs.receipt, "asec": actual_asec.receipt} + for channel, original in zip(("acs", "asec"), native_frames, strict=True): + _verify_normalized_copy(original, source_copies[channel]) + origins = _origins(frame, source_copies, plan.selected, native_receipts) + origins["storage_transitions"] = list(transitions) + origins["observed_age_normalization"] = { + "rule": observed_age.rule_document(), + "sources": { + channel: { + "native_frame_sha256": source_seals[i], + "normalized_frame_sha256": _frame_identity(source_copies[channel]), + "common_age_preexisting": "age" in native_frames[i].person, + } + for i, channel in enumerate(("acs", "asec")) + }, + } + documents = {"acs": acs.receipt, "asec": asec.receipt} + catalogues = { + "acs": { + "receipt_sha256": _sha(acs.to_bytes()), + "records_sha256": documents["acs"]["counts"]["canonical_record_sha256"], + "counts": documents["acs"]["counts"], + }, + "asec": { + "receipt_sha256": _sha(asec.to_bytes()), + "records_sha256": documents["asec"]["records"]["sha256"], + "counts": documents["asec"]["counts"], + }, + } + identity = _frame_identity(frame) + payload = _encode( + { + "protocol": PROTOCOL, + "request": request, + "request_sha256": _sha(request_bytes), + "source_files": files_before, + "producer": json.loads(producer), + "catalogues": catalogues, + "native": { + channel: { + "receipt_sha256": _sha(value.payload), + "frame_sha256": source_seals[i], + "households": native_frames[i].n("household"), + "persons": native_frames[i].n("person"), + } + for i, (channel, value) in enumerate( + (("acs", actual_acs), ("asec", actual_asec)) + ) + }, + "selection": _plan_document(plan), + "origins": origins, + "frame_sha256": identity, + "context_sha256": _sha(context), + "release_eligible": False, + } + ) + state = _State( + root, + files_before, + file_stats, + producer, + authority, + (acs, asec), + (actual_acs, actual_asec), + tuple(value.payload for value in (acs, asec, actual_acs, actual_asec)), + native_frames, + source_seals, + frame, + identity, + context, + plan, + _digest(_plan_document(plan)), + _nested_seals((acs, asec), (actual_acs, actual_asec)), + ) + _validate(state) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + result = AuthenticatedSurveyPopulationPreparation(payload, _token=_TOKEN) + key = id(result) + + def cleanup(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, cleanup), payload, state) + _pure_final(state) + _require( + type(result.payload) is bytes + and result.payload == payload + and _ISSUED[key][0]() is result, + "UNISSUED_OR_CHANGED", + ) + return result + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("PREPARATION_ISSUANCE_REFUSED") from None + + +_BYTES = _code_bytes() + + +def _current_survey_wage_projection(preparation, entry): + """Project one just-checked retained owner; bytes grant no source authority. + + The caller must have obtained ``entry`` from ``preparation._checked()`` in + the current operation, and must seal its receiving Populations afterwards. + The one ready() call retains the existing all-field/full-parent admission. + No old reported-income contract or prior-wage column is manufactured. + """ + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation + and _ISSUED.get(id(preparation)) is entry + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1], + "WAGE_PREPARATION_ISSUANCE", + ) + state = entry[2] + native_entry = asec_native._ISSUED.get(id(state.native[1])) + _require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "WAGE_NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + ready = parent.ready() # Exactly once, outside both row and feature loops. + field = ready.field("WSAL_VAL") + domain = next(d for d in parent.spec.fields if d.name == "WSAL_VAL") + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + _require(len(positions) == len(scope.person_ids), "WAGE_PARENT_ROSTER") + native_ids = set(state.source_frames[1].person.person_id) + rows, budget = [], [0] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + _require( + set(selected[spine_source_id_column("person")]) == native_ids, + "WAGE_SELECTED_ROSTER", + ) + for stacked, native in selected[ + ["person_id", spine_source_id_column("person")] + ].itertuples(index=False, name=None): + _require(int(native) in positions, "WAGE_PARENT_MEMBER") + i = positions[int(native)] + _require(scope.person_years[i] == 2024, "WAGE_CURRENT_COHORT") + amount = field.amount_bytes[8 * i : 8 * (i + 1)] + number = np.frombuffer(amount, dtype="= 0, + "WAGE_CURRENT_FEATURE", + ) + _bounded_append( + rows, + [ + int(stacked), + int(native), + 2024, + amount.hex(), + field.status_bytes[i], + field.validity_bytes[i], + field.zero_origin_bytes[i], + ], + budget, + ) + document = { + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(native_entry[1]), + "money_header": json.loads(ready.header), + "money_header_sha256": _sha(ready.header), + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "zero_origin_code": int(current_money._origin_code(parent.spec, "WSAL_VAL")), + "columns": [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "rows": rows, + } + payload = _encode(document) + # ready() may perform I/O. Recheck the actual retained entries and all pure + # owner seals after it; this does not reread the national catalogues. + _pure_final(state) + _require( + _ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and asec_native._ISSUED.get(id(state.native[1])) is native_entry + and state.native[1].payload == native_entry[1] + and native_entry[2].parent is parent, + "WAGE_FINAL_ISSUANCE", + ) + return payload + + +_LIVE = _live() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py new file mode 100644 index 000000000..dd55a5e65 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py @@ -0,0 +1,208 @@ +"""Directional ContentStore replay checks, never source or population authority. + +The caller must separately retain the original source/population seals. These +comparisons permit the store's missing-buffer normalization and scalar object +encoding only at a named materialization boundary. They are not mutation seals. +""" + +from __future__ import annotations + +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import MassRecord, Population +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_POPULATION_REPLAY_" + reason) + + +def _array_bytes_equal(left, right): + return ( + type(left) is np.ndarray + and type(right) is np.ndarray + and left.dtype == right.dtype + and not left.dtype.hasobject + and left.shape == right.shape + and left.tobytes() == right.tobytes() + ) + + +def _object_bytes(value): + try: + return _encode_object_scalar(value) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_OBJECT") from None + + +def _series(expected, actual): + _require( + type(expected) is pd.Series + and type(actual) is pd.Series + and type(expected.dtype) is type(actual.dtype) + and expected.dtype == actual.dtype + and len(expected) == len(actual), + "SERIES_DTYPE_OR_LENGTH", + ) + dtype = expected.dtype + left, right = expected.array, actual.array + if isinstance(dtype, pd.BooleanDtype) or ( + isinstance(dtype, pd.api.extensions.ExtensionDtype) + and pd.api.types.is_integer_dtype(dtype) + ): + # The v1 country profile supports pandas' actual masked integer/bool + # storage. Other extension-array implementations must be reviewed first. + ld, rd = getattr(left, "_data", None), getattr(right, "_data", None) + lm, rm = getattr(left, "_mask", None), getattr(right, "_mask", None) + _require( + _array_bytes_equal(lm, rm) + and lm.dtype == np.dtype(np.bool_) + and lm.shape == (len(expected),) + and type(ld) is np.ndarray + and type(rd) is np.ndarray + and ld.dtype == rd.dtype + and not ld.dtype.hasobject + and ld.shape == rd.shape == lm.shape, + "MASKED_STORAGE", + ) + _require(_array_bytes_equal(ld[~lm], rd[~rm]), "PRESENT_BITS") + _require( + _array_bytes_equal(ld, rd) or bool(np.all(rd[rm] == 0)), + "NONCANONICAL_NULL_BACKING", + ) + return + if isinstance(dtype, pd.StringDtype): + _require( + dtype.storage == actual.dtype.storage + and (dtype.na_value is pd.NA) == (actual.dtype.na_value is pd.NA), + "STRING_POLICY", + ) + lm = expected.isna().to_numpy(dtype=np.bool_, copy=False) + rm = actual.isna().to_numpy(dtype=np.bool_, copy=False) + _require(_array_bytes_equal(lm, rm), "STRING_MASK") + for missing, a, b in zip(lm, expected, actual, strict=True): + if not missing: + _require(type(a) is str and type(b) is str and a == b, "STRING_VALUE") + return + if pd.api.types.is_object_dtype(dtype): + # Reuse the store's typed scalar bytes. None/pd.NA/NaT, bool/int, + # float signed zero and NaN payloads remain distinguished by that codec. + # Object pointers and np scalar wrappers are not portable identities. + for a, b in zip(expected, actual, strict=True): + _require(_object_bytes(a) == _object_bytes(b), "OBJECT_VALUE") + return + _require( + not isinstance(dtype, pd.api.extensions.ExtensionDtype), + "UNSUPPORTED_EXTENSION_DTYPE", + ) + _require( + _array_bytes_equal(expected.to_numpy(copy=False), actual.to_numpy(copy=False)), + "NATIVE_BITS", + ) + + +def _name_bytes(value): + try: + return canonical_json(_axis_name_payload(value)) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_AXIS_NAME") from None + + +def _axis(expected, actual): + _require( + type(expected) is type(actual) + and expected.identical(actual) + and not isinstance(expected, pd.MultiIndex), + "AXIS", + ) + _require(_name_bytes(expected.name) == _name_bytes(actual.name), "AXIS_NAME") + _series(pd.Series(expected.array, copy=False), pd.Series(actual.array, copy=False)) + + +def same_replayed_frame(expected: Frame, actual: Frame) -> None: + """Compare actual Frames, allowing only the named store representation rule. + + No I/O or allocation of another Frame. Temporary buffers are proportional + to individual columns and context metadata, without a total input RAM cap. + """ + _require(isinstance(expected, Frame) and isinstance(actual, Frame), "FRAME_TYPE") + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links == () + and canonical_json(_encode_frame_metadata(expected.metadata)) + == canonical_json(_encode_frame_metadata(actual.metadata)) + and all( + type(r) is MassChangeRecord for r in (*expected.mass_log, *actual.mass_log) + ) + and canonical_json([asdict(r) for r in expected.mass_log]) + == canonical_json([asdict(r) for r in actual.mass_log]) + and expected.weighted_entities == actual.weighted_entities, + "FRAME_CONTEXT", + ) + for entity in expected.entities: + left, right = expected.table(entity), actual.table(entity) + _require( + type(left) is pd.DataFrame + and type(right) is pd.DataFrame + and left.flags == right.flags, + "TABLE_TYPE_OR_FLAGS", + ) + _axis(left.index, right.index) + _axis(left.columns, right.columns) + for column in left: + _series(left[column], right[column]) + _axis(expected.strata.index, actual.strata.index) + _require( + _name_bytes(expected.strata.name) == _name_bytes(actual.strata.name), + "STRATA_NAME", + ) + _series(expected.strata, actual.strata) + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind and _array_bytes_equal(left.values, right.values), + "WEIGHT_BYTES", + ) + + +def same_replayed_population(expected: Population, actual: Population) -> None: + """Require complete receiving state as well as the directional Frame check.""" + _require( + type(expected) is Population and type(actual) is Population, "POPULATION_TYPE" + ) + same_replayed_frame(expected.frame, actual.frame) + _require( + type(expected.version) is type(actual.version) is str + and expected.version == actual.version + and tuple(sorted(expected.owners.items())) + == tuple(sorted(actual.owners.items())) + and all(type(v) is str for v in actual.owners.values()) + and tuple(expected.weight_kind.items()) == tuple(actual.weight_kind.items()) + and all(type(v) is WeightKind for v in actual.weight_kind.values()) + and type(expected.mass_ledger) is type(actual.mass_ledger) is tuple + and all( + type(r) is MassRecord for r in (*expected.mass_ledger, *actual.mass_ledger) + ) + and canonical_json([asdict(r) for r in expected.mass_ledger]) + == canonical_json([asdict(r) for r in actual.mass_ledger]) + and tuple(expected.design_weights) == tuple(actual.design_weights), + "POPULATION_CONTEXT", + ) + for entity in expected.design_weights: + _require( + _array_bytes_equal( + expected.design_weights[entity], actual.design_weights[entity] + ), + "DESIGN_BYTES", + ) diff --git a/packages/microcosm-build/tests/test_us_acs_housing_source.py b/packages/microcosm-build/tests/test_us_acs_housing_source.py new file mode 100644 index 000000000..eb5a461c9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_housing_source.py @@ -0,0 +1,2634 @@ +"""Independent invented-source tests for the approved ACS housing-unit source bridge. + +Written against the approved v2 plan contract, not against the production +module: no byte of ``acs_housing_universe_source.py`` or of the packaged +``acs_2024_housing_universe.json`` was read while authoring this file, so a +failure here is evidence about the contract rather than a restatement of the +implementation. + +Every archive, member, header, identifier and row below is wholly invented. No +genuine ACS/Census archive, candidate, target or dictionary file is opened, +statted or referenced. Nothing here authenticates real data, and nothing here +makes a scientific claim: the fixtures exercise the closed source rules only. + +Scope is the source producer/readback pair. ``prepare_acs_housing_population`` +builds a Frame and is deliberately left to the production author's own +Frame/graph tests. + +Two reconciliation seams are collected in one place each, so integrating this +file with the finished module is a rename exercise and never a semantic one: + +* ``NAMES`` - column, constant, filename and definition-name spellings. +* ``EXPECTED_REASONS`` - exact refusal reason codes, asserted only once + ``REASON_CODES_AGREED`` is flipped to True after agreeing them with the + production author. Until then every refusal test still asserts the exception + type, message sanitisation, and that semantically distinct causes carry + distinct codes, which is the part that does not need agreement. +""" + +import csv +import hashlib +import io +import json +import os +import re +import signal +import stat +import struct +import tempfile +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +# The production import must surface as a loud collection error naming the +# module while it is absent. It must never skip: no engine is involved here. +from microcosm.build.us_runtime import acs_housing_universe_source as source +from microcosm.build.us_runtime.acs_housing_universe import ( + classify_acs_housing_universe, +) + +# -------------------------------------------------------------------------- +# Reconciliation seam 1: identifier spellings. +# -------------------------------------------------------------------------- + +NAMES = { + # Frame columns carrying the archive-member/row lineage of each row. + "member_column": "source_member", + "ordinal_column": "source_row_ordinal", + # The normalised two-digit state column exposed on the household frame. + # ``_state_series`` falls back to the raw ST/STATE header aliases, so only + # the dedicated naming test depends on this spelling. + "state_column": "ST", + # Files written into the source artifact directory. + "projection_filename": "projection.json", + "receipt_filename": "receipt.json", + # Named source definitions bound by the artifact receipt. + "vacancy_definition": "acs_2024_source_vacancy_v1", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + # Module byte caps, monkeypatched tiny so no test allocates real GiBs. + "source_max_bytes": "ACS_HU_SOURCE_MAX_BYTES", + "receipt_max_bytes": "ACS_HU_RECEIPT_MAX_BYTES", + # The private archive pin tuple; entries are (role, filename, sha256, size). + "archive_pins": "_ARCHIVE_PINS", + "household_role": "household", + "person_role": "person", +} + +HOUSEHOLD_ARCHIVE = "csv_hus.zip" +PERSON_ARCHIVE = "csv_pus.zip" + +# -------------------------------------------------------------------------- +# Reconciliation seam 2: refusal reason codes. +# -------------------------------------------------------------------------- + +REASON_CODES_AGREED = True + +EXPECTED_REASONS: dict[str, str] = { + "artifact_forgery": "RECONSTRUCTION_MISMATCH", + "capture_digest": "SOURCE_SHA256", + "csv_record": "CSV_ROW_WIDTH", + "header_contract": "CSV_STATE_REQUIRED", + "join_integrity": "ORPHAN_PERSON", + "output_precondition": "OUTPUT_EXISTS", + "selection": "SELECTION_UNKNOWN", + "value_domain": "TEN", + "zip_preflight": "ZIP_DUPLICATE_MEMBER", +} + +# Coarse refusal families that must not collapse onto one reason code. +DISTINCT_FAMILIES = ( + "capture_digest", + "zip_preflight", + "csv_record", + "header_contract", + "value_domain", + "join_integrity", + "selection", + "artifact_forgery", + "output_precondition", +) + +# -------------------------------------------------------------------------- +# Invented source bytes. +# -------------------------------------------------------------------------- + +HOUSEHOLD_HEADER = ( + "RT", + "SERIALNO", + "DIVISION", + "PUMA", + "REGION", + "ST", + "ADJHSG", + "ADJINC", + "WGTP", + "NP", + "TYPEHUGQ", + "ACR", + "BDSP", + "TEN", + "VALP", + "HINCP", +) + +PERSON_HEADER = ( + "RT", + "SERIALNO", + "SPORDER", + "PUMA", + "ST", + "ADJINC", + "PWGTP", + "AGEP", + "SEX", + "WAGP", +) + +HOUSEHOLD_DEFAULTS = { + "RT": "H", + "DIVISION": "9", + "PUMA": "00100", + "REGION": "4", + "ST": "06", + "ADJHSG": "1000000", + "ADJINC": "1010000", + "ACR": "1", + "BDSP": "3", + "VALP": "410000", + "HINCP": "82000", +} + +PERSON_DEFAULTS = { + "RT": "P", + "PUMA": "00100", + "ST": "06", + "ADJINC": "1010000", + "AGEP": "41", + "SEX": "1", + "WAGP": "52000", +} + +# Projected observation rosters, per the closed plan. +HOUSEHOLD_ROSTER = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +PERSON_ROSTER = ("SERIALNO", "SPORDER", "PWGTP") + +# Fixed classifier output order, read from the pinned pure module. +CODE_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) + + +def household_row(serialno, typehugq, persons, ten, wgtp, **overrides): + """One invented household record as an ordered tuple of lexical tokens.""" + values = dict(HOUSEHOLD_DEFAULTS) + values.update(SERIALNO=serialno, TYPEHUGQ=typehugq, NP=persons, TEN=ten, WGTP=wgtp) + values.update(overrides) + return tuple(values[name] for name in HOUSEHOLD_HEADER) + + +def person_row(serialno, sporder, pwgtp, **overrides): + """One invented person record as an ordered tuple of lexical tokens.""" + values = dict(PERSON_DEFAULTS) + values.update(SERIALNO=serialno, SPORDER=sporder, PWGTP=pwgtp) + values.update(overrides) + return tuple(values[name] for name in PERSON_HEADER) + + +# Eight invented households covering every accepted source class exactly once, +# plus both WGTP boundaries and the whole observed TEN domain. +BASE_HOUSEHOLDS = ( + # occupied owner with a mortgage, two people + household_row("2024HU0000001", "1", "2", "1", "118", PUMA="00100", ST="06"), + # occupied renter paying rent, one person + household_row("2024HU0000002", "1", "1", "3", "76", PUMA="03701", ST="06"), + # occupied housing unit whose tenure is not available, three people + household_row("2024HU0000003", "1", "3", "", "1", PUMA="81003", ST="36"), + # vacant housing unit: no people, blank tenure, positive weight + household_row("2024HU0000004", "1", "0", "", "55", PUMA="00100", ST="11"), + # institutional group quarters placeholder + household_row("2024GQ0000005", "2", "1", "", "0", PUMA="03701", ST="36"), + # noninstitutional group quarters placeholder + household_row("2024GQ0000006", "3", "1", "", "0", PUMA="00100", ST="01"), + # occupied owner outright + household_row("2024HU0000007", "1", "1", "2", "200", PUMA="00100", ST="01"), + # occupied renter paying no rent, maximum housing weight + household_row("2024HU0000008", "1", "1", "4", "9999", PUMA="81003", ST="11"), +) + +BASE_PERSONS = ( + person_row("2024HU0000001", "01", "121", WAGP="52000"), + person_row("2024HU0000001", "02", "97", WAGP="0"), + person_row("2024HU0000002", "1", "80", WAGP="31000"), + person_row("2024HU0000003", "1", "12", WAGP="17000"), + person_row("2024HU0000003", "2", "11", WAGP="0"), + person_row("2024HU0000003", "3", "9", WAGP="0"), + person_row("2024GQ0000005", "01", "64", WAGP="0"), + person_row("2024GQ0000006", "1", "9999", WAGP="7000"), + person_row("2024HU0000007", "01", "205", WAGP="99000"), + person_row("2024HU0000008", "1", "1", WAGP="4000"), +) + +# The base population is split across two members per role so that member and +# ordinal lineage is exercised by every ordinary test rather than by one. +BASE_HOUSEHOLD_MEMBERS = ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:5]), + ("psam_husb.csv", BASE_HOUSEHOLDS[5:]), +) +BASE_PERSON_MEMBERS = ( + ("psam_pusa.csv", BASE_PERSONS[:6]), + ("psam_pusb.csv", BASE_PERSONS[6:]), +) + +BASE_SERIALNOS = tuple( + row[HOUSEHOLD_HEADER.index("SERIALNO")] for row in BASE_HOUSEHOLDS +) + +# The plan fixes what each row contains, never the order rows are emitted in. +# Content assertions below are therefore keyed by identifier, and the ordering +# decision lives in exactly one place: this constant and the two tests that +# read it. "sorted_serialno" is the order the production module emits; +# "source" would be member-then-row-ordinal order. See the author report. +CANONICAL_ROW_ORDER = "sorted_serialno" + + +def expected_row_order(serialnos): + if CANONICAL_ROW_ORDER == "sorted_serialno": + return sorted(serialnos) + if CANONICAL_ROW_ORDER == "source": + return list(serialnos) + raise AssertionError(f"unknown row-order contract {CANONICAL_ROW_ORDER!r}") + + +# (occupied_hu, hu_tenure_class, tenure_subtype, unresolved_reasons, +# physical_unit, TEN_valid) recomputed by hand from the pinned classifier. +BASE_EXPECTED_CODES = { + "interview_scope": [0, 0, 0, 0, 0, 0, 0, 0], + "physical_unit": [1, 1, 1, 1, 2, 2, 1, 1], + "household_kind": [0, 0, 0, 0, 0, 0, 0, 0], + "tenure_subtype": [1, 3, 0, 0, 0, 0, 2, 4], + "occupied_hu": [1, 1, 1, 0, 2, 2, 1, 1], + "hu_tenure_class": [1, 2, 3, 0, 0, 0, 1, 2], + "unresolved_reasons": [0, 0, 2, 1, 0, 0, 0, 0], + "TEN_valid": [1, 1, 0, 0, 0, 0, 1, 1], +} + + +# -------------------------------------------------------------------------- +# Building invented archives. +# -------------------------------------------------------------------------- + + +EXPECTED_HOUSEHOLDS = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: { + name: row[HOUSEHOLD_HEADER.index(name)] for name in HOUSEHOLD_ROSTER + ("ST",) + } + for row in BASE_HOUSEHOLDS +} + +EXPECTED_PERSONS = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): row[PERSON_HEADER.index("PWGTP")] + for row in BASE_PERSONS +} + +EXPECTED_CODES = { + serialno: tuple(BASE_EXPECTED_CODES[name][index] for name in CODE_COLUMNS) + for index, serialno in enumerate(BASE_SERIALNOS) +} + +EXPECTED_HOUSEHOLD_MEMBER = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: member + for member, rows in BASE_HOUSEHOLD_MEMBERS + for row in rows +} + +EXPECTED_PERSON_MEMBER = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): member + for member, rows in BASE_PERSON_MEMBERS + for row in rows +} + + +def csv_bytes(header, rows, *, newline="\n", trailing_newline=True): + """Serialise invented tokens without any quoting the fixtures do not ask for.""" + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, lineterminator=newline, quoting=csv.QUOTE_MINIMAL) + writer.writerow(header) + writer.writerows(rows) + text = buffer.getvalue() + if not trailing_newline and text.endswith(newline): + text = text[: -len(newline)] + return text.encode("utf-8") + + +@dataclass(frozen=True) +class Member: + """One planned archive entry, data or auxiliary.""" + + name: str + data: bytes + compress_type: int = zipfile.ZIP_DEFLATED + external_attr: int | None = None + create_system: int = 3 + + +def data_members(header, members, *, newline="\n", **kwargs): + return tuple( + Member(name, csv_bytes(header, rows, newline=newline, **kwargs)) + for name, rows in members + ) + + +def write_archive(path, members): + """Write invented members exactly as planned, preserving order and duplicates.""" + with zipfile.ZipFile(path, "w") as archive: + for member in members: + info = zipfile.ZipInfo(member.name, date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = member.compress_type + info.create_system = member.create_system + info.external_attr = ( + member.external_attr + if member.external_attr is not None + else (0o644 << 16) + ) + archive.writestr(info, member.data) + return path + + +def flip_archive_byte(path, offset=None): + """Change one compressed byte in place: same size, different digest. + + A deflated archive holds no plaintext, so a token replacement here would + silently do nothing and the test would stop testing what it claims. + """ + payload = Path(path).read_bytes() + index = len(payload) // 2 if offset is None else offset + mutated = payload[:index] + bytes([payload[index] ^ 0xFF]) + payload[index + 1 :] + assert len(mutated) == len(payload) and mutated != payload + Path(path).write_bytes(mutated) + return payload + + +def digest_and_size(path): + payload = Path(path).read_bytes() + return hashlib.sha256(payload).hexdigest(), len(payload) + + +@dataclass +class Fixture: + """An invented source directory plus everything a test needs to drive it.""" + + tmp_path: Path + source_dir: Path + snapshot_root: Path + monkeypatch: pytest.MonkeyPatch + households: tuple = () + persons: tuple = () + secrets: list = field(default_factory=list) + _output_index: int = 0 + + @property + def household_zip(self): + return self.source_dir / HOUSEHOLD_ARCHIVE + + @property + def person_zip(self): + return self.source_dir / PERSON_ARCHIVE + + def pin(self): + """Point the private pin tuple at the invented archives as they are now.""" + pins = [] + for role, name in ( + (NAMES["household_role"], HOUSEHOLD_ARCHIVE), + (NAMES["person_role"], PERSON_ARCHIVE), + ): + path = self.source_dir / name + sha, size = digest_and_size(path) + pins.append((role, name, sha, size)) + self.monkeypatch.setattr(source, NAMES["archive_pins"], tuple(pins)) + return tuple(pins) + + def output_dir(self, label="artifact"): + self._output_index += 1 + return self.tmp_path / f"out-{self._output_index}-{label}" + + def produce(self, *, output_dir=None, serialnos=None, source_dir=None): + return source.produce_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + snapshot_root=self.snapshot_root, + output_dir=self.output_dir() if output_dir is None else output_dir, + serialnos=serialnos, + ) + + def load(self, output_dir, *, serialnos=None, source_dir=None): + return source.load_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + output_dir, + snapshot_root=self.snapshot_root, + serialnos=serialnos, + ) + + def produce_and_load(self, *, serialnos=None): + output_dir = self.output_dir() + produced = self.produce(output_dir=output_dir, serialnos=serialnos) + return produced, self.load(output_dir, serialnos=serialnos), output_dir + + +def build_fixture( + tmp_path, + monkeypatch, + *, + households=BASE_HOUSEHOLD_MEMBERS, + persons=BASE_PERSON_MEMBERS, + household_members=None, + person_members=None, + household_header=HOUSEHOLD_HEADER, + person_header=PERSON_HEADER, + newline="\n", + pin=True, +): + """Assemble an invented source directory; explicit members override the rows.""" + # Resolve first: on macOS pytest's tmp_path sits under /var, a symlink to + # /private/var, and the capture refuses symlinked path components. The + # deliberate symlink refusals below build their own links explicitly. + tmp_path = Path(tmp_path).resolve() + source_dir = Path(tmp_path) / "acs-source" + source_dir.mkdir(parents=True, exist_ok=True) + snapshot_root = Path(tmp_path) / "snapshots" + snapshot_root.mkdir(parents=True, exist_ok=True) + + if household_members is None: + household_members = data_members(household_header, households, newline=newline) + if person_members is None: + person_members = data_members(person_header, persons, newline=newline) + + write_archive(source_dir / HOUSEHOLD_ARCHIVE, household_members) + write_archive(source_dir / PERSON_ARCHIVE, person_members) + + fixture = Fixture( + tmp_path=Path(tmp_path), + source_dir=source_dir, + snapshot_root=snapshot_root, + monkeypatch=monkeypatch, + households=households, + persons=persons, + ) + fixture.secrets = sanitisation_secrets(fixture) + if pin: + fixture.pin() + return fixture + + +def sanitisation_secrets(fixture): + """Strings a sanitised refusal must never echo back to the caller.""" + secrets = [ + str(fixture.tmp_path), + str(fixture.source_dir), + str(fixture.snapshot_root), + ] + secrets.extend(BASE_SERIALNOS) + # Unprojected invented money tokens: no legitimate reason code names these, + # unlike the domain bounds (1..9999, 100..81003) a code may legitimately cite. + secrets.extend(["82000", "52000", "410000", "31000", "99000", "17000"]) + return secrets + + +@pytest.fixture(autouse=True) +def _invented_disk_budget(monkeypatch): + """Eight invented rows must not require a full-source disk reserve in CI.""" + monkeypatch.setattr( + source.shutil, "disk_usage", lambda _path: SimpleNamespace(free=64 * 1024**3) + ) + + +@pytest.fixture +def acs(tmp_path, monkeypatch): + """The plain accepted invented population, pinned and ready to produce.""" + return build_fixture(tmp_path, monkeypatch) + + +# -------------------------------------------------------------------------- +# Refusal helpers. +# -------------------------------------------------------------------------- + + +def refuses(fixture, call, *args, **kwargs): + """Assert the exact refusal type, capture the reason, and check sanitisation.""" + with pytest.raises(source.ACSHousingSourceError) as caught: + call(*args, **kwargs) + reason = str(caught.value) + assert type(caught.value) is source.ACSHousingSourceError + assert reason, "a refusal must carry a reason" + assert len(reason) <= 120, "a reason code must not be prose" + assert "\n" not in reason + for secret in fixture.secrets: + assert secret not in reason, "refusal leaked invented source content" + return reason + + +def flat_scalars(payload): + """Every scalar reachable in a decoded receipt, for shape-tolerant assertions.""" + found = [] + stack = [payload] + while stack: + item = stack.pop() + if isinstance(item, dict): + stack.extend(item.keys()) + stack.extend(item.values()) + elif isinstance(item, (list, tuple)): + stack.extend(item) + else: + found.append(item) + return found + + +def flat_strings(payload): + return {item for item in flat_scalars(payload) if isinstance(item, str)} + + +def flat_numbers(payload): + return { + item + for item in flat_scalars(payload) + if isinstance(item, int) and not isinstance(item, bool) + } + + +def state_series(households): + """Read the state column under whichever spelling the module exposes.""" + for name in (NAMES["state_column"], "ST", "STATE"): + if name in households.columns: + return households[name] + raise AssertionError("household frame exposes no state column") + + +def household_keys(households): + keys = lexical(households["SERIALNO"]) + assert len(set(keys)) == len(keys), "household identifiers must be unique" + return keys + + +def household_map(households, names=HOUSEHOLD_ROSTER + ("ST",)): + """Every projected household token, keyed by identifier rather than by row.""" + keys = household_keys(households) + columns = { + name: lexical(state_series(households) if name == "ST" else households[name]) + for name in names + } + return { + key: {name: columns[name][row] for name in names} + for row, key in enumerate(keys) + } + + +def person_map(persons): + """Person weight keyed by identifier and the raw SPORDER token.""" + entries = list( + zip( + lexical(persons["SERIALNO"]), + lexical(persons["SPORDER"]), + lexical(persons["PWGTP"]), + strict=True, + ) + ) + keyed = {(serialno, sporder): weight for serialno, sporder, weight in entries} + assert len(keyed) == len(entries), "person keys must be unique" + return keyed + + +def codes_map(result): + """The eight native codes as a tuple per household identifier.""" + keys = household_keys(result.households) + codes = result.codes + return { + key: tuple(int(codes[name].iloc[row]) for name in CODE_COLUMNS) + for row, key in enumerate(keys) + } + + +def lexical(series): + """A projected column must be lexical text with no NA conversion.""" + values = series.tolist() + assert all(isinstance(value, str) for value in values), ( + f"{series.name} is not lexical: {[type(v).__name__ for v in values]}" + ) + return values + + +def strict_arrays(households): + """Rebuild the classifier inputs from the projected lexical household tokens.""" + tenure_tokens = lexical(households["TEN"]) + tenure_valid = np.asarray([token != "" for token in tenure_tokens], dtype=bool) + frame = pd.DataFrame( + { + "TYPEHUGQ": np.asarray( + [int(token) for token in lexical(households["TYPEHUGQ"])], dtype="int64" + ), + "NP": np.asarray( + [int(token) for token in lexical(households["NP"])], dtype="int64" + ), + "TEN": np.asarray( + [int(token) if token != "" else 0 for token in tenure_tokens], + dtype="int64", + ), + }, + index=households.index, + ) + return frame, tenure_valid + + +def assert_lineage(frame, keys, expected_member): + """Find member/ordinal lineage without depending on either column spelling. + + Order-independent by construction: the member column must agree row by row + with the invented layout, and each member's ordinals must form one + consecutive run, whatever order the rows are emitted in. + """ + members = [expected_member[key] for key in keys] + member_column = next( + ( + name + for name in frame.columns + if [value if isinstance(value, str) else None for value in frame[name]] + == members + ), + None, + ) + assert member_column is not None, ( + "no column carries the archive member of each row; expected " + f"{sorted(set(members))}" + ) + + for name in frame.columns: + if name == member_column: + continue + try: + values = [int(value) for value in frame[name].tolist()] + except (TypeError, ValueError): + continue + if len(values) != len(members): + continue + runs = {} + for member, value in zip(members, values, strict=True): + runs.setdefault(member, []).append(value) + bases = {min(run) for run in runs.values()} + if bases <= {0} or bases <= {1}: + if all( + sorted(run) == list(range(min(run), min(run) + len(run))) + for run in runs.values() + ): + return member_column, name + raise AssertionError( + "no column carries a within-member row ordinal restarting at each member" + ) + + +# -------------------------------------------------------------------------- +# Accepted population: projection, classification and roundtrip. +# -------------------------------------------------------------------------- + + +def test_projected_households_are_lexical_and_complete(acs): + result = acs.produce() + households = result.households + assert len(households) == len(BASE_HOUSEHOLDS) + assert households.columns.is_unique + assert set(HOUSEHOLD_ROSTER) <= set(households.columns) + for name in HOUSEHOLD_ROSTER: + lexical(households[name]) + # Every projected token survives byte for byte: blank tenure stays an empty + # string rather than NaN, None or the dictionary's display character, and + # leading zeros on PUMA and ST are preserved. + assert household_map(households) == EXPECTED_HOUSEHOLDS + + +def test_projected_persons_keep_raw_sporder_tokens(acs): + persons = acs.produce().persons + assert len(persons) == len(BASE_PERSONS) + assert set(PERSON_ROSTER) <= set(persons.columns) + # "01" and "1" are both accepted and both retained exactly as written. + assert person_map(persons) == EXPECTED_PERSONS + + +def test_native_codes_are_uint8_in_the_pinned_order(acs): + result = acs.produce() + codes = result.codes + assert list(codes.columns) == list(CODE_COLUMNS) + for name in CODE_COLUMNS: + assert codes[name].dtype == np.dtype("uint8") + assert codes.index.equals(result.households.index) + assert codes_map(result) == EXPECTED_CODES + + +def test_codes_equal_the_pinned_pure_classifier_on_the_strict_arrays(acs): + """The source result must not invent its own semantics for the same rows.""" + result = acs.produce() + frame, tenure_valid = strict_arrays(result.households) + expected, _header = classify_acs_housing_universe(frame, tenure_valid=tenure_valid) + pd.testing.assert_frame_equal(result.codes, expected, check_like=False) + + +def test_np0_housing_unit_stays_occupancy_unknown_in_the_native_codes(acs): + """Source vacancy evidence must not be back-written into the pure classes.""" + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000004"], strict=True) + ) + assert codes["occupied_hu"] == 0 + assert codes["hu_tenure_class"] == 0 + assert codes["unresolved_reasons"] == 1 + assert codes["physical_unit"] == 1 + assert codes["TEN_valid"] == 0 + + +def test_occupied_unit_with_unavailable_tenure_is_retained_not_recoded(acs): + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000003"], strict=True) + ) + assert codes["occupied_hu"] == 1 + assert codes["hu_tenure_class"] == 3 + assert codes["tenure_subtype"] == 0 + assert codes["unresolved_reasons"] == 2 + + +def test_member_and_row_ordinal_lineage_is_accessible(acs): + result = acs.produce() + assert_lineage( + result.households, + household_keys(result.households), + EXPECTED_HOUSEHOLD_MEMBER, + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + EXPECTED_PERSON_MEMBER, + ) + + +def test_rows_are_emitted_in_the_agreed_canonical_order(acs): + """The plan leaves row order open; this is the single place it is decided.""" + assert household_keys(acs.produce().households) == expected_row_order( + BASE_SERIALNOS + ) + + +def test_row_order_does_not_depend_on_the_member_split(tmp_path, monkeypatch): + """Splitting the same invented rows differently must not reorder them.""" + one = single_member(tmp_path / "one", monkeypatch).produce() + three = build_fixture( + tmp_path / "three", + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ).produce() + assert household_keys(three.households) == household_keys(one.households) + assert household_map(three.households) == household_map(one.households) + + +def test_source_artifact_holds_exactly_the_projection_and_receipt(acs): + output_dir = acs.output_dir() + result = acs.produce(output_dir=output_dir) + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + # The house convention appends a trailing newline to published JSON. + assert (output_dir / NAMES["projection_filename"]).read_bytes().rstrip( + b"\n" + ) == result.projection_json.rstrip(b"\n") + assert (output_dir / NAMES["receipt_filename"]).read_bytes().rstrip( + b"\n" + ) == result.receipt_json.rstrip(b"\n") + json.loads(result.projection_json) + json.loads(result.receipt_json) + + +def test_readback_returns_the_same_bytes_and_the_same_population(acs): + produced, loaded, _output_dir = acs.produce_and_load() + assert loaded.projection_json == produced.projection_json + assert loaded.receipt_json == produced.receipt_json + pd.testing.assert_frame_equal(loaded.households, produced.households) + pd.testing.assert_frame_equal(loaded.persons, produced.persons) + pd.testing.assert_frame_equal(loaded.codes, produced.codes) + + +def test_two_produces_from_identical_inputs_are_byte_identical(acs): + """Readback compares exact bytes, so both artifacts must be deterministic.""" + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert first.receipt_json == second.receipt_json + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_returned_frames_are_owned_copies(acs, attribute): + result = acs.produce() + first = getattr(result, attribute) + assert getattr(result, attribute) is not first + before = first.copy(deep=True) + projection = result.projection_json + first.iloc[0, 0] = first.iloc[-1, 0] + first.rename(columns={first.columns[0]: "clobbered"}, inplace=True) + first.index = pd.RangeIndex(1000, 1000 + len(first)) + after = getattr(result, attribute) + pd.testing.assert_frame_equal(after, before) + assert result.projection_json == projection + + +def test_receipt_records_auditable_member_and_definition_evidence(acs): + result = acs.produce() + receipt = json.loads(result.receipt_json) + strings = flat_strings(receipt) + numbers = flat_numbers(receipt) + + assert NAMES["vacancy_definition"] in strings + assert NAMES["gq_definition"] in strings + assert {HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE} <= strings + assert {"psam_husa.csv", "psam_husb.csv", "psam_pusa.csv", "psam_pusb.csv"} <= ( + strings + ) + + for archive, names in ( + (acs.household_zip, ("psam_husa.csv", "psam_husb.csv")), + (acs.person_zip, ("psam_pusa.csv", "psam_pusb.csv")), + ): + with zipfile.ZipFile(archive) as opened: + for name in names: + info = opened.getinfo(name) + payload = opened.read(name) + assert hashlib.sha256(payload).hexdigest() in strings, ( + f"receipt omits the decompressed digest of {name}" + ) + assert info.CRC in numbers, f"receipt omits the CRC of {name}" + assert info.file_size in numbers + + # Counts an auditor needs: the whole population and the vacancy evidence. + assert len(BASE_HOUSEHOLDS) in numbers + assert len(BASE_PERSONS) in numbers + + +def test_receipt_binds_the_pinned_archive_digests(acs): + pins = acs.pin() + strings = flat_strings(json.loads(acs.produce().receipt_json)) + for _role, name, sha, _size in pins: + assert sha in strings, f"receipt omits the pinned digest of {name}" + + +# -------------------------------------------------------------------------- +# Population variants: one deliberate change against the accepted invented rows. +# -------------------------------------------------------------------------- + + +def with_household(index, **fields): + rows = list(BASE_HOUSEHOLDS) + values = dict(zip(HOUSEHOLD_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in HOUSEHOLD_HEADER) + return tuple(rows) + + +def with_person(index, **fields): + rows = list(BASE_PERSONS) + values = dict(zip(PERSON_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in PERSON_HEADER) + return tuple(rows) + + +def rename_serialno(old, new): + """Rename a key on both sides so the only change is the identifier token.""" + key = HOUSEHOLD_HEADER.index("SERIALNO") + households = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_HOUSEHOLDS + ) + key = PERSON_HEADER.index("SERIALNO") + persons = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_PERSONS + ) + return households, persons + + +def single_member(tmp_path, monkeypatch, *, households=None, persons=None, **kwargs): + """One member per role, so a variant's only difference is the row content.""" + return build_fixture( + tmp_path, + monkeypatch, + households=( + ("psam_husa.csv", BASE_HOUSEHOLDS if households is None else households), + ), + persons=(("psam_pusa.csv", BASE_PERSONS if persons is None else persons),), + **kwargs, + ) + + +# -------------------------------------------------------------------------- +# Key selection. +# -------------------------------------------------------------------------- + + +def test_selection_none_retains_the_whole_invented_source(acs): + result = acs.produce(serialnos=None) + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert len(result.persons) == len(BASE_PERSONS) + + +def test_selection_keeps_whole_households_and_no_other_person(acs): + result = acs.produce(serialnos=("2024HU0000001", "2024GQ0000005")) + assert set(household_keys(result.households)) == { + "2024HU0000001", + "2024GQ0000005", + } + assert person_map(result.persons) == { + ("2024HU0000001", "01"): "121", + ("2024HU0000001", "02"): "97", + ("2024GQ0000005", "01"): "64", + } + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024HU0000001", "2024GQ0000005") + } + + +def test_a_vacancy_only_selection_is_a_valid_source_projection(acs): + """Source-only preparation may validly retain no people at all.""" + result = acs.produce(serialnos=("2024HU0000004",)) + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert result.codes["occupied_hu"].tolist() == [0] + assert result.codes["unresolved_reasons"].tolist() == [1] + + +def test_selected_rows_ignore_the_caller_tuple_order(acs): + """The request is a set of keys; canonical order is the module's own.""" + forward = acs.produce( + output_dir=acs.output_dir("forward"), + serialnos=("2024HU0000002", "2024HU0000008"), + ) + reversed_request = acs.produce( + output_dir=acs.output_dir("reversed"), + serialnos=("2024HU0000008", "2024HU0000002"), + ) + assert forward.projection_json == reversed_request.projection_json + assert household_keys(forward.households) == expected_row_order( + ("2024HU0000002", "2024HU0000008") + ) + + +def test_selection_changes_the_projection_bytes(acs): + whole = acs.produce() + selected = acs.produce(serialnos=("2024HU0000001",)) + assert selected.projection_json != whole.projection_json + + +def test_readback_under_a_different_selection_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir, serialnos=("2024HU0000001",)) + refuses(acs, acs.load, output_dir, serialnos=("2024HU0000002",)) + refuses(acs, acs.load, output_dir, serialnos=None) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(("2024HU0000009",), id="unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000009"), id="one_unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000001"), id="duplicate_key"), + pytest.param((), id="empty_selection"), + pytest.param(("",), id="blank_key"), + pytest.param((" 2024HU0000001",), id="whitespace_padded_key"), + pytest.param(("2024hu0000001",), id="case_folded_key"), + ], +) +def test_invalid_selection_refuses(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +# -------------------------------------------------------------------------- +# Complete-join integrity, checked before any exclusion or selection. +# -------------------------------------------------------------------------- + +JOIN_VARIANTS = [ + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + id="person_attached_to_a_vacant_household", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + id="orphan_person_without_a_household", + ), + pytest.param( + BASE_HOUSEHOLDS + (BASE_HOUSEHOLDS[0],), + BASE_PERSONS, + id="duplicate_household_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "1", "44"),), + id="duplicate_person_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "01", "44"),), + id="duplicate_person_key_via_leading_zero_alias", + ), + pytest.param( + with_household(0, NP="3"), + BASE_PERSONS, + id="household_np_above_linked_person_count", + ), + pytest.param( + with_household(0, NP="1"), + BASE_PERSONS, + id="household_np_below_linked_person_count", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_without_its_person", + ), + pytest.param( + with_household(4, NP="2"), + BASE_PERSONS + (person_row("2024GQ0000005", "02", "51"),), + id="group_quarters_placeholder_with_two_people", + ), + pytest.param( + with_household(4, NP="0"), + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_with_no_people", + ), +] + + +@pytest.mark.parametrize("households,persons", JOIN_VARIANTS) +def test_join_integrity_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(None, id="whole_source"), + pytest.param(("2024HU0000001",), id="unrelated_single_key"), + ], +) +def test_vacant_household_person_refuses_before_exclusion_or_selection( + tmp_path, monkeypatch, serialnos +): + """An NP0 household with a person must refuse even when nothing selects it. + + An implementation that dropped NP0 rows, or applied the key selection, + before validating would silently accept this invented contradiction. + """ + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=serialnos) + + +def test_orphan_person_refuses_even_when_selection_excludes_it(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=("2024HU0000002",)) + + +# -------------------------------------------------------------------------- +# Closed source-value domains. +# -------------------------------------------------------------------------- + +DOMAIN_VARIANTS = [ + # SERIALNO: exactly 13 ASCII characters, 2024HU/2024GQ plus seven digits, + # numeric suffix 1..9999999, prefix agreeing with TYPEHUGQ, no whitespace, + # no empty identifier and no normalisation of the accepted bytes. + pytest.param( + *rename_serialno("2024HU0000001", " 2024HU000001"), id="serialno_leading_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001 "), id="serialno_trailing_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU 000001"), id="serialno_interior_space" + ), + pytest.param(*rename_serialno("2024HU0000001", ""), id="serialno_empty"), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001"), id="serialno_too_short" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU00000010"), id="serialno_too_long" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2023HU0000001"), id="serialno_wrong_vintage" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024XX0000001"), + id="serialno_unknown_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024hu0000001"), + id="serialno_lowercase_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU0000000"), id="serialno_zero_suffix" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000000A"), + id="serialno_nondigit_suffix", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024GQ0000001"), + id="serialno_gq_prefix_on_housing_unit", + ), + pytest.param( + *rename_serialno("2024GQ0000005", "2024HU0000005"), + id="serialno_hu_prefix_on_group_quarters", + ), + # SPORDER: one or two ASCII digits, value 1..20, no sign/exponent/point. + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="0"), id="sporder_zero"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="21"), id="sporder_above_domain" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="001"), id="sporder_three_digits" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER=" 1"), id="sporder_leading_space" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1 "), id="sporder_trailing_space" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="+1"), id="sporder_signed"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="-1"), id="sporder_negative"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1.0"), id="sporder_decimal_point" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="1e0"), id="sporder_exponent"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER=""), id="sporder_empty"), + # TYPEHUGQ: 1, 2 or 3 only. + pytest.param(with_household(0, TYPEHUGQ="0"), BASE_PERSONS, id="typehugq_zero"), + pytest.param( + with_household(0, TYPEHUGQ="4"), BASE_PERSONS, id="typehugq_above_domain" + ), + pytest.param(with_household(0, TYPEHUGQ=""), BASE_PERSONS, id="typehugq_empty"), + pytest.param( + with_household(0, TYPEHUGQ="H"), BASE_PERSONS, id="typehugq_nonnumeric" + ), + # NP: published integer domain 0..20. + pytest.param(with_household(0, NP="21"), BASE_PERSONS, id="np_above_domain"), + pytest.param(with_household(0, NP="-1"), BASE_PERSONS, id="np_negative"), + pytest.param(with_household(0, NP=""), BASE_PERSONS, id="np_empty"), + pytest.param(with_household(0, NP=" 2"), BASE_PERSONS, id="np_leading_space"), + pytest.param(with_household(0, NP="2.0"), BASE_PERSONS, id="np_decimal_point"), + # WGTP: 1..9999 for housing units including vacant ones; zero only for the + # group-quarters placeholders. + pytest.param( + with_household(0, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_occupied_unit" + ), + pytest.param( + with_household(3, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_vacant_unit" + ), + pytest.param(with_household(0, WGTP="10000"), BASE_PERSONS, id="wgtp_above_domain"), + pytest.param(with_household(0, WGTP="-5"), BASE_PERSONS, id="wgtp_negative"), + pytest.param(with_household(0, WGTP=""), BASE_PERSONS, id="wgtp_empty"), + # PWGTP: 1..9999 for every person, group quarters included. + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="0"), id="pwgtp_zero"), + pytest.param( + BASE_HOUSEHOLDS, + with_person(6, PWGTP="0"), + id="pwgtp_zero_on_group_quarters_person", + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, PWGTP="10000"), id="pwgtp_above_domain" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="-1"), id="pwgtp_negative"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP=""), id="pwgtp_empty"), + # TEN: blank means missing; observed values are 1..4; group quarters and + # vacancy require the blank token. + pytest.param(with_household(0, TEN="5"), BASE_PERSONS, id="ten_above_domain"), + pytest.param(with_household(0, TEN="0"), BASE_PERSONS, id="ten_zero"), + pytest.param(with_household(0, TEN="b"), BASE_PERSONS, id="ten_display_character"), + pytest.param(with_household(0, TEN=" "), BASE_PERSONS, id="ten_single_space"), + pytest.param( + with_household(3, TEN="1"), BASE_PERSONS, id="ten_present_on_vacant_unit" + ), + pytest.param( + with_household(4, TEN="1"), BASE_PERSONS, id="ten_present_on_group_quarters" + ), + # PUMA: exactly five ASCII digits with integer value 100..81003. + pytest.param(with_household(0, PUMA="0010"), BASE_PERSONS, id="puma_four_digits"), + pytest.param(with_household(0, PUMA="000100"), BASE_PERSONS, id="puma_six_digits"), + pytest.param(with_household(0, PUMA="00099"), BASE_PERSONS, id="puma_below_domain"), + pytest.param(with_household(0, PUMA="81004"), BASE_PERSONS, id="puma_above_domain"), + pytest.param(with_household(0, PUMA="0010A"), BASE_PERSONS, id="puma_nondigit"), + pytest.param(with_household(0, PUMA=""), BASE_PERSONS, id="puma_empty"), + # ST: exactly two ASCII digits from the 50-state and DC subset. + pytest.param(with_household(0, ST="72"), BASE_PERSONS, id="state_puerto_rico"), + pytest.param(with_household(0, ST="99"), BASE_PERSONS, id="state_unassigned_code"), + pytest.param(with_household(0, ST="6"), BASE_PERSONS, id="state_single_digit"), + pytest.param(with_household(0, ST="006"), BASE_PERSONS, id="state_three_digits"), + pytest.param( + with_household(0, ST="CA"), BASE_PERSONS, id="state_postal_abbreviation" + ), + pytest.param(with_household(0, ST=""), BASE_PERSONS, id="state_empty"), +] + + +@pytest.mark.parametrize("households,persons", DOMAIN_VARIANTS) +def test_source_value_domain_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +def test_sporder_leading_zero_is_accepted_and_the_raw_token_is_retained( + tmp_path, monkeypatch +): + """Leading-zero SPORDER normalises for the join without losing its bytes.""" + fixture = single_member(tmp_path, monkeypatch, persons=with_person(2, SPORDER="01")) + assert ("2024HU0000002", "01") in person_map(fixture.produce().persons) + + +def test_maximum_and_minimum_accepted_weights_are_retained(acs): + weights = household_map(acs.produce().households) + assert weights["2024HU0000003"]["WGTP"] == "1" + assert weights["2024HU0000008"]["WGTP"] == "9999" + + +# -------------------------------------------------------------------------- +# Strict lexical parsing and the ordered-header contract. +# -------------------------------------------------------------------------- + +BASE_HOUSEHOLD_CSV = csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS) +BASE_PERSON_CSV = csv_bytes(PERSON_HEADER, BASE_PERSONS) + + +def household_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_husa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member(name, payload),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def person_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_pusa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member(name, payload),), + ) + + +def splice_line(payload, index, line): + """Insert an invented physical line at a record boundary.""" + lines = payload.split(b"\n") + lines.insert(index, line) + return b"\n".join(lines) + + +def drop_column(header, rows, name): + position = header.index(name) + trimmed = header[:position] + header[position + 1 :] + return trimmed, tuple(row[:position] + row[position + 1 :] for row in rows) + + +def rename_column(header, old, new): + return tuple(new if name == old else name for name in header) + + +WIDE_SUFFIX = tuple(f"AUX{index:02d}" for index in range(24)) +WIDE_HEADER = HOUSEHOLD_HEADER + WIDE_SUFFIX +WIDE_ROWS = tuple(row + ("0",) * len(WIDE_SUFFIX) for row in BASE_HOUSEHOLDS) + + +def oversized_record_csv(): + """One physical record above 1 MiB whose every field stays under 64 KiB.""" + padded = ("4" * 50_000,) * len(WIDE_SUFFIX) + rows = (WIDE_ROWS[0][: len(HOUSEHOLD_HEADER)] + padded,) + WIDE_ROWS[1:] + return csv_bytes(WIDE_HEADER, rows) + + +def oversized_field_csv(): + """One field token above 64 KiB in a column that is never projected.""" + return csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="7" * 65_537)) + + +HOUSEHOLD_PARSE_VARIANTS = [ + pytest.param(b"\xef\xbb\xbf" + BASE_HOUSEHOLD_CSV, id="utf8_byte_order_mark"), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"2024HU0000002", b"2024HU\xff000002"), + id="invalid_utf8_byte", + ), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"82000", b"820\x0000"), id="embedded_nul_in_field" + ), + pytest.param(splice_line(BASE_HOUSEHOLD_CSV, 3, b""), id="interior_blank_record"), + pytest.param(BASE_HOUSEHOLD_CSV + b"\n", id="trailing_blank_record"), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + id="short_record", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0] + ("extra",)).encode() + + b"\n", + id="long_record", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", "ST"), BASE_HOUSEHOLDS), + id="duplicate_header_name", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", ""), BASE_HOUSEHOLDS), + id="blank_header_name", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", " SERIALNO"), BASE_HOUSEHOLDS + ), + id="whitespace_padded_header_name", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "NP")), + id="missing_np_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "SERIALNO")), + id="missing_serialno_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "TEN")), + id="missing_ten_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "WGTP")), + id="missing_wgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "PUMA")), + id="missing_puma_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + id="neither_st_nor_state_column", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\r82000")), + id="embedded_carriage_return_in_field", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\n82000")), + id="embedded_line_feed_in_field", + ), + pytest.param(oversized_field_csv(), id="field_token_above_64_kib"), + pytest.param(oversized_record_csv(), id="physical_record_above_1_mib"), + pytest.param(BASE_HOUSEHOLD_CSV.replace(b",", b";"), id="semicolon_delimiter"), + pytest.param(b"", id="empty_member_without_a_header"), +] + + +@pytest.mark.parametrize("payload", HOUSEHOLD_PARSE_VARIANTS) +def test_household_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +PERSON_PARSE_VARIANTS = [ + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SPORDER")), + id="missing_sporder_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "PWGTP")), + id="missing_pwgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SERIALNO")), + id="missing_person_serialno_column", + ), + pytest.param(splice_line(BASE_PERSON_CSV, 2, b""), id="person_blank_record"), + pytest.param( + csv_bytes(PERSON_HEADER, BASE_PERSONS[1:]) + + ",".join(BASE_PERSONS[0][:-2]).encode() + + b"\n", + id="person_short_record", + ), +] + + +@pytest.mark.parametrize("payload", PERSON_PARSE_VARIANTS) +def test_person_parser_refusals(tmp_path, monkeypatch, payload): + fixture = person_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_cross_member_reordered_header_refuses(tmp_path, monkeypatch): + """Identical column sets in a different order are still a header mismatch.""" + reordered = list(HOUSEHOLD_HEADER) + first, second = reordered.index("REGION"), reordered.index("ADJHSG") + reordered[first], reordered[second] = reordered[second], reordered[first] + rows = tuple( + tuple(dict(zip(HOUSEHOLD_HEADER, row, strict=True))[name] for name in reordered) + for row in BASE_HOUSEHOLDS[5:] + ) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("psam_husb.csv", csv_bytes(tuple(reordered), rows)), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_cross_member_extra_column_refuses(tmp_path, monkeypatch): + extended = HOUSEHOLD_HEADER + ("FINCP",) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member( + "psam_husb.csv", + csv_bytes(extended, tuple(row + ("0",) for row in BASE_HOUSEHOLDS[5:])), + ), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_header_only_member_is_allowed_and_contributes_no_rows(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, ())), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())), + Member("psam_pusb.csv", BASE_PERSON_CSV), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert person_map(result.persons) == EXPECTED_PERSONS + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + dict.fromkeys(EXPECTED_PERSONS, "psam_pusb.csv"), + ) + # The empty members are still inventoried even though they carry no rows. + strings = flat_strings(json.loads(result.receipt_json)) + assert {"psam_husb.csv", "psam_pusa.csv"} <= strings + + +def test_an_entirely_header_only_source_is_not_a_population(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + refuses(fixture, fixture.produce) + + +def test_state_only_household_header_is_supported(tmp_path, monkeypatch): + header = rename_column(HOUSEHOLD_HEADER, "ST", "STATE") + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(header, BASE_HOUSEHOLDS) + ) + result = fixture.produce() + assert household_map(result.households)["2024HU0000001"]["ST"] == "06" + # The header alias itself is preserved in the source evidence. + assert "STATE" in flat_strings(json.loads(result.receipt_json)) + + +def test_agreeing_st_and_state_columns_are_accepted(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple(row + (row[state],) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + result = fixture.produce() + assert household_map(result.households)["2024HU0000003"]["ST"] == "36" + + +def test_disagreeing_st_and_state_columns_refuse(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple( + row + ("36" if index == 0 else row[state],) + for index, row in enumerate(BASE_HOUSEHOLDS) + ) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_both_physical_line_terminators_project_identically( + tmp_path, monkeypatch, newline +): + """A genuine archive's terminator choice must not change the projection.""" + fixture = build_fixture(tmp_path, monkeypatch, newline=newline) + result = fixture.produce() + assert household_map(result.households) == EXPECTED_HOUSEHOLDS + assert person_map(result.persons) == EXPECTED_PERSONS + + +# -------------------------------------------------------------------------- +# Archive preflight, run on the central directory before any by-name lookup. +# -------------------------------------------------------------------------- + +SYMLINK_ATTR = (stat.S_IFLNK | 0o777) << 16 +DIRECTORY_ATTR = (stat.S_IFDIR | 0o755) << 16 | 0x10 + + +def household_archive_fixture(tmp_path, monkeypatch, members): + return build_fixture( + tmp_path, + monkeypatch, + household_members=members, + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def aux_members(count, *, prefix="ACS2024_note"): + return tuple( + Member(f"{prefix}{index:03d}.txt", b"invented auxiliary text\n") + for index in range(count) + ) + + +ARCHIVE_VARIANTS = [ + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + id="duplicate_member_name", + ), + pytest.param( + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("PSAM_HUSA.CSV", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[5:])), + ), + id="case_folded_member_alias", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_hus/", b"", external_attr=DIRECTORY_ATTR), + ), + id="directory_entry", + ), + pytest.param( + (Member("pums/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="nested_path_member", + ), + pytest.param( + (Member("/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="absolute_path_member", + ), + pytest.param( + (Member("../psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="parent_traversal_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.txt", BASE_HOUSEHOLD_CSV), + ), + id="prefix_matching_non_csv_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member( + "psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_BZIP2 + ), + ), + id="bzip2_compressed_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_link.csv", b"psam_husa.csv", external_attr=SYMLINK_ATTR), + ), + id="symlink_member", + ), + pytest.param( + (Member("psam_pusa.csv", BASE_PERSON_CSV),), + id="wrong_role_members_only", + ), + pytest.param( + (Member("ACS2024_readme.txt", b"invented auxiliary text\n"),), + id="no_applicable_csv_member", + ), + pytest.param((), id="empty_archive"), + pytest.param( + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(64), + id="member_count_above_cap", + ), +] + + +@pytest.mark.parametrize("members", ARCHIVE_VARIANTS) +def test_archive_preflight_refusals(tmp_path, monkeypatch, members): + fixture = household_archive_fixture(tmp_path, monkeypatch, members) + refuses(fixture, fixture.produce) + + +def test_member_count_at_the_cap_is_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(63), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_ordinary_auxiliary_members_are_inventory_only(tmp_path, monkeypatch): + """A non-matching ordinary file is recorded but never read as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_PUMS_README.txt", b"invented auxiliary text\n"), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert "ACS2024_PUMS_README.txt" in flat_strings(json.loads(result.receipt_json)) + + +def test_a_non_matching_csv_member_is_auxiliary_not_data(tmp_path, monkeypatch): + """The role prefix, not the extension, decides what counts as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_shells.csv", csv_bytes(("A", "B"), (("1", "2"),))), + ), + ) + result = fixture.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert "ACS2024_shells.csv" in flat_strings(json.loads(result.receipt_json)) + + +def test_stored_members_are_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_several_members_per_role_join_in_name_order(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[:4])), + Member("psam_pusb.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[4:])), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + members = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: name + for name, rows in ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:3]), + ("psam_husb.csv", BASE_HOUSEHOLDS[3:6]), + ("psam_husc.csv", BASE_HOUSEHOLDS[6:]), + ) + for row in rows + } + assert_lineage(result.households, household_keys(result.households), members) + # 2024HU0000003's three people straddle the two person members. + assert lexical(result.persons["SERIALNO"]).count("2024HU0000003") == 3 + assert person_map(result.persons) == EXPECTED_PERSONS + person_members = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): name + for name, rows in ( + ("psam_pusa.csv", BASE_PERSONS[:4]), + ("psam_pusb.csv", BASE_PERSONS[4:]), + ) + for row in rows + } + assert_lineage(result.persons, list(person_map(result.persons)), person_members) + + +@pytest.mark.parametrize("archive", [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE]) +def test_missing_role_archive_refuses(acs, archive): + (acs.source_dir / archive).unlink() + refuses(acs, acs.produce) + + +def test_corrupted_member_payload_fails_the_recorded_crc(tmp_path, monkeypatch): + """A stored member whose bytes were swapped keeps its declared size and CRC.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + payload = fixture.household_zip.read_bytes() + assert b"82000" in payload, "the stored member is not plain in the archive" + fixture.household_zip.write_bytes(payload.replace(b"82000", b"82001", 1)) + fixture.pin() + refuses(fixture, fixture.produce) + + +# -------------------------------------------------------------------------- +# Private capture: paths, pins, originals and byte bounds. +# -------------------------------------------------------------------------- + + +@contextmanager +def bounded_time(seconds=30): + """Fail rather than hang if a non-regular original is opened blocking.""" + + def expire(signum, frame): + raise TimeoutError("the call did not return within the bounded test window") + + previous = signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def test_missing_snapshot_root_refuses(acs): + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=acs.tmp_path / "absent-root", + output_dir=acs.output_dir(), + ) + + +def test_snapshot_root_that_is_a_file_refuses(acs): + root = acs.tmp_path / "root-file" + root.write_bytes(b"invented\n") + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=root, + output_dir=acs.output_dir(), + ) + + +def test_symlinked_snapshot_root_refuses(acs): + link = acs.tmp_path / "root-link" + link.symlink_to(acs.snapshot_root, target_is_directory=True) + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=link, + output_dir=acs.output_dir(), + ) + + +def test_existing_output_directory_refuses(acs): + output_dir = acs.output_dir() + output_dir.mkdir() + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_existing_output_file_refuses(acs): + output_dir = acs.output_dir() + output_dir.write_bytes(b"invented\n") + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_output_directory_without_a_parent_refuses(acs): + refuses(acs, acs.produce, output_dir=acs.tmp_path / "absent" / "artifact") + + +def test_missing_source_directory_refuses(acs): + refuses(acs, acs.produce, source_dir=acs.tmp_path / "absent-source") + + +def test_source_directory_that_is_a_file_refuses(acs): + path = acs.tmp_path / "source-file" + path.write_bytes(b"invented\n") + refuses(acs, acs.produce, source_dir=path) + + +def test_symlinked_source_directory_refuses(acs): + link = acs.tmp_path / "source-link" + link.symlink_to(acs.source_dir, target_is_directory=True) + refuses(acs, acs.produce, source_dir=link) + + +def test_symlinked_original_archive_refuses(acs): + real = acs.tmp_path / "real-csv_hus.zip" + real.write_bytes(acs.household_zip.read_bytes()) + acs.household_zip.unlink() + acs.household_zip.symlink_to(real) + refuses(acs, acs.produce) + + +def test_fifo_original_archive_refuses(acs): + acs.household_zip.unlink() + os.mkfifo(acs.household_zip) + with bounded_time(): + refuses(acs, acs.produce) + + +def test_directory_in_place_of_an_original_archive_refuses(acs): + acs.household_zip.unlink() + acs.household_zip.mkdir() + refuses(acs, acs.produce) + + +def test_pinned_digest_mismatch_at_the_same_size_refuses(acs): + """Authentication is over the whole archive bytes, not over the projection.""" + pins = acs.pin() + flip_archive_byte(acs.household_zip) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_pinned_size_mismatch_refuses(acs): + pins = acs.pin() + with acs.household_zip.open("ab") as handle: + handle.write(b"\n") + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_original_archives_are_never_modified(acs): + before = { + path: (path.read_bytes(), path.stat().st_mtime_ns) + for path in (acs.household_zip, acs.person_zip) + } + acs.produce() + for path, (payload, mtime) in before.items(): + assert path.read_bytes() == payload + assert path.stat().st_mtime_ns == mtime + + +def test_capture_writes_only_under_the_supplied_snapshot_root(acs): + assert not any(acs.snapshot_root.iterdir()) + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + children = list(acs.snapshot_root.iterdir()) + assert children, "the private capture did not land under the snapshot root" + for child in children: + assert child.is_dir() and not child.is_symlink() + assert stat.S_IMODE(child.stat().st_mode) == 0o700 + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + + +def test_capture_never_falls_back_to_system_temporary_storage(acs, monkeypatch): + absent = acs.tmp_path / "no-such-tmpdir" + monkeypatch.setenv("TMPDIR", str(absent)) + monkeypatch.setattr(tempfile, "tempdir", str(absent)) + result = acs.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert not absent.exists() + + +def test_two_produces_share_one_snapshot_root_without_collision(acs): + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert len(list(acs.snapshot_root.iterdir())) >= 2 + + +def test_a_refused_produce_leaves_no_receipt_behind(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, monkeypatch, households=with_household(0, TEN="5") + ) + output_dir = fixture.output_dir() + refuses(fixture, fixture.produce, output_dir=output_dir) + assert not (output_dir / NAMES["receipt_filename"]).exists() + + +def byte_cap_names(): + return tuple(sorted(name for name in dir(source) if name.endswith("_MAX_BYTES"))) + + +def test_module_declares_its_own_byte_caps(): + assert byte_cap_names(), "the module declares no explicit byte caps" + + +@pytest.mark.parametrize("cap", [0, 1, 64]) +def test_tiny_byte_caps_refuse_before_any_payload_is_returned(acs, cap): + """Exercise the streaming bounds with mocked caps, never with real GiBs.""" + for name in byte_cap_names(): + acs.monkeypatch.setattr(source, name, cap) + refuses(acs, acs.produce) + + +def test_declared_cap_values_match_the_approved_plan(): + """Reconciliation: the plan fixes both ACS source caps by name and value.""" + assert getattr(source, NAMES["source_max_bytes"]) == 8 * 1024**3 + assert getattr(source, NAMES["receipt_max_bytes"]) == 1024**2 + + +# -------------------------------------------------------------------------- +# Artifact custody: readback reconstructs, it never trusts what it decoded. +# -------------------------------------------------------------------------- + + +def make_writable(path): + """The producer publishes read-only files; tampering needs write access.""" + path = Path(path) + path.chmod(stat.S_IMODE(path.stat().st_mode) | 0o600) + return path + + +def forge(output_dir, old, new): + """Rewrite the artifact so its own internal digests stay self-consistent.""" + projection = (output_dir / NAMES["projection_filename"]).read_bytes() + receipt = (output_dir / NAMES["receipt_filename"]).read_bytes() + assert old in projection, "the invented forgery target is not in the projection" + forged = projection.replace(old, new) + assert len(forged) == len(projection), "a forgery must not change the length" + old_digest = hashlib.sha256(projection).hexdigest().encode() + new_digest = hashlib.sha256(forged).hexdigest().encode() + make_writable(output_dir / NAMES["projection_filename"]).write_bytes(forged) + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes( + receipt.replace(old_digest, new_digest) + ) + return old_digest in receipt + + +def test_rehashed_projection_forgery_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + refuses(acs, acs.load, output_dir) + + +def test_receipt_binds_the_projection_digest(acs): + """Otherwise a forged projection would be self-consistent on its face.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + assert forge(output_dir, b"2024HU0000002", b"2024HU0000012") + + +def test_forging_a_weight_to_another_valid_value_refuses(acs): + """Custody is byte reconstruction, so an in-domain forgery refuses too.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"118", b"119") + refuses(acs, acs.load, output_dir) + + +def test_rewritten_receipt_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + receipt = json.loads((output_dir / NAMES["receipt_filename"]).read_bytes()) + forged = json.dumps(receipt, sort_keys=True).encode() + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes(forged) + refuses(acs, acs.load, output_dir) + + +@pytest.mark.parametrize( + "filename", [NAMES["projection_filename"], NAMES["receipt_filename"]] +) +def test_truncated_or_missing_artifact_files_refuse(acs, filename): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + make_writable(output_dir / filename).write_bytes( + (output_dir / filename).read_bytes()[:20] + ) + refuses(acs, acs.load, output_dir) + + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / filename).unlink() + refuses(acs, acs.load, output_dir) + + +def test_missing_artifact_directory_refuses(acs): + refuses(acs, acs.load, acs.tmp_path / "absent-artifact") + + +def test_extra_file_in_the_artifact_directory_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / "notes.json").write_bytes(b"{}\n") + refuses(acs, acs.load, output_dir) + + +def test_parent_change_between_produce_and_readback_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + flip_archive_byte(acs.household_zip) + refuses(acs, acs.load, output_dir) + + +def test_reparented_readback_refuses_even_after_repinning(acs): + """Fresh authentic parents that no longer reconstruct the artifact refuse.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + rows = with_household(1, TEN="4") + write_archive( + acs.household_zip, + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, rows[:5])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, rows[5:])), + ), + ) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_against_swapped_role_archives_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + household = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(acs.person_zip.read_bytes()) + acs.person_zip.write_bytes(household) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_from_an_unrelated_artifact_refuses(tmp_path, monkeypatch): + other = build_fixture( + tmp_path / "other", + monkeypatch, + households=(("psam_husa.csv", BASE_HOUSEHOLDS[:4]),), + persons=(("psam_pusa.csv", BASE_PERSONS[:6]),), + ) + output_dir = other.output_dir() + other.produce(output_dir=output_dir) + + mine = build_fixture(tmp_path / "mine", monkeypatch) + refuses(mine, mine.load, output_dir) + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_readback_frames_are_owned_copies(acs, attribute): + _produced, loaded, _output_dir = acs.produce_and_load() + first = getattr(loaded, attribute) + assert getattr(loaded, attribute) is not first + before = first.copy(deep=True) + first.iloc[0, 0] = first.iloc[-1, 0] + pd.testing.assert_frame_equal(getattr(loaded, attribute), before) + + +# -------------------------------------------------------------------------- +# Refusal semantics: sanitised, and one code per cause. +# -------------------------------------------------------------------------- + + +def collect_family_reasons(tmp_path, monkeypatch): + """One refusal per coarse family, each from its own invented source.""" + reasons = {} + + def record(family, build, call): + fixture = build(tmp_path / family, monkeypatch) + fixture.pin() + reasons[family] = refuses(fixture, call, fixture) + + def digest_change(fixture): + pins = fixture.pin() + flip_archive_byte(fixture.household_zip) + fixture.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + fixture.produce() + + def forged_artifact(fixture): + output_dir = fixture.output_dir() + fixture.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + fixture.load(output_dir) + + def existing_output(fixture): + output_dir = fixture.output_dir() + output_dir.mkdir() + fixture.produce(output_dir=output_dir) + + plain = build_fixture + + record("capture_digest", plain, digest_change) + record("artifact_forgery", plain, forged_artifact) + record("output_precondition", plain, existing_output) + record("selection", plain, lambda f: f.produce(serialnos=("2024HU0000009",))) + record( + "zip_preflight", + lambda base, patch: household_archive_fixture( + base, + patch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + ), + lambda f: f.produce(), + ) + record( + "csv_record", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + ), + lambda f: f.produce(), + ) + record( + "header_contract", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + ), + lambda f: f.produce(), + ) + record( + "value_domain", + lambda base, patch: single_member( + base, patch, households=with_household(0, TEN="5") + ), + lambda f: f.produce(), + ) + record( + "join_integrity", + lambda base, patch: single_member( + base, + patch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ), + lambda f: f.produce(), + ) + return reasons + + +@pytest.fixture +def family_reasons(tmp_path, monkeypatch): + return collect_family_reasons(tmp_path, monkeypatch) + + +def test_refusal_type_is_a_sanitised_value_error(): + assert issubclass(source.ACSHousingSourceError, ValueError) + + +def test_every_refusal_family_is_exercised(family_reasons): + assert set(family_reasons) == set(DISTINCT_FAMILIES) + + +def test_distinct_causes_carry_distinct_reason_codes(family_reasons): + """A single catch-all code would make every refusal above unfalsifiable.""" + collisions = {} + for family, reason in family_reasons.items(): + collisions.setdefault(reason, []).append(family) + shared = {reason: names for reason, names in collisions.items() if len(names) > 1} + assert not shared, f"distinct causes share a reason code: {shared}" + + +def test_no_refusal_reason_echoes_invented_source_content(family_reasons): + """refuses() checks each call; this states the whole-sweep contract once.""" + forbidden = set(BASE_SERIALNOS) | {"psam_husa.csv", "82000", "52000"} + for family, reason in family_reasons.items(): + for secret in forbidden: + assert secret not in reason, f"{family} leaked {secret!r}" + + +def test_reason_codes_match_the_agreed_table(family_reasons): + """Shape always; exact spellings once EXPECTED_REASONS is filled in. + + The house idiom in the committed sibling source modules is a bare + UPPER_SNAKE token, so that much is asserted unconditionally and nothing is + parked behind a skip. Only the agreement on which token names which cause + waits on the production author. + """ + for family, reason in family_reasons.items(): + assert re.fullmatch(r"[A-Z][A-Z0-9_]*", reason), ( + f"{family} reason {reason!r} is not a bare reason code" + ) + if REASON_CODES_AGREED: + assert family_reasons == EXPECTED_REASONS + else: + assert not EXPECTED_REASONS, ( + "EXPECTED_REASONS is populated; set REASON_CODES_AGREED = True" + ) + + +# -------------------------------------------------------------------------- +# Naming reconciliation: everything this file had to guess, in one place. +# -------------------------------------------------------------------------- + + +def test_module_exposes_the_agreed_public_surface(): + for name in ( + "ACSHousingSourceError", + "produce_acs_housing_source", + "load_acs_housing_source", + "prepare_acs_housing_population", + ): + assert hasattr(source, name), f"missing agreed API name {name}" + + +def test_archive_pins_are_consulted_at_call_time(acs): + """The whole suite rests on this seam, so assert it directly. + + Every fixture authenticates its invented archives by pointing the private + pin tuple at them. That only works if the module reads the attribute when + it captures, rather than copying it at import. + """ + real = acs.pin() + assert acs.produce().households is not None + + wrong = tuple((role, name, "f" * 64, size) for role, name, _sha, size in real) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], wrong) + refuses(acs, acs.produce) + + +def test_private_archive_pins_are_a_sentinel_or_the_two_fixed_archives(): + """Naming seam, reconciled with the production author, not a defect. + + The brief described `_ARCHIVE_PINS` as a populated tuple of + (role, filename, sha256, size). The module instead defaults it to None and + loads the packaged manifest lazily, while tests still patch the same + private name - which is what + ``test_archive_pins_are_consulted_at_call_time`` proves still works. Both + readings are admitted here; the shape is checked whenever it is populated, + so a wrong shape cannot hide behind the sentinel. + """ + pins = getattr(source, NAMES["archive_pins"]) + if pins is None: + return + assert type(pins) is tuple and len(pins) == 2 + assert [entry[0] for entry in pins] == [ + NAMES["household_role"], + NAMES["person_role"], + ] + assert [entry[1] for entry in pins] == [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE] + for _role, _name, sha, size in pins: + assert type(sha) is str and len(sha) == 64 and sha == sha.lower() + assert int(sha, 16) >= 0 + assert type(size) is int and size > 0 + + +def test_lineage_and_state_columns_use_the_agreed_names(acs): + households = acs.produce().households + assert NAMES["member_column"] in households.columns + assert NAMES["ordinal_column"] in households.columns + assert NAMES["state_column"] in households.columns + + +# -------------------------------------------------------------------------- +# Tail coverage: cases an obvious pass over the five dimensions omits. +# -------------------------------------------------------------------------- + + +def set_encrypted_flag(path): + """Mark every entry encrypted in place; zipfile itself will not write this.""" + raw = bytearray(Path(path).read_bytes()) + marked = 0 + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + index = 0 + while True: + index = raw.find(signature, index) + if index < 0: + break + flag = struct.unpack_from("= 2 + Path(path).write_bytes(bytes(raw)) + + +def raw_household_record(**overrides): + """A physical household line written by hand, bypassing the csv writer.""" + values = dict(zip(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[0], strict=True)) + values.update(overrides) + return ",".join(values[name] for name in HOUSEHOLD_HEADER).encode() + + +def appended_record(record): + return csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + record + b"\n" + + +def test_encrypted_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, monkeypatch, (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + ) + set_encrypted_flag(fixture.household_zip) + fixture.pin() + refuses(fixture, fixture.produce) + + +def test_lzma_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_LZMA), + ), + ) + refuses(fixture, fixture.produce) + + +def test_bytes_that_are_not_an_archive_refuse(acs): + acs.household_zip.write_bytes(b"this is invented, and it is not a zip archive\n") + acs.pin() + refuses(acs, acs.produce) + + +def test_person_archive_holding_household_members_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + ) + refuses(fixture, fixture.produce) + + +UTF16_MEMBER = BASE_HOUSEHOLD_CSV.decode().encode("utf-16") + +TAIL_PARSE_VARIANTS = [ + pytest.param(UTF16_MEMBER, id="utf16_encoded_member"), + pytest.param( + appended_record(raw_household_record(ST='"06"x')), + id="character_after_a_closing_quote", + ), + pytest.param( + appended_record(raw_household_record(HINCP='"82000')), + id="unterminated_quote", + ), + pytest.param( + splice_line(BASE_HOUSEHOLD_CSV, 3, b" "), id="whitespace_only_record" + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, TEN="NA")), + id="na_token_is_not_a_missing_tenure", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", "serialno"), BASE_HOUSEHOLDS + ), + id="lowercase_roster_column_name", + ), +] + + +@pytest.mark.parametrize("payload", TAIL_PARSE_VARIANTS) +def test_tail_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_household_rows_absent_while_person_rows_remain_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_a_final_record_without_a_trailing_newline_is_accepted(tmp_path, monkeypatch): + fixture = household_bytes_fixture( + tmp_path, + monkeypatch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, trailing_newline=False), + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_many_unprojected_columns_are_accepted(tmp_path, monkeypatch): + """Genuine PUMS members carry hundreds of columns this projection ignores.""" + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(WIDE_HEADER, WIDE_ROWS) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_roster_columns_in_any_header_position_are_accepted(tmp_path, monkeypatch): + """The plan pins header order across members of a role, not within one.""" + reversed_header = tuple(reversed(HOUSEHOLD_HEADER)) + rows = tuple(tuple(reversed(row)) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(reversed_header, rows) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_maximum_serialno_suffix_is_accepted(tmp_path, monkeypatch): + households, persons = rename_serialno("2024HU0000001", "2024HU9999999") + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + keys = household_keys(fixture.produce().households) + assert "2024HU9999999" in keys and "2024HU0000001" not in keys + + +def test_np_at_the_published_upper_bound_is_accepted(tmp_path, monkeypatch): + households = (household_row("2024HU0000020", "1", "20", "1", "300"),) + persons = tuple( + person_row("2024HU0000020", str(order), str(order + 10)) + for order in range(1, 21) + ) + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + result = fixture.produce() + assert lexical(result.households["NP"]) == ["20"] + assert len(result.persons) == 20 + assert result.codes["occupied_hu"].tolist() == [1] + + +def test_an_all_group_quarters_source_is_retained(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + households=BASE_HOUSEHOLDS[4:6], + persons=BASE_PERSONS[6:8], + ) + result = fixture.produce() + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024GQ0000005", "2024GQ0000006") + } + + +def test_a_vacancy_only_source_is_retained(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:4])), + ), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + result = fixture.produce() + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert codes_map(result) == {"2024HU0000004": EXPECTED_CODES["2024HU0000004"]} + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param((1,), id="integer_element"), + pytest.param((None,), id="none_element"), + pytest.param((b"2024HU0000001",), id="bytes_element"), + ], +) +def test_non_string_selection_elements_refuse(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +def test_an_explicit_full_selection_matches_the_none_selection(acs): + whole = acs.produce(output_dir=acs.output_dir("whole")) + explicit = acs.produce( + output_dir=acs.output_dir("explicit"), serialnos=BASE_SERIALNOS + ) + assert explicit.projection_json == whole.projection_json + + +def test_a_published_artifact_survives_a_later_refused_produce(acs): + output_dir = acs.output_dir("kept") + produced = acs.produce(output_dir=output_dir) + before = {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} + acs.household_zip.write_bytes(b"not an archive\n") + acs.pin() + refuses(acs, acs.produce, output_dir=acs.output_dir("doomed")) + assert {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} == ( + before + ) + assert before[NAMES["projection_filename"]].rstrip(b"\n") == ( + produced.projection_json.rstrip(b"\n") + ) + + +def test_readback_with_a_missing_snapshot_root_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + refuses( + acs, + source.load_acs_housing_source, + acs.source_dir, + output_dir, + snapshot_root=acs.tmp_path / "absent-root", + ) + + +def test_wrong_size_candidate_refuses_as_a_candidate_not_as_a_parent(acs): + """Readback size failures must name the candidate, not the pinned archives.""" + for filename, mutate in ( + (NAMES["projection_filename"], lambda raw: raw + b" "), + (NAMES["projection_filename"], lambda raw: raw[:20]), + (NAMES["receipt_filename"], lambda raw: raw + b" "), + (NAMES["receipt_filename"], lambda raw: raw[:20]), + ): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + path = output_dir / filename + make_writable(path).write_bytes(mutate(path.read_bytes())) + assert refuses(acs, acs.load, output_dir) == "RECONSTRUCTION_SIZE" + + +def test_parent_archive_size_failures_keep_their_own_reason_codes(acs): + """The candidate code must not be borrowed by, or borrow from, the parents.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + intact = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(intact[:-1]) + assert refuses(acs, acs.load, output_dir) == "SOURCE_SIZE" + acs.household_zip.write_bytes(intact + b"\x00") + assert refuses(acs, acs.load, output_dir) == "FILE_TOO_LARGE" + acs.household_zip.write_bytes(intact) + assert acs.load(output_dir) is not None + + +def test_failure_record_write_failure_does_not_mask_the_original_refusal( + acs, monkeypatch +): + """A failed failure record must never replace the refusal it was recording.""" + actual_write = source._write + attempted = [] + + def refuse_the_failure_record(path, data, cap): + if Path(path).name == "failure.json": + attempted.append(Path(path).name) + raise OSError(28, "No space left on device") + return actual_write(path, data, cap) + + monkeypatch.setattr(source, "_write", refuse_the_failure_record) + corrupted = bytearray(acs.household_zip.read_bytes()) + corrupted[-1] ^= 0xFF + acs.household_zip.write_bytes(bytes(corrupted)) + assert refuses(acs, acs.produce) == "SOURCE_SHA256" + assert attempted == ["failure.json"] diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py new file mode 100644 index 000000000..ea76a97ab --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py @@ -0,0 +1,588 @@ +"""Invented-only source authority and explicitly ungranted native consistency.""" + +import csv +import hashlib +import importlib.util +import io +import json +import os +import sys +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_housing_universe_source as custody +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs +from microcosm.build.us_runtime.acs_pums import AcsPumsSource, build_acs_pums_unit_frame + + +def _helper(filename): + spec = importlib.util.spec_from_file_location( + "coverage_helpers_" + filename, Path(__file__).with_name(filename + ".py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_authority = _helper("test_us_acs_housing_source") +Member = _authority.Member +build_fixture = _authority.build_fixture +flip_archive_byte = _authority.flip_archive_byte +write_archive = _authority.write_archive +_native = _helper("test_us_acs_pums") +_household, _person = _native._household, _native._person + + +def _csv(rows, *, bom=False, ending="\n"): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=list(rows[0]), quoting=csv.QUOTE_ALL, lineterminator=ending + ) + writer.writeheader() + writer.writerows(rows) + return (("\ufeff" if bom else "") + stream.getvalue()).encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + monkeypatch.setattr( + custody.shutil, "disk_usage", lambda _p: SimpleNamespace(free=64 * 1024**3) + ) + h = [_household("2024HU0000001", NP=2), _household("2024HU0000002", NP=1)] + p = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4"), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + ] + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(h)),), + person_members=( + Member("psam_pusa.csv", _csv(p[:1])), + Member("psam_pusb.csv", _csv(p[1:])), + ), + ) + frame = map_acs_native_inputs( + build_acs_pums_unit_frame( + AcsPumsSource(fixture.household_zip, fixture.person_zip) + )[0] + ).frame + return fixture, frame, h, p + + +def _load(invented, **kwargs): + fixture, frame, _h, _p = invented + return owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=frame, **kwargs + ) + + +def _rewrite( + invented, *, persons=None, households=None, members=None, bom=False, ending="\n" +): + fixture, _frame, h, p = invented + if households is not None: + write_archive( + fixture.household_zip, (Member("psam_husa.csv", _csv(households)),) + ) + if persons is not None or members is not None: + p = persons if persons is not None else p + write_archive( + fixture.person_zip, + members + if members is not None + else ( + Member("psam_pusa.csv", _csv(p[:1], bom=bom, ending=ending)), + Member("psam_pusb.csv", _csv(p[1:], bom=bom, ending=ending)), + ), + ) + fixture.pin() + + +def _refuses(code): + return pytest.raises(owner.ACSCoverageAuthenticationError, match="^" + code + "$") + + +def test_source_authentication_preserves_low_receipt_and_parent_without_population_grant( + invented, +): + fixture, frame, _h, _p = invented + before = custody.frame_content_sha256(frame) + original_tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + result = _load(invented) + table, receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(fixture.household_zip, fixture.person_zip), + person_keys=owner._native_roster(frame)[0], + chunksize=3, + ) + pd.testing.assert_frame_equal(result.table, table) + assert result.receipt["original_literal_receipt"] == receipt + assert receipt["source_authenticated"] is False + assert result.receipt["source_authenticated"] is True + assert result.native_binding.receipt["population_binding_authenticated"] is False + assert ( + result.native_binding.receipt["missing_authority"] + == "closed_prepared_acs_native_population" + ) + assert ( + result.native_binding.receipt["original_age"]["relation"] == "numeric_identity" + ) + for flag in ( + "domain_assignment_authenticated", + "period_harmonized", + "release_eligible", + "cross_survey_coverage_equivalence_established", + "population_binding_authenticated", + ): + assert result.receipt[flag] is False + assert custody.frame_content_sha256(frame) == before + for entity, original in original_tables.items(): + pd.testing.assert_frame_equal(frame.table(entity), original) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + assert [m["rows"] for m in result.receipt["members"]["person"]] == [1, 2] + assert all(len(m["sha256"]) == 64 for m in result.receipt["members"]["person"]) + + +def test_first_frame_hash_precedes_the_roster_snapshot(invented, monkeypatch): + """The returned binding must describe one state, including at hash entry.""" + _fixture, frame, _h, _p = invented + original_hash = custody.frame_content_sha256 + first = True + + def at_hash_entry(value): + nonlocal first + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return original_hash(value) + + monkeypatch.setattr(custody, "frame_content_sha256", at_hash_entry) + result = _load(invented) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + + +def test_mutation_during_roster_snapshot_refuses_issuance(invented, monkeypatch): + original_roster = owner._native_roster + first = True + + def after_roster_read(value): + nonlocal first + result = original_roster(value) + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return result + + monkeypatch.setattr(owner, "_native_roster", after_roster_read) + with _refuses("NATIVE_FRAME_CHANGED"): + _load(invented) + + +def test_closed_default_manifest_has_no_caller_pin_or_prepared_authority( + invented, monkeypatch +): + fixture, frame, _h, _p = invented + pins = custody._ARCHIVE_PINS + monkeypatch.setattr(custody, "_ARCHIVE_PINS", None) + calls = [] + + def default_manifest(*args, **kwargs): + assert not args and not kwargs + calls.append(True) + return SimpleNamespace( + artifacts=tuple( + SimpleNamespace(role=r, filename=n, sha256=d, size_bytes=s) + for r, n, d, s in pins + ) + ) + + monkeypatch.setattr(custody, "load_acs_source_manifest", default_manifest) + result = _load(invented) + assert len(calls) == 2 + with pytest.raises(TypeError): + _load(invented, manifest={"sha256": "0" * 64}) + for constructor, payload in ( + (owner.AuthenticatedACSPersonCoverage, result.payload), + (owner.UngrantedACSNativeBinding, result.native_binding.receipt_json), + ): + with _refuses("SOURCE_CONSTRUCTOR|BINDING_CONSTRUCTOR"): + constructor(payload) + for forged in ( + frame.person, + {"frame": frame, "frame_sha256": custody.frame_content_sha256(frame)}, + "0" * 64, + ): + with _refuses("NATIVE_FRAME_REQUIRED"): + owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=forged + ) + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("SPORDER", 3, "NATIVE_ALIASES"), + ("A_LINENO", 3, "NATIVE_ALIASES"), + ("source_person_id", "2", "NATIVE_ALIASES"), + ("source_household_id", 2, "NATIVE_ALIASES"), + ("source_year", 2023, "NATIVE_VINTAGE"), + ("person_household_id", 2, "NATIVE_ALIASES"), + ("source_row_id", 1, "NATIVE_IDS"), + ], +) +def test_native_alias_and_membership_changes_with_same_person_ids_refuse( + invented, field, value, code +): + invented[1].person.loc[0, field] = value + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize( + "change", ["serial", "age", "membership", "weight", "source_row"] +) +def test_live_binding_refuses_same_ids_changed_native_parent(invented, change): + _fixture, frame, _h, _p = invented + result = _load(invented) + if change == "serial": + h = frame.table("household") + h["SERIALNO"] = h.SERIALNO.iloc[::-1].to_numpy() + elif change == "age": + frame.person.loc[2, "age"] = 82 + elif change == "membership": + for c in ("person_household_id", "source_household_id"): + frame.person.loc[[1, 2], c] = frame.person.loc[[2, 1], c].to_numpy() + elif change == "weight": + frame.table("household")["extra_cell"] = ( + 123 # all parent cells bind, including additions + ) + else: + frame.person.loc[0, "source_row_id"] = 100 + with _refuses( + "NATIVE_FRAME_CHANGED|NATIVE_HOUSEHOLD_COUNT|NATIVE_CONSISTENCY_REFUSED" + ): + owner.verify_acs_coverage_native_consistency(result, frame) + + +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "duplicate", + "household_np", + "household_missing", + "wrong_key", + "wrong_vintage", + ], +) +def test_source_complete_household_mismatch_refuses(invented, change): + _fixture, _frame, h, p = invented + if change == "missing": + p = p[:2] + elif change == "extra": + p += [{**p[1], "SPORDER": 3}] + elif change == "duplicate": + p[1] = p[0] + elif change == "household_np": + h[0]["NP"] = 1 + elif change == "household_missing": + h = h[:1] + elif change == "wrong_vintage": + p[0]["SERIALNO"] = "2023HU0000001" + else: + p[0]["SPORDER"] = 3 + _rewrite(invented, persons=p, households=h) + with _refuses("SOURCE_PERSON_ROSTER|SOURCE_HOUSEHOLD_ROSTER|SOURCE_DUPLICATE_KEY"): + _load(invented) + + +@pytest.mark.parametrize( + "age,mil,esr", + [ + ("", "", ""), + ("NA", "NA", "1.0"), + ("30 ", " 1", "4 "), + ("30", '1,"quoted"\r\n\t', "4\r\n\t"), + ("30\t", "1", "4"), + ("100", "0", "7"), + ], +) +@pytest.mark.parametrize("ending", ["\n", "\r", "\r\n"]) +def test_bom_multiline_tabs_and_malformed_literals_survive_envelope( + invented, age, mil, esr, ending +): + _fixture, frame, _h, p = invented + p[0].update(AGEP=age, MIL=mil, ESR=esr) + _rewrite(invented, persons=p, bom=True, ending=ending) + result = _load(invented) + assert result.table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert not result.table.isna().any().any() + assert frame.person.loc[0, "age"] == 30 + if age != "30": + assert result.table.loc[0, "MIL_state"] == "age_unresolved" + assert result.native_binding.receipt["original_age"]["relation"] == "unproven" + + +@pytest.mark.parametrize( + "column,value", [("AGEP", 82), ("A_AGE", 82), ("age", 82), ("age", float("nan"))] +) +def test_model_age_never_fills_or_authenticates_original_relation( + invented, column, value +): + frame = invented[1] + if isinstance(value, float): + frame.person[column] = frame.person[column].astype("float64") + frame.person.loc[2, column] = value + result = _load(invented) + assert result.table.loc[2, "AGEP"] == "80" + assert result.native_binding.receipt["original_age"]["mismatching_rows"] == 1 + assert result.native_binding.receipt["population_binding_authenticated"] is False + + +def test_candidate_reconstructed_before_comparison_and_self_rehash_cannot_authorize( + invented, monkeypatch +): + result = _load(invented) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes(result.payload) + assert _load(invented, candidate_path=candidate).payload == result.payload + header_raw, body = result._parts() + header = json.loads(header_raw) + tampered = body.replace(b'"30","1","4"', b'"30","2","4"', 1) + assert tampered != body + header["body_sha256"] = hashlib.sha256(tampered).hexdigest() + raw = owner._json(header) + candidate.write_bytes(owner._MAGIC + len(raw).to_bytes(4, "big") + raw + tampered) + with _refuses("CANDIDATE_MISMATCH"): + _load(invented, candidate_path=candidate) + flip_archive_byte(invented[0].person_zip) + original_copy = custody._copy + + def guarded_copy(source, destination, *args, **kwargs): + assert source != candidate, ( + "candidate was accessed before source reconstruction" + ) + return original_copy(source, destination, *args, **kwargs) + + monkeypatch.setattr(custody, "_copy", guarded_copy) + with _refuses("SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +@pytest.mark.parametrize("replacement", [None, "", "different"]) +def test_candidate_null_empty_and_field_contract_tampering_refuse( + invented, replacement +): + result = _load(invented) + header, body = result._parts() + rows = [json.loads(row) for row in body.splitlines()] + rows[1][3] = replacement + if replacement == "": + metadata = json.loads(header) + metadata["field_contract"]["fields"]["MIL"]["minimum_age"] = 16 + else: + metadata = json.loads(header) + changed_body = b"".join(owner._json(row) + b"\n" for row in rows) + metadata.update( + body_sha256=hashlib.sha256(changed_body).hexdigest(), + body_bytes=len(changed_body), + ) + changed_header = owner._json(metadata) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes( + owner._MAGIC + + len(changed_header).to_bytes(4, "big") + + changed_header + + changed_body + ) + with _refuses("CANDIDATE_MISMATCH|SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +def test_views_and_closed_objects_cannot_mutate_owned_evidence(invented): + result = _load(invented) + before = result.payload + view = result.table + view.loc[0, "MIL"] = "2" + result.receipt["field_contract"].clear() + result.native_binding.receipt["population_binding_authenticated"] = True + assert result.payload == before + assert result.table.loc[0, "MIL"] == "1" + assert result.native_binding.receipt["population_binding_authenticated"] is False + with pytest.raises(FrozenInstanceError): + result.payload = b"forged" + with _refuses("SOURCE_CONSTRUCTOR"): + replace(result, payload=b"forged") + + +@pytest.mark.parametrize( + "kind", ["duplicate", "case_duplicate", "unsafe", "symlink", "prefix_non_csv"] +) +def test_entire_zip_directory_is_checked_before_literal_reader( + invented, monkeypatch, kind +): + p = invented[3] + extra = { + "duplicate": Member("psam_pusa.csv", _csv(p)), + "case_duplicate": Member("PSAM_PUSA.CSV", _csv(p)), + "unsafe": Member("../unrelated.txt", b"ignored"), + "symlink": Member("unrelated.txt", b"target", external_attr=0o120777 << 16), + "prefix_non_csv": Member("psam_pus.txt", b"ignored"), + }[kind] + if kind == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + else: + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + monkeypatch.setattr( + literal, + "read_acs_person_coverage_columns", + lambda *a, **k: pytest.fail("reader entered"), + ) + with _refuses( + "ZIP_DUPLICATE_MEMBER|ZIP_MEMBER_PATH|ZIP_MEMBER_TYPE|ZIP_PREFIX_NONCSV" + ): + _load(invented) + + +@pytest.mark.parametrize( + "target,constant,value,code", + [ + (custody, "_MEMBER_COUNT_MAX", 1, "ZIP_MEMBER_COUNT"), + (custody, "_MEMBER_MAX", 100, "ZIP_MEMBER_SIZE"), + (custody, "_EXPANDED_MAX", 100, "ZIP_EXPANDED_SIZE"), + (owner, "MAX_CSV_HEADER_BYTES", 30, "CSV_RECORD_BYTES"), + (owner, "MAX_RECORD_BYTES", 64, "CSV_RECORD_BYTES"), + (owner, "MAX_TOKEN_BYTES", 10, "CSV_TOKEN_BYTES"), + (owner, "MAX_BODY_BYTES", 100, "SELECTED_BODY_BUDGET"), + (owner, "MAX_HEADER_BYTES", 100, "CANONICAL_SIZE"), + (literal, "MAX_ROWS", 1, "SOURCE_ROWS"), + (literal, "MAX_SELECTED_ROWS", 2, "NATIVE_ROWS"), + ], +) +def test_small_limits_refuse_before_relevant_allocation( + invented, monkeypatch, target, constant, value, code +): + monkeypatch.setattr(target, constant, value) + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize("kind", ["corrupt", "growth", "symlink", "fifo"]) +def test_exact_capture_refusals_are_static_and_hide_cause_chains(invented, kind): + path = invented[0].person_zip + if kind == "corrupt": + flip_archive_byte(path) + elif kind == "growth": + with path.open("ab") as stream: + stream.write(b"extra") + else: + saved = path.with_suffix(".saved") + path.rename(saved) + if kind == "symlink": + path.symlink_to(saved) + else: + os.mkfifo(path) + # Keep directory roster exact; saved bytes live outside the source dir. + saved.rename(invented[0].tmp_path / "saved.zip") + with _refuses("SOURCE_RECONSTRUCTION_REFUSED") as exc: + _load(invented) + assert exc.value.__cause__ is None + assert exc.value.__suppress_context__ + + +@pytest.mark.parametrize( + "race", ["private_bytes", "original_replace", "pins", "producer", "frame"] +) +def test_races_refuse_before_issuance(invented, monkeypatch, race): + original = literal.read_acs_person_coverage_columns + + def raced(source, **kwargs): + result = original(source, **kwargs) + if race == "private_bytes": + source.person_zip.chmod(0o600) + flip_archive_byte(source.person_zip) + elif race == "original_replace": + # Replace during _copy instead, below. + pass + elif race == "pins": + monkeypatch.setattr(custody, "_ARCHIVE_PINS", ()) + elif race == "producer": + monkeypatch.setattr(owner, "MAX_TOKEN_BYTES", owner.MAX_TOKEN_BYTES - 1) + else: + invented[1].person.loc[0, "age"] = 31 + return result + + if race == "original_replace": + old_identity = custody._identity + changed = False + + def replaced_identity(value): + nonlocal changed + if not changed: + changed = True + path = invented[0].household_zip + raw = path.read_bytes() + path.unlink() + path.write_bytes(raw) + return old_identity(value) + + monkeypatch.setattr(custody, "_identity", replaced_identity) + monkeypatch.setattr(literal, "read_acs_person_coverage_columns", raced) + with _refuses( + "SNAPSHOT_CHANGED|SOURCE_RECONSTRUCTION_REFUSED|PRODUCER_CHANGED|NATIVE_FRAME_CHANGED" + ): + _load(invented) + + +@pytest.mark.parametrize("ending", [b"\n", b"\r", b"\r\n", b""]) +def test_raw_record_byte_boundary_precedes_decoding_and_preserves_endings( + monkeypatch, ending +): + header = b"A,B\n" + body = b'"x\r\n\ty",z' + ending + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body)) + assert list(owner._records(io.BytesIO(header + body))) == [header, body] + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + with _refuses("CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(header + body))) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "\r\n\t", + '😀"\\\b\f', + ["", None, "é"], + {"b": True, "a": [1, -2, False]}, + ], +) +def test_canonical_size_is_checked_before_string_encoding(monkeypatch, value): + expected = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + assert owner._json(value, len(expected)) == expected + + def forbidden_encoder(*a, **k): + pytest.fail("JSON encoder allocated before preflight refusal") + + monkeypatch.setattr(owner.json, "JSONEncoder", forbidden_encoder) + with _refuses("CANONICAL_SIZE"): + owner._json(value, len(expected) - 1) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py new file mode 100644 index 000000000..5ac09804c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py @@ -0,0 +1,496 @@ +"""Literal coverage projections from invented, multi-member ACS archives.""" + +import csv +import hashlib +import io +import json +from zipfile import ZipFile + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_columns as module +from microcosm.build.us_runtime import acs_pums + + +def _row(serial="2024HU0000001", order="1", age="30", mil="4", esr="6"): + return dict(zip(module.READ_COLUMNS, (serial, order, age, mil, esr), strict=True)) + + +def _archive(tmp_path, members): + path = tmp_path / "persons.zip" + with ZipFile(path, "w") as archive: + for name, rows in members.items(): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + archive.writestr(name, stream.getvalue()) + return acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", path, max_households=1 + ) + + +def _keys(rows): + return pd.DataFrame(rows).loc[:, list(module.KEYS)] + + +def test_private_scanner_exhausts_members_despite_consumer_return_value(tmp_path): + first = _row(order="02", age="16", mil="", esr="1") + second = _row(order="01", age="17", mil="4", esr=" 6") + third = _row("2024GQ0000001", age="", mil='x,"\t', esr="") + source = _archive( + tmp_path, + {"psam_pusb.csv": [second, third], "psam_pusa.csv": [first]}, + ) + received = [] + + def consume(cells): + received.append(cells) + return True # No return value can turn a prefix into a successful scan. + + members, rows = module._scan_acs_person_coverage(source, consume) + assert members == ["psam_pusa.csv", "psam_pusb.csv"] + assert rows == 3 + assert received == [ + ("2024HU0000001", "02", "16", "", "1"), + ("2024HU0000001", "01", "17", "4", " 6"), + ("2024GQ0000001", "1", "", 'x,"\t', ""), + ] + + +@pytest.mark.parametrize( + "tail,match", + [ + ("2024HU0000009,1,30,4,6,EXTRA\n", "record width"), + ("2024HU0000009,1,30,4,6\x00\n", "forbidden control"), + ('2024HU0000009,1,30,"unfinished,6\n', "invalid literal CSV"), + ], +) +def test_private_scanner_refuses_invalid_tail_after_delivering_expected_rows( + tmp_path, tail, match +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,4,6\n" + tail, + ) + received = [] + with pytest.raises(ValueError, match=match): + module._scan_acs_person_coverage(source, received.append) + assert received == [("2024HU0000001", "1", "30", "4", "6")] + + +def test_exact_household_roster_across_members_preserves_blanks_and_request_order( + tmp_path, +): + rows = [ + _row(mil="1", esr="4"), + _row(order="2", age="10", mil="", esr=""), + _row("2024GQ0000001", age="16", mil="", esr="1"), + ] + unrelated = _row("2024HU0000009") + source = _archive( + tmp_path, + { + "nested/psam_pusb.csv": [rows[0], unrelated], + "psam_pusa.csv": [rows[2], rows[1]], + }, + ) + requested = _keys([rows[1], rows[2], rows[0]]) + requested["SPORDER"] = requested.SPORDER.astype("int64") + result, receipt = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=1 + ) + assert result.MIL.tolist() == ["", "", "1"] + assert result.ESR.tolist() == ["", "1", "4"] + assert result.MIL_state.tolist() == [ + "outside_age_universe", + "outside_age_universe", + "observed_code", + ] + assert result.ESR_state.tolist() == [ + "outside_age_universe", + "observed_code", + "observed_code", + ] + assert result.SERIALNO.tolist() == requested.SERIALNO.tolist() + assert result.SPORDER.tolist() == requested.SPORDER.tolist() + assert not result.isna().any().any() + assert receipt["source_rows_streamed"] == 4 + assert receipt["selected_person_rows"] == 3 + assert receipt["source_authenticated"] is False + assert receipt["release_eligible"] is False + assert receipt["legacy_max_households_applied"] is False + assert receipt["coverage_status"] == "literal_source_fields_only" + second, other = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=3 + ) + pd.testing.assert_frame_equal(result, second) + assert other == receipt + + +@pytest.mark.parametrize( + "age,mil,esr,mil_state,esr_state", + [ + ("17", "2", "2", "observed_code", "observed_code"), + ("45", "3", "5", "observed_code", "observed_code"), + ("30", "", "", "missing_in_universe", "missing_in_universe"), + ("30", "0", "7", "unlabelled_code", "unlabelled_code"), + ("30", "NA", "1.0", "unlabelled_code", "unlabelled_code"), + ("30", " 1", "4 ", "unlabelled_code", "unlabelled_code"), + ("15", "1", "4", "value_below_age_universe", "value_below_age_universe"), + ("", "1", "4", "age_unresolved", "age_unresolved"), + ("100", "4", "6", "age_unresolved", "age_unresolved"), + ], +) +def test_unresolved_states_preserve_literal_source_tokens( + tmp_path, age, mil, esr, mil_state, esr_state +): + row = _row(age=age, mil=mil, esr=esr) + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + table, _ = module.read_acs_person_coverage_columns(source, person_keys=_keys([row])) + assert table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert table.MIL_state.tolist() == [mil_state] + assert table.ESR_state.tolist() == [esr_state] + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("extra", "extra selected-household"), + ("missing", "native roster differ"), + ("duplicate", "keys repeat"), + ("wrong_order", "native roster differ"), + ], +) +def test_partial_household_or_conflicting_native_roster_refuses( + tmp_path, mutation, match +): + rows = [_row(), _row(order="2")] + observed = rows.copy() + if mutation == "extra": + observed += [_row(order="3")] + elif mutation == "missing": + observed.pop() + elif mutation == "duplicate": + observed[1] = rows[0] + else: + observed[1] = _row(order="3") + source = _archive(tmp_path, {"psam_pusa.csv": observed}) + with pytest.raises(ValueError, match=match): + module.read_acs_person_coverage_columns( + source, person_keys=_keys(rows), chunksize=1 + ) + + +@pytest.mark.parametrize( + "serial,order", + [ + ("2023HU0000001", "1"), + ("2024HU0000001", "0"), + ("2024HU0000001", "21"), + ("2024HU0000001", "1.0"), + ("2024HU0000001\x00other", "1"), + ("2024HU0000001", "1\x00other"), + (None, "1"), + ("2024HU0000001", True), + ], +) +def test_invalid_native_keys_refuse_before_file_read(tmp_path, serial, order): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + with pytest.raises(ValueError, match="native"): + module.read_acs_person_coverage_columns( + source, person_keys=pd.DataFrame({"SERIALNO": [serial], "SPORDER": [order]}) + ) + + +@pytest.mark.parametrize( + "mutation", ["missing_column", "duplicate_column", "duplicate_member"] +) +def test_ambiguous_source_schema_refuses(tmp_path, mutation): + source = _archive(tmp_path, {"psam_pusa.csv": [_row()]}) + with ZipFile(source.person_zip, "r") as archive: + raw = archive.read("psam_pusa.csv") + if mutation == "missing_column": + raw = raw.replace(b"MIL", b"OTHER") + elif mutation == "duplicate_column": + raw = raw.replace(b"ESR", b"MIL") + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + if mutation == "duplicate_member": + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("psam_pusa.csv", raw) + with pytest.raises(ValueError, match="missing or duplicated"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_explicit_row_bound_refuses_without_truncation(tmp_path, monkeypatch): + row = _row() + source = _archive(tmp_path, {"psam_pusa.csv": [row, _row("2024HU0000002")]}) + monkeypatch.setattr(module, "MAX_ROWS", 1) + with pytest.raises(ValueError, match="row bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([row]), chunksize=1 + ) + + +def test_selected_row_ceiling_refuses_before_opening_the_source(tmp_path, monkeypatch): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + monkeypatch.setattr(module, "MAX_SELECTED_ROWS", 1) + with pytest.raises(ValueError, match="selected person count is outside the bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]) + ) + + +def test_contract_is_owned_and_separate_from_legacy_native_roster(): + contract = module.coverage_field_contract() + assert contract["fields"]["MIL"]["minimum_age"] == 17 + assert contract["fields"]["ESR"]["minimum_age"] == 16 + contract["fields"]["MIL"]["codes"].clear() + assert len(module.coverage_field_contract()["fields"]["MIL"]["codes"]) == 4 + assert not {"MIL", "ESR"}.intersection(acs_pums._PERSON_REQUIRED) + + +def _raw_archive(tmp_path, raw): + source = acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", tmp_path / "literal-persons.zip" + ) + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + return source + + +@pytest.mark.parametrize("field", module.READ_COLUMNS) +def test_nul_suffix_refuses_before_truncating_value_or_identity(tmp_path, field): + row = _row(mil="1") + row[field] += "\x00garbage" + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "control", + [chr(code) for code in range(32) if code not in (9, 10, 13)] + + [chr(code) for code in range(127, 160)], +) +def test_non_csv_control_characters_refuse(tmp_path, control): + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil="1" + control)]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("location", ["header", "ignored_column", "unselected_row"]) +def test_control_validation_precedes_column_and_household_selection(tmp_path, location): + header = "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + body = "2024HU0000001,1,30,1,6,ok\n" + if location == "header": + header = header.replace("OTHER", "OTHER\x00suffix") + elif location == "ignored_column": + body = body.replace("ok", "ok\x00suffix") + else: + body += "2024HU0000009,1,30,1\x00suffix,6,ok\n" + source = _raw_archive(tmp_path, header + body) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("serial", ["2024HU0000001", "2024HU0000009"]) +@pytest.mark.parametrize( + "tail", + ["1,30,4", "1,30,4,6", "1,30,4,6,ok,extra", "1,30,4,6,ok,"], + ids=["missing_required", "missing_unused", "too_many", "trailing_delimiter"], +) +def test_full_record_width_refuses_even_outside_projection(tmp_path, serial, tail): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + serial + "," + tail + "\n", + ) + with pytest.raises(ValueError, match="record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "body", + ["\n", '2024HU0000001,1,30,4,"unterminated\n', '2024HU0000001,1,30,"1"garbage,6\n'], +) +def test_blank_records_and_malformed_quoting_refuse(tmp_path, body): + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + with pytest.raises(ValueError, match="CSV|record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["\n", "\r", "\r\n"]) +def test_quoted_delimiters_newlines_tabs_and_digest_preserve_tokens(tmp_path, line_end): + stream = io.StringIO(newline="") + writer = csv.writer(stream, quoting=csv.QUOTE_ALL, lineterminator=line_end) + writer.writerow(["OTHER", "ESR", "MIL", "AGEP", "SPORDER", "SERIALNO"]) + writer.writerow( + [ + 'ignored, "quoted"\r\nfield', + "4\r\n\t", + '1,"quoted"\n\r', + "30", + "01", + "2024HU0000001", + ] + ) + source = _raw_archive(tmp_path, "\ufeff" + stream.getvalue()) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + expected = [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "AGEP": "30", + "MIL": '1,"quoted"\n\r', + "ESR": "4\r\n\t", + "MIL_state": "unlabelled_code", + "ESR_state": "unlabelled_code", + } + ] + assert table.to_dict("records") == expected + assert receipt["source_rows_streamed"] == 1 + + def digest(records): + return hashlib.sha256( + json.dumps( + records, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + assert receipt["projection_sha256"] == digest(expected) + for token in ("4\n\t", "4\r\t", r"4\r\n\t", "4"): + assert receipt["projection_sha256"] != digest([{**expected[0], "ESR": token}]) + + +@pytest.mark.parametrize("field", ["AGEP", "MIL", "ESR"]) +@pytest.mark.parametrize("control", ["\t", "\r", "\n", "\r\n"]) +def test_accepted_csv_whitespace_is_literal_and_never_an_observed_code( + tmp_path, field, control +): + row = _row(mil="1", esr="4") + row[field] += control + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, quoting=csv.QUOTE_ALL + ) + writer.writeheader() + writer.writerow(row) + source = _raw_archive(tmp_path, stream.getvalue()) + table, _ = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.loc[0, field] == row[field] + if field == "AGEP": + assert ( + table.MIL_state.tolist() == table.ESR_state.tolist() == ["age_unresolved"] + ) + else: + assert table[field + "_state"].tolist() == ["unlabelled_code"] + + +@pytest.mark.parametrize("field", module.KEYS) +@pytest.mark.parametrize("suffix", ["\t", "\n", "\r\n", " other", ",other"]) +def test_literal_identity_suffix_cannot_join_canonical_requested_key( + tmp_path, field, suffix +): + row = _row() + row[field] += suffix + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="missing|native"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_duplicate_canonical_person_order_in_later_member_refuses(tmp_path): + source = _archive( + tmp_path, + {"psam_pusa.csv": [_row(order="01")], "psam_pusb.csv": [_row(order="1")]}, + ) + with pytest.raises(ValueError, match="keys repeat"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]), chunksize=1 + ) + + +def test_invalid_utf8_refuses(tmp_path): + source = _raw_archive( + tmp_path, b"SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,1\xff,6\n" + ) + with pytest.raises(ValueError): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("multiline", [False, True]) +def test_record_character_bound_includes_all_quoted_physical_lines( + tmp_path, monkeypatch, multiline +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100, raising=False) + suffix = ("x\n" if multiline else "xx") * 60 + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil=suffix)]}) + with pytest.raises(ValueError, match="record character bound"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["", "\n", "\r\n"]) +def test_record_character_bound_accepts_exact_limit_and_resets_per_record( + tmp_path, monkeypatch, line_end +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100) + prefix, suffix = '2024HU0000001,1,30,"', '",6' + line_end + token = "x\n" + "x" * (100 - len(prefix) - len(suffix) - 2) + body = prefix + token + suffix + assert len(body) == 100 + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.MIL.tolist() == [token] + assert receipt["source_rows_streamed"] == 1 + + +def test_stream_opens_member_once_and_stops_at_refusal_without_full_validation( + tmp_path, monkeypatch +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + + "2024HU0000001,1,30,4,6\n" * 2 + + "2024HU0000009,1,30,4,6\n" * 20_000, + ) + opened, consumed = [], [] + + class GuardedZipFile(ZipFile): + def open(self, name, *args, **kwargs): + opened.append(name) + member = super().open(name, *args, **kwargs) + original_read, original_read1 = member.read, member.read1 + + def bounded(method, size=-1): + assert size >= 0, "unbounded archive read" + result = method(size) + consumed.append(len(result)) + return result + + member.read = lambda size=-1: bounded(original_read, size) + member.read1 = lambda size=-1: bounded(original_read1, size) + return member + + def read(self, *args, **kwargs): + pytest.fail("whole-member validation read") + + monkeypatch.setattr(module, "ZipFile", GuardedZipFile) + with pytest.raises(ValueError, match="extra selected-household"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + assert opened == ["psam_pusa.csv"] + assert sum(consumed) < 32_768 diff --git a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py new file mode 100644 index 000000000..ab98b77dc --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py @@ -0,0 +1,966 @@ +"""Actual pinned invented archives; never a native Frame or fake source issuer.""" + +import _csv +import ast +import copy +import csv +import gc +import hashlib +import inspect +import io +import json +import pickle +import sys +import weakref +import zipfile +from pathlib import Path +from types import CodeType, FunctionType, SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import acs_population_catalogue as owner +from microcosm.frame import Frame + + +def csv_bytes(rows): + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return stream.getvalue().encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + source, snapshots = tmp_path / "source", tmp_path / "snapshots" + source.mkdir() + snapshots.mkdir() + households = [ + dict( + SERIALNO="2024HU0000001", + TYPEHUGQ="1", + NP="2", + WGTP="0010", + TEN="1", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000001", + TYPEHUGQ="2", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000002", + TYPEHUGQ="1", + NP="0", + WGTP="20", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000002", + TYPEHUGQ="3", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000003", + TYPEHUGQ="1", + NP="1", + WGTP="30", + TEN="3", + ST="01", + PUMA="00100", + ), + ] + people = [ + dict( + SERIALNO="2024HU0000001", + SPORDER="01", + PWGTP="0011", + AGEP="30", + MIL="1", + ESR="4", + ), + dict( + SERIALNO="2024GQ0000001", + SPORDER="01", + PWGTP="0077", + AGEP="40", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000001", + SPORDER="02", + PWGTP="12", + AGEP="15", + MIL="", + ESR="", + ), + dict( + SERIALNO="2024GQ0000002", + SPORDER="1", + PWGTP="78", + AGEP="50", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000003", SPORDER="1", PWGTP="31", AGEP="99", MIL="", ESR="" + ), + ] + + def write(): + for _role, filename, prefix, rows in ( + ("household", "csv_hus.zip", "psam_hus", households), + ("person", "csv_pus.zip", "psam_pus", people), + ): + with zipfile.ZipFile(source / filename, "w") as archive: + empty = ( + b"SERIALNO,SPORDER,PWGTP,AGEP,MIL,ESR\n" + if _role == "person" + else b"SERIALNO,TYPEHUGQ,NP,WGTP,TEN,ST,PUMA\n" + ) + archive.writestr( + prefix + "a.csv", csv_bytes(rows[:2]) if rows[:2] else empty + ) + archive.writestr( + prefix + "b.csv", csv_bytes(rows[2:]) if rows[2:] else empty + ) + pins = tuple( + ( + role, + name, + hashlib.sha256((source / name).read_bytes()).hexdigest(), + (source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", pins) + + monkeypatch.setattr( + owner.housing.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=64 * 1024**3), + ) + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 2) + write() + return SimpleNamespace( + source=source, + snapshots=snapshots, + households=households, + people=people, + write=write, + root=tmp_path, + ) + + +def issue(fixture, **kwargs): + return owner.issue_acs_source_catalogue( + fixture.source, snapshot_root=fixture.snapshots, **kwargs + ) + + +def test_real_complete_catalogue_without_population_or_classification(invented): + forbidden = [] + + def trace(frame, event, arg): + module = frame.f_globals.get("__name__", "") + if event == "call" and ( + frame.f_code is Frame.__init__.__code__ + or module.startswith("microunit") + or frame.f_code.co_name + in { + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "classify_household", + "classify_households", + } + ): + forbidden.append(frame.f_code.co_name) + raise AssertionError("Population construction/classification occurred") + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + rows = result.households + receipt = result.receipt + finally: + sys.setprofile(previous) + assert not forbidden + assert result.validate() is result + assert owner.verify_acs_source_catalogue(result) is result + assert len(rows) == 4 and len(result.exclusion_ledger) == 1 + assert {r.key.native_id for r in rows} == { + r["SERIALNO"] for r in invented.households if r["NP"] != "0" + } + row = rows[0] + assert row.wgtp == "0010" + assert [(p.sporder, p.age, p.esr, p.mil, p.pwgtp) for p in row.persons] == [ + ("01", "30", "4", "1", "0011"), + ("02", "15", "", "", "12"), + ] + assert row.persons[1].esr_state == "outside_age_universe" + assert rows[-1].persons[0].esr_state == "missing_in_universe" + assert rows[1].persons[0].pwgtp == "0077" + assert result.lineage[row.key.native_id]["persons"] == ( + ("01", "psam_pusa.csv", 1), + ("02", "psam_pusb.csv", 1), + ) + assert receipt["counts"]["people"] == 5 + assert receipt["counts"]["literal_batches"] == 3 + assert ( + receipt["source_authenticated"] + and not receipt["population_binding_authenticated"] + ) + assert ( + not receipt["domain_assignment_authenticated"] + and not receipt["selection_performed"] + ) + assert not receipt["execution"]["full_person_scan_per_literal_batch"] + assert receipt["execution"]["literal_full_scans"] == 1 + assert issue(invented, candidate=result.to_bytes()).to_bytes() == result.to_bytes() + (invented.root / "catalogue.receipt.json").write_bytes(result.to_bytes()) + held = owner._lookup(result) + (invented.root / "invented-records.json").write_bytes( + owner.native.coverage._json( + {"households": held.records, "vacancy_ledger": held.vacancies} + ) + ) + + +def test_independent_views_cannot_rewrite_issued_rows(invented): + result = issue(invented) + original = result.to_bytes() + row = result.households[0] + object.__setattr__(row, "wgtp", "9999") + object.__setattr__(row.persons[0], "esr", "6") + result.receipt["source_authenticated"] = False + result.lineage.clear() + assert result.households[0].wgtp == "0010" + assert result.households[0].persons[0].esr == "4" + assert result.to_bytes() == original + + +def expected_records(): + """Explicit invented source-order records, independent of collector logic.""" + return ( + ( + "2024HU0000001", + "1", + "2", + "0010", + "psam_husa.csv", + 1, + ( + ( + "01", + "30", + "4", + "observed_code", + "1", + "observed_code", + "0011", + "psam_pusa.csv", + 1, + ), + ( + "02", + "15", + "", + "outside_age_universe", + "", + "outside_age_universe", + "12", + "psam_pusb.csv", + 1, + ), + ), + ), + ( + "2024GQ0000001", + "2", + "1", + "0", + "psam_husa.csv", + 2, + ( + ( + "01", + "40", + "6", + "observed_code", + "4", + "observed_code", + "0077", + "psam_pusa.csv", + 2, + ), + ), + ), + ("2024HU0000002", "1", "0", "20", "psam_husb.csv", 1, ()), + ( + "2024GQ0000002", + "3", + "1", + "0", + "psam_husb.csv", + 2, + ( + ( + "1", + "50", + "6", + "observed_code", + "4", + "observed_code", + "78", + "psam_pusb.csv", + 2, + ), + ), + ), + ( + "2024HU0000003", + "1", + "1", + "30", + "psam_husb.csv", + 3, + ( + ( + "1", + "99", + "", + "missing_in_universe", + "", + "missing_in_universe", + "31", + "psam_pusb.csv", + 3, + ), + ), + ), + ) + + +@pytest.mark.parametrize("batch_limit", [2, 3, 5]) +def test_explicit_canonical_records_and_bytes_ignore_batch_boundaries( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + result = issue(invented) + held, expected = owner._lookup(result), expected_records() + assert held.records == (expected[0], expected[1], expected[3], expected[4]) + assert held.vacancies == (expected[2],) + raw = b"".join( + json.dumps(row, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + b"\n" + for row in expected + ) + assert result.receipt["counts"]["canonical_record_bytes"] == len(raw) + assert ( + result.receipt["counts"]["canonical_record_sha256"] + == hashlib.sha256(raw).hexdigest() + ) + + +@pytest.mark.parametrize("batch_limit", [2, 5]) +def test_catalogue_literal_member_scan_count_is_independent_of_batches( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + scanner = owner.literal._scan_acs_person_coverage.__code__ + active, scans, literal_opens, all_person_opens = set(), [], [], [] + + def trace(frame, event, arg): + if frame.f_code is scanner: + if event == "call": + scans.append(True) + active.add(id(frame)) + elif event == "return": + active.remove(id(frame)) + if event == "call" and frame.f_code is zipfile.ZipFile.open.__code__: + name = frame.f_locals["name"] + name = name.filename if isinstance(name, zipfile.ZipInfo) else name + if name.startswith("psam_pus"): + all_person_opens.append(name) + if active: + literal_opens.append(name) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert scans == [True] and not active + assert literal_opens == ["psam_pusa.csv", "psam_pusb.csv"] + assert all_person_opens == literal_opens * 3 + assert result.receipt["counts"]["literal_batches"] == (3 if batch_limit == 2 else 1) + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("duplicate", "DUPLICATE_LITERAL_PERSON"), + ("extra", "UNEXPECTED_LITERAL_PERSON"), + ("orphan", "UNEXPECTED_LITERAL_PERSON"), + ("missing", "GLOBAL_PERSON_KEYS"), + ("key_suffix", "LITERAL_PERSON_KEY"), + ], +) +def test_collector_reconciles_every_literal_slot_after_housing_validation( + invented, mutation, code +): + # Exercise this private reconciliation separately from the unchanged + # housing/preflight checks. A mismatched helper input issues no authority. + original = issue(invented) + projection = json.loads(owner._lookup(original).projection_path.read_bytes()) + if mutation == "duplicate": + invented.people.append(dict(invented.people[0], SPORDER="1")) + elif mutation == "extra": + invented.people.append(dict(invented.people[0], SPORDER="3")) + elif mutation == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif mutation == "missing": + invented.people.pop() + else: + invented.people[-1]["SPORDER"] = "1 " + invented.write() + paths = { + "household": invented.source / "csv_hus.zip", + "person": invented.source / "csv_pus.zip", + } + with pytest.raises(owner.ACSSourceCatalogueError, match=f"^{code}$"): + owner._collect(projection, paths) + + +def test_prospective_byte_charge_refuses_before_retaining_the_next_person( + invented, monkeypatch +): + original = issue(invented) + held = owner._lookup(original) + projection = json.loads(held.projection_path.read_bytes()) + expected = expected_records() + + def encoded(value): + return json.dumps(value, separators=(",", ":"), allow_nan=False).encode() + + overhead = sum(len(encoded((*row[:6], ()))) + 1 for row in expected) + monkeypatch.setattr( + owner, "_CATALOGUE_BYTES", overhead + len(encoded(expected[0][6][0])) + ) + retained = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_globals.get("__name__") == owner.__name__ + and frame.f_code.co_qualname == "_collect..consume_row" + ): + retained.append( + sum(person is not None for person in frame.f_locals["people"]) + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.native.coverage.ACSCoverageAuthenticationError, + match="^CANONICAL_SIZE$", + ): + owner._collect(projection, dict(held.paths)) + finally: + sys.setprofile(previous) + assert retained == [1, 1] + + +def test_source_order_and_original_person_order_survive_reverse_zip_insertion(invented): + # Keep serials unsorted and make the split household's raw person order + # descending; canonical order remains source-member then source-row order. + invented.people[0]["SPORDER"], invented.people[2]["SPORDER"] = "02", "01" + invented.write() + first = issue(invented) + before = owner._lookup(first) + for name in ("csv_hus.zip", "csv_pus.zip"): + path = invented.source / name + with zipfile.ZipFile(path) as archive: + members = [ + (member.filename, archive.read(member)) for member in archive.infolist() + ] + with zipfile.ZipFile(path, "w") as archive: + for member, raw in reversed(members): + archive.writestr(member, raw) + pins = tuple( + ( + role, + name, + hashlib.sha256((invented.source / name).read_bytes()).hexdigest(), + (invented.source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + # New physical ZIPs have their own invented pins; compare source records, + # not archive/producer receipts from different physical artifacts. + owner.housing._ARCHIVE_PINS = pins + second = issue(invented) + after = owner._lookup(second) + assert after.records == before.records and after.vacancies == before.vacancies + assert [p[0] for p in after.records[0][6]] == ["02", "01"] + + +@pytest.mark.parametrize("kind", ["new", "copy", "deepcopy", "pickle", "payload"]) +def test_foreign_changed_or_copied_authority_refuses(invented, kind): + result = issue(invented) + if kind == "new": + other = object.__new__(owner.AuthenticatedACSSourceCatalogue) + object.__setattr__(other, "payload", result.payload) + elif kind == "payload": + other = result + raw = json.loads(result.payload) + raw["counts"]["people"] = 16 + object.__setattr__(other, "payload", owner.native.coverage._json(raw)) + else: + try: + other = { + "copy": copy.copy, + "deepcopy": copy.deepcopy, + "pickle": lambda x: pickle.loads(pickle.dumps(x)), + }[kind](result) + except (TypeError, owner.ACSSourceCatalogueError): + return + with pytest.raises( + owner.ACSSourceCatalogueError, match="^ISSUANCE_(NOT_OWNED|CHANGED)$" + ): + other.validate() + + +def test_independent_reissue_and_weak_cleanup(invented): + one, two = issue(invented), issue(invented) + assert one.to_bytes() == two.to_bytes() and one is not two + identity, reference = id(one), weakref.ref(one) + del one + gc.collect() + assert reference() is None and identity not in owner._ISSUED + assert two.validate() is two + + +@pytest.mark.parametrize("kind", ["duplicate", "orphan", "missing", "late_extra", "np"]) +def test_global_complete_roster_refuses_across_members_and_batches(invented, kind): + if kind == "duplicate": + invented.people[-1] = dict(invented.people[0]) + elif kind == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif kind == "missing": + invented.people.pop() + elif kind == "late_extra": + extra = dict(invented.people[0], SPORDER="3") + invented.people.append(extra) + else: + invented.households[0]["NP"] = "3" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize( + "cap,value", + [ + ("_ARCHIVE_BYTES", 1), + ("_EXPANDED_BYTES", 1), + ("_MEMBERS", 1), + ("_SOURCE_ROWS", 1), + ("_BATCH_PEOPLE", 1), + ("_CATALOGUE_BYTES", 1), + ("_RECEIPT_BYTES", 1), + ], +) +def test_prospective_small_caps_refuse(invented, monkeypatch, cap, value): + monkeypatch.setattr(owner, cap, value) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize("escaped_literals", [False, True]) +def test_byte_ceiling_exact_boundary_is_record_based( + invented, monkeypatch, escaped_literals +): + if escaped_literals: + invented.people[0]["MIL"] = 'x,"\\\t' + invented.people[0]["ESR"] = "é" + invented.write() + size = issue(invented).receipt["counts"]["canonical_record_bytes"] + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size) + assert issue(invented).receipt["counts"]["canonical_record_bytes"] == size + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size - 1) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +def test_candidate_is_reconstructed_and_cannot_authorize_changed_values(invented): + result = issue(invented) + candidate = result.to_bytes() + invented.people[0]["ESR"] = "6" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError, match="^CANDIDATE_MISMATCH$"): + issue(invented, candidate=candidate) + assert issue(invented).households[0].persons[0].esr == "6" + with pytest.raises( + owner.ACSSourceCatalogueError, match="^SOURCE_AUTHORITY_CHANGED$" + ): + result.validate() + + +@pytest.mark.parametrize("target", ["original", "projection", "producer"]) +def test_live_source_projection_and_producer_changes_refuse( + invented, monkeypatch, target +): + result = issue(invented) + if target == "original": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + elif target == "projection": + owned = owner._lookup(result) + raw = owned.projection_path.read_bytes() + owned.projection_path.chmod(0o600) + owned.projection_path.write_bytes(b" " + raw[1:]) + else: + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 3) + with pytest.raises(owner.ACSSourceCatalogueError): + result.validate() + + +def test_new_owner_static_surface_excludes_population_selection_and_classification(): + tree = ast.parse(Path(owner.__file__).read_text()) + calls = { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, (ast.Name, ast.Attribute)) + } + assert not calls & { + "Frame", + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "assign_us_unit_structure", + "classify_household", + "classify_households", + "issue_acs_native_coverage", + "sample", + "choice", + } + assert {"_reconstruct", "_scan_acs_person_coverage", "_live_code"} <= calls + assert "read_acs_person_coverage_columns" not in calls + + +@pytest.mark.parametrize("mutation", ["source", "limit"]) +def test_mutations_at_literal_return_refuse(invented, mutation): + target = owner.literal._scan_acs_person_coverage + fired = [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + if mutation == "source": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + else: + owner._BATCH_PEOPLE = 3 + + before, previous = owner._BATCH_PEOPLE, sys.getprofile() + sys.setprofile(timing) + try: + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + finally: + sys.setprofile(previous) + owner._BATCH_PEOPLE = before + assert fired + + +def test_returned_housing_capsule_cannot_replace_actual_projected_anchors(invented): + target, fired = owner.housing._reconstruct, [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + changed = json.loads(arg.projection_json) + changed["households"][0][changed["household_columns"].index("WGTP")] = ( + "9999" + ) + object.__setattr__(arg, "projection_json", owner.housing._json(changed)) + + previous = sys.getprofile() + sys.setprofile(timing) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert fired and result.households[0].wgtp == "0010" + + +@pytest.mark.parametrize("aliases", ["csv", "native", "both"]) +@pytest.mark.parametrize("moment", ["entry", "before_call"]) +def test_actual_csv_alias_timing_cannot_supply_catalogue_values( + invented, aliases, moment +): + target = owner.literal._literal_csv_records + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "reader = csv_reader(" in line + ) + original, native, fired, calls = csv.reader, _csv.reader, [], [] + + def restore(): + csv.reader, _csv.reader = original, native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + return original(*args, **kwargs) + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + if aliases in ("csv", "both"): + csv.reader = replacement + if aliases in ("native", "both"): + _csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + if moment == "entry": + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + else: + result = issue(invented) + finally: + sys.settrace(previous) + restore() + assert fired and not calls + if moment == "before_call": + assert result.households[0].persons[0].esr == "4" + + +def test_public_constructor_has_no_authority(invented): + raw = issue(invented).to_bytes() + with pytest.raises(owner.ACSSourceCatalogueError, match="^ISSUANCE_CONSTRUCTOR$"): + owner.AuthenticatedACSSourceCatalogue(raw) + + +def test_full_source_can_exceed_unchanged_selected_reader_cap(invented, monkeypatch): + monkeypatch.setattr(owner.literal, "MAX_SELECTED_ROWS", 2) + result = issue(invented) + assert result.receipt["counts"]["people"] == 5 + assert result.receipt["counts"]["literal_batches"] == 3 + + +def test_vacancy_only_catalogue_has_no_literal_batch(invented): + invented.households[:] = [dict(invented.households[2])] + invented.people.clear() + invented.write() + result = issue(invented) + assert result.households == () + assert len(result.exclusion_ledger) == 1 + assert result.exclusion_ledger[0].persons == () + assert result.receipt["counts"]["literal_batches"] == 0 + assert result.receipt["execution"]["literal_full_scans"] == 0 + + +def test_reordered_literal_column_declaration_refuses_before_capture( + invented, monkeypatch +): + monkeypatch.setattr( + owner.literal, "READ_COLUMNS", ("SERIALNO", "SPORDER", "ESR", "MIL", "AGEP") + ) + with pytest.raises( + owner.ACSSourceCatalogueError, match="^LITERAL_CONTRACT_CHANGED$" + ): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] + + +@pytest.mark.parametrize( + "interface", + ["validate", "to_bytes", "receipt", "households", "exclusion_ledger", "lineage"], +) +@pytest.mark.parametrize("moment", ["source_checks", "final_producer"]) +def test_final_source_registry_seal_all_borrows( + invented, monkeypatch, interface, moment +): + result = issue(invented) + pins = owner.housing._ARCHIVE_PINS + changed = tuple((r, n, "0" * 64, s) for r, n, d, s in pins) + fired, producers = [], [] + + def timing(frame, event, arg): + if event != "return": + return + if frame.f_code is owner._producer.__code__: + producers.append(True) + if not fired and ( + moment == "source_checks" + and frame.f_code is owner._source_checks.__code__ + or moment == "final_producer" + and frame.f_code is owner._producer.__code__ + and len(producers) == 2 + ): + fired.append(True) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises( + owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_CHANGED" + ): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize("semantic", [False, True]) +def test_import_time_bytes_and_matching_live_code_remain_authority( + invented, monkeypatch, semantic +): + source = Path(owner.__file__).read_text() + if semantic: + before = " weight,\n tuple(" + assert source.count(before) == 1 + source = source.replace(before, ' "9999",\n tuple(') + else: + source += "\n# invented drift after the real module import\n" + replacement = invented.root / "changed-catalogue.py" + replacement.write_text(source) + monkeypatch.setattr(owner, "__file__", str(replacement)) + if semantic: + code = next( + c + for c in compile( + source, str(replacement), "exec", dont_inherit=True + ).co_consts + if isinstance(c, CodeType) and c.co_name == "_household" + ) + monkeypatch.setattr(owner._household, "__code__", code) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("change", ["bytes", "generated_constructor"]) +def test_raw_domain_import_authority_precedes_first_issuance( + invented, monkeypatch, change +): + if change == "bytes": + replacement = invented.root / "changed-raw-types.py" + replacement.write_bytes( + Path(owner.domains.__file__).read_bytes() + b"\n# drift\n" + ) + monkeypatch.setattr(owner.domains, "__file__", str(replacement)) + else: + # Native live-code traversal explicitly skips generated code; + # the local import-time seal must still bind those actual constructors. + code = next( + c + for c in compile( + "def __init__(self, *args, **kwargs): pass", "", "exec" + ).co_consts + if isinstance(c, CodeType) + ) + monkeypatch.setattr( + owner.domains.AcsHousehold, + "__init__", + FunctionType(code, vars(owner.domains)), + ) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("issuance", [False, True]) +@pytest.mark.parametrize("change", ["pins", "live_code"]) +def test_final_producer_return_seal_issuance_and_borrow( + invented, monkeypatch, issuance, change +): + result = None if issuance else issue(invented) + fired, producers = [], [] + original = owner._household + + def changed(record): + return original(record) + + def timing(frame, event, arg): + if event == "return" and frame.f_code is owner._producer.__code__: + producers.append(True) + if len(producers) == 2: + fired.append(True) + if change == "pins": + monkeypatch.setattr( + owner.housing, + "_ARCHIVE_PINS", + tuple( + (r, n, "0" * 64, s) + for r, n, d, s in owner.housing._ARCHIVE_PINS + ), + ) + else: + monkeypatch.setattr(owner, "_household", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) if issuance else result.households + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize( + "shape", ["outer_list", "inner_list", "string_subclass", "bool_size"] +) +def test_source_authority_requires_deeply_immutable_primitive_pins( + invented, monkeypatch, shape +): + original = owner.housing._ARCHIVE_PINS + if shape == "outer_list": + changed = list(original) + elif shape == "inner_list": + changed = tuple(list(pin) for pin in original) + elif shape == "string_subclass": + + class MutableString(str): + pass + + changed = ((MutableString(original[0][0]), *original[0][1:]), original[1]) + else: + changed = ((*original[0][:3], True), original[1]) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + with pytest.raises(owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_TYPE"): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] diff --git a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py new file mode 100644 index 000000000..32ff4bbb6 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py @@ -0,0 +1,664 @@ +"""Real closed loaders over invented source bytes; no genuine source admission.""" + +import csv +import hashlib +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_2024_native_population as native +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_household_observations as household_owner +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.frame import Frame, WeightKind, Weights + + +def _helpers(): + spec = importlib.util.spec_from_file_location( + "native_2024_invented_fixture", + Path(__file__).with_name("test_us_asec_coverage_authentication.py"), + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rebuild_observations( + tmp_path, + monkeypatch, + persons, + *, + extra_household, + legacy_weights, + missing_money=None, +): + """Build larger invented real checkpoints and the actual observation attachment.""" + parent_path = tmp_path / "invented-original-v4.h5" + household_path = tmp_path / "invented-household-attachment.h5" + loaded = load_frame_checkpoint(parent_path) + tables = {e: loaded.frame.table(e).copy(deep=True) for e in loaded.frame.entities} + strata = loaded.frame.strata.copy(deep=True) + if extra_household: + extra = tables["person"].iloc[-2:].copy(deep=True) + extra["person_id"] = np.array([107, 108], dtype=np.int64) + extra["PERIDNUM"] = pd.array([str(i).zfill(22) for i in (7, 8)], dtype="string") + extra["source_household_id"] = 8 + extra["PH_SEQ"] = 4 + for entity in loaded.frame.schema.group_entities: + extra[f"person_{entity}_id"] = 4 + group = tables[entity].iloc[-1:].copy(deep=True) + group[f"{entity}_id"] = 4 + tables[entity] = pd.concat([tables[entity], group], ignore_index=True) + tables["person"] = pd.concat([tables["person"], extra]) + strata = pd.concat([strata, strata.iloc[-2:]]) + cohort_path = tmp_path / "invented-cohort-2024.h5" + cohort = pd.read_hdf(cohort_path, key="household") + extra_cohort = cohort.iloc[-1:].copy(deep=True) + extra_cohort["H_SEQ"] = 8 + pd.concat([cohort, extra_cohort], ignore_index=True).to_hdf( + cohort_path, key="household", format="fixed", mode="w" + ) + + def add_people(rows): + extra_rows = [r.copy() for r in rows[1:]] + for row in extra_rows: + row[0] = str(int(row[0]) + 2).zfill(22) + row[1] = "8" + rows.extend(extra_rows) + + _helpers()._rewrite(persons[2024], add_people) + if missing_money is not None: + year, field = missing_money + assert (year, field) in ((2024, "ANN_VAL"), (2022, "WSAL_VAL")) + person = tables["person"] + position = np.flatnonzero(person.source_year.to_numpy() == year)[0] + # Alter only invented source construction, before identities, attachments + # and normal issuance. Never mutate an already authenticated parent. + person.iloc[position, person.columns.get_loc(field)] = np.nan + values = ( + legacy_weights + if legacy_weights is not None + else ([1, 2, 3, 9] if extra_household else [1, 2, 3]) + ) + frame = Frame( + tables, + loaded.frame.schema, + {"household": Weights(np.array(values), WeightKind.DESIGN)}, + strata, + ) + metadata = loaded.metadata + for key in ("identity", "source_construction_identity"): + metadata[key] = frame_identity(frame).to_payload() + sources = [] + for year in (2022, 2023, 2024): + path = tmp_path / f"invented-cohort-{year}.h5" + sources.append( + household_owner.AsecHouseholdObservationSource( + year, path, hashlib.sha256(path.read_bytes()).hexdigest() + ) + ) + metadata["source_receipt"]["sources"] = [ + {"year": source.year, "path": str(source.path), "sha256": source.sha256} + for source in sources + ] + write_frame_checkpoint(parent_path, frame, metadata=metadata) + parent_sha = hashlib.sha256(parent_path.read_bytes()).hexdigest() + attachment = household_owner.with_asec_household_observations( + frame, + checkpoint_metadata=metadata, + checkpoint_sha256=parent_sha, + sources=sources, + ) + write_frame_checkpoint( + household_path, + attachment.frame, + metadata={ + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observations_source", + "parent_checkpoint_sha256": parent_sha, + "household_observations": attachment.receipt, + }, + ) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + parent_sha, + hashlib.sha256(household_path.read_bytes()).hexdigest(), + tuple((s.year, s.sha256) for s in sources), + ), + ) + _helpers()._pin(persons, monkeypatch) + + +def _fixture( + tmp_path, + monkeypatch, + *, + household_rows=None, + extra_household=False, + legacy_weights=None, + missing_money=None, + **changes, +): + helpers = _helpers() + _, persons = helpers._fixtures(tmp_path, monkeypatch, **changes) + if extra_household or legacy_weights is not None or missing_money is not None: + _rebuild_observations( + tmp_path, + monkeypatch, + persons, + extra_household=extra_household, + legacy_weights=legacy_weights, + missing_money=missing_money, + ) + # Both real owners must pin the same final invented original member bytes. + monkeypatch.setattr(restoration, "_MEMBER_PINS", coverage._MEMBER_PINS) + parent = tmp_path / "invented-original-v4.h5" + household = tmp_path / "invented-household-attachment.h5" + output = tmp_path / "restored" + restoration.restore_asec_person_income_source( + parent, household, member_paths=persons, output_dir=output + ) + member = tmp_path / "hhpub25.csv" + rows = ( + household_rows + if household_rows is not None + else [ + ["00007", "1", "000255212", "6", "1", "2"], + ["00008", "2", "", "0", "0", "0"], + ] + ) + if extra_household and household_rows is None: + rows[1] = ["00008", "1", "00000000", "6", "1", "2"] + with member.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + ) + writer.writerows(rows) + data = member.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + ( + origin.AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="invented/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256=origin.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[ + 2024 + ].zip_sha256, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + rows=len(rows), + ), + ), + ) + return dict( + parent_path=parent, + household_attachment_path=household, + person_income_attachment_path=output / restoration.CHECKPOINT_FILENAME, + person_member_paths=persons, + household_member_path=member, + ) + + +def test_real_three_cohort_loader_selects_2024_original_anchors(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + frame, receipt = result.frame, result.receipt + assert frame.person.source_year.tolist() == [2024, 2024] + assert frame.person.person_id.tolist() == [105, 106] + assert frame.person.A_AGE.tolist() == [55, 14] + assert frame.person.asec_PTOTVAL.tolist() == [-9999, 0] + assert frame.weights_for("household").kind is WeightKind.DESIGN + assert frame.weights_for("household").values.tolist() == [2552.12] + assert dict(frame.metadata) == {} and frame.mass_log == () + assert json.loads(result.context)["entities"]["person"]["rows"] == 2 + assert receipt["parent_custody"]["cohorts"] == [2022, 2023, 2024] + assert receipt["parent_custody"]["population_cohorts"] == [2024] + assert receipt["households"][0]["HSUP_WGT"] == "000255212" + assert receipt["households"][0]["fraction"] == [63803, 25] + assert receipt["households"][0]["native_key"] == [2024, 7] + assert not receipt["release_eligible"] + assert not receipt["selection"]["inclusion_probabilities_known"] + assert {r["A_LINENO"] for r in receipt["persons"]} == {1, 2} + for entity in frame.schema.group_entities: + assert len(frame.table(entity)) == 1 + # Real parent remains separate, complete and sealed after all child borrows. + state = native._ISSUED[id(result)][2] + assert state.parent.frame.person.source_year.tolist() == [ + 2022, + 2022, + 2023, + 2023, + 2024, + 2024, + ] + assert state.parent.frame.weights_for("household").values.tolist() == [1, 2, 3] + state.parent.validate() + + +def test_exact_selection_candidate_reconstruction_and_value_equal_copy_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + candidate = result.to_bytes() + reconstructed = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),), candidate=candidate + ) + assert reconstructed == result and reconstructed is not result + reconstructed.validate() + for copy in ( + replace(result), + native.AuthenticatedAsec2024NativePopulation(candidate), + ): + assert copy == result + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED"): + copy.validate() + with pytest.raises(native.AsecNativePopulationError, match="CANDIDATE_MISMATCH"): + native.load_authenticated_asec_2024_native_population(**paths, candidate=b"{}") + + +@pytest.mark.parametrize( + "selection", + [ + (), + ((2023, 7),), + ((2024, 7), (2024, 7)), + ((2024, True),), + [(2024, 7)], + ((2024, 8),), + ], +) +def test_non_exact_or_absent_selection_refuses(tmp_path, monkeypatch, selection): + paths = _fixture(tmp_path, monkeypatch) + with pytest.raises(native.AsecNativePopulationError, match="SELECTION"): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=selection + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("H_NUMPER", "1"), + ("H_NUMPER", ""), + ("H_SEQ", "9"), + ("HSUP_WGT", "2552.12"), + ("HSUP_WGT", ""), + ("HSUP_WGT", "-1"), + ("HSUP_WGT", "0"), + ("H_HHTYPE", "2"), + ], +) +def test_source_household_roster_or_unusable_anchor_refuses( + tmp_path, monkeypatch, field, value +): + names = ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + row = ["00007", "1", "255212", "6", "1", "2"] + row[names.index(field)] = value + paths = _fixture(tmp_path, monkeypatch, household_rows=[row]) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_absent_interview_household_not_silently_lost(tmp_path, monkeypatch): + paths = _fixture( + tmp_path, + monkeypatch, + household_rows=[ + ["7", "1", "100", "6", "1", "2"], + ["8", "1", "200", "6", "1", "1"], + ], + ) + with pytest.raises( + native.AsecNativePopulationError, match="HOUSEHOLD_MEMBER_COUNT" + ): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "age", + "unrelated", + "group", + "weight", + "strata", + "strata_flags", + "table_flags", + "attrs", + "columns", + "metadata", + "membership", + ], +) +def test_full_receiving_frame_mutation_refuses(tmp_path, monkeypatch, mutation): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + if mutation == "age": + frame.person.loc[:, "A_AGE"] += 1 + elif mutation == "unrelated": + frame.person.loc[:, "unrelated_raw_observation"] += 1 + elif mutation == "group": + frame.table("household").loc[:, "state_fips"] += 1 + elif mutation == "weight": + vector = frame.weights_for("household").values + vector.setflags(write=True) + vector[0] += 1 + vector.setflags(write=False) + elif mutation == "strata": + frame.strata.iloc[0] = "other" + elif mutation == "strata_flags": + frame.strata.flags.allows_duplicate_labels = False + elif mutation == "table_flags": + frame.table("household").flags.allows_duplicate_labels = False + elif mutation == "attrs": + frame.person.attrs["changed"] = True + elif mutation == "columns": + frame.person.columns.name = "changed" + elif mutation == "metadata": + frame._metadata = {"changed": True} + elif mutation == "membership": + frame.person.iloc[0, frame.person.columns.get_loc("person_tax_unit_id")] = 999 + with pytest.raises(native.AsecNativePopulationError): + result.validate() + + +@pytest.mark.parametrize( + "which", + [ + "parent_path", + "household_attachment_path", + "person_income_attachment_path", + "household_member_path", + "older_person", + ], +) +def test_changed_original_source_refuses_existing_borrow(tmp_path, monkeypatch, which): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + path = ( + paths["person_member_paths"][2022] if which == "older_person" else paths[which] + ) + with path.open("ab") as handle: + handle.write(b"changed") + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + result.validate() + + +def test_capsule_payload_rehash_and_live_owner_changes_refuse(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + state = native._ISSUED[id(result)][2] + payload = state.anchors.payload + b" " + object.__setattr__(state.anchors, "payload", payload) + object.__setattr__( + state.anchors, "_issued_sha256", hashlib.sha256(payload).hexdigest() + ) + with pytest.raises(native.AsecNativePopulationError): + result.validate() + original = restoration._implementation + monkeypatch.setattr(restoration, "_implementation", lambda: original()) + with pytest.raises(native.AsecNativePopulationError, match="PRODUCER_CODE_CHANGED"): + native.load_authenticated_asec_2024_native_population(**paths) + + +@pytest.mark.parametrize("legacy", [[100, 200, 17], [0.001, 10_000, 0.0001]]) +def test_original_anchor_ignores_legacy_pooled_weights(tmp_path, monkeypatch, legacy): + paths = _fixture(tmp_path, monkeypatch, legacy_weights=legacy) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.weights_for("household").values.tolist() == [2552.12] + assert result.receipt["households"][0]["fraction"] == [63803, 25] + assert ( + native._ISSUED[id(result)][2] + .parent.frame.weights_for("household") + .values.tolist() + == legacy + ) + + +def test_mixed_zero_complete_households_and_selected_zero_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch, extra_household=True) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.person.person_id.tolist() == [105, 106, 107, 108] + assert result.frame.person.A_AGE.tolist() == [55, 14, 55, 14] + assert result.frame.weights_for("household").values.tolist() == [2552.12, 0] + assert [r["fraction"] for r in result.receipt["households"]] == [ + [63803, 25], + [0, 1], + ] + selected = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + assert selected.frame.person.person_id.tolist() == [105, 106] + for entity in selected.frame.schema.group_entities: + assert selected.frame.table(entity)[f"{entity}_id"].tolist() == [3] + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 8),) + ) + + +def test_late_mutation_during_final_file_verification_refuses(tmp_path, monkeypatch): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + frame.person.loc[:, "A_AGE"] += 1 + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError, match="FRAME_CHANGED"): + result.validate() + finally: + sys.settrace(None) + assert fired + + +def test_real_loaders_run_without_prepared_population_or_money_execution( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + called = set() + + def trace(stack, event, argument): + if event == "call": + module = stack.f_globals.get("__name__", "") + name = stack.f_code.co_name + assert module not in ( + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.engine", + ) + assert name not in ( + "classify_asec_money", + "prepare_asec_current_money_population", + ) + if module.startswith("microcosm.build.us_runtime."): + called.add(name) + + sys.setprofile(trace) + try: + native.load_authenticated_asec_2024_native_population(**paths) + finally: + sys.setprofile(None) + assert { + "load_authenticated_restored_current_money_source", + "authenticate_asec_coverage", + "load_authenticated_asec_household_weights", + "load_authenticated_asec_household_coverage_fields", + } <= called + + +@pytest.mark.parametrize( + "mutation", ["missing", "duplicate", "substitute", "split", "age"] +) +def test_original_person_mutation_refuses_reconstruction( + tmp_path, monkeypatch, mutation +): + paths = _fixture(tmp_path, monkeypatch) + + def change(rows): + if mutation == "missing": + rows.pop() + elif mutation == "duplicate": + rows[2] = rows[1].copy() + elif mutation == "substitute": + rows[1][0] = "9" * 22 + elif mutation == "split": + rows[1][1] = "8" + elif mutation == "age": + rows[1][3] = "54" + + _helpers()._rewrite(paths["person_member_paths"][2024], change) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_external_state_and_rehashed_population_cannot_be_rewritten( + tmp_path, monkeypatch +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + with pytest.raises(AttributeError): + object.__setattr__(state, "frame_identity", "a" * 64) + changed = json.loads(result.payload) + changed["frame_sha256"] = "a" * 64 + object.__setattr__(result, "payload", native._encode(changed)) + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED_OR_CHANGED"): + result.validate() + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage"]) +def test_equal_unissued_source_evidence_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + evidence = getattr(state, owner) + copy = object.__new__(type(evidence)) + # Copy/deserialization cannot inherit the reader's external issuance seal. + for name in evidence.__slots__: + if name != "__weakref__": + object.__setattr__(copy, name, getattr(evidence, name)) + if owner != "coverage": + assert copy == evidence + verifier = { + "anchors": native.anchor_owner.verify_asec_household_weights_source, + "fields": native.field_owner.verify_asec_household_coverage_fields, + "coverage": lambda value: native.coverage_owner.verify_asec_coverage_parent( + value, state.parent + ), + }[owner] + with pytest.raises(ValueError): + verifier(copy) + result.validate() + + +@pytest.mark.parametrize( + "interface", ["validate", "frame", "context", "receipt", "to_bytes"] +) +def test_late_population_payload_mutation_refuses_every_interface( + tmp_path, monkeypatch, interface +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + object.__setattr__(result, "payload", b'{"release_eligible":true}') + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.settrace(None) + assert fired + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage", "parent"]) +def test_late_attached_evidence_mutation_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + if owner == "parent": + object.__setattr__( + state.parent.source, "evidence", state.parent.source.evidence + b" " + ) + elif owner == "coverage": + object.__setattr__(state.coverage, "_body", state.coverage._body + b"x") + else: + source = getattr(state, owner) + object.__setattr__(source, "payload", source.payload + b" ") + + return_trace = trace + + def tracing(stack, event, argument): + return_trace(stack, event, argument) + return tracing + + sys.settrace(tracing) + try: + with pytest.raises(native.AsecNativePopulationError): + result.to_bytes() + finally: + sys.settrace(None) + assert fired diff --git a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py new file mode 100644 index 000000000..22ee44b3c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py @@ -0,0 +1,944 @@ +"""Invented closed source-parent fixtures only; no genuine preparation.""" + +import _csv +import csv +import hashlib +import importlib.util +import json +import os +import struct +import subprocess +import sys +import textwrap +import traceback +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_person_coverage_source as literal + + +def _helper(name): + spec = importlib.util.spec_from_file_location( + "coverage_invented_" + name, Path(__file__).with_name(name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pin(paths, monkeypatch): + pins = [] + for year, member, archive, _, _, _ in coverage._MEMBER_PINS: + value = paths[year].read_bytes() + with paths[year].open(newline="") as handle: + rows = list(csv.reader(handle)) + pins.append( + (year, member, archive, coverage._sha(value), len(rows) - 1, len(value)) + ) + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + + +def _rewrite(path, change): + with path.open(newline="") as handle: + rows = list(csv.reader(handle)) + change(rows) + with path.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + + +def _changed_parent(parent, attachment, monkeypatch, changes): + for path in (parent, attachment): + loaded = load_frame_checkpoint(path) + for column, values in changes.items(): + loaded.frame.person[column] = values + identity = frame_identity(loaded.frame) + for key in ("identity", "source_construction_identity"): + if key in loaded.metadata: + loaded.metadata[key] = identity.to_payload() + if path == parent: + parent_identity = identity + else: + digest = coverage._sha(parent.read_bytes()) + loaded.metadata["parent_checkpoint_sha256"] = digest + loaded.metadata["household_observations"]["input_checkpoint_sha256"] = ( + digest + ) + loaded.metadata["household_observations"][ + "input_structural_identity_sha256" + ] = parent_identity.sha256 + write_frame_checkpoint(path, loaded.frame, metadata=loaded.metadata) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + coverage._sha(parent.read_bytes()), + coverage._sha(attachment.read_bytes()), + money._SOURCE_PINS[2], + ), + ) + + +def _fixtures( + tmp_path, + monkeypatch, + tokens=("1", "2", "3", "", "-4", "NA"), + *, + native_ids=None, + person_changes=None, + raw_ages=None, +): + parent, attachment, paths = _helper( + "test_us_asec_person_income_source" + ).invented_sources(tmp_path, monkeypatch, missing=False) + if native_ids is not None: + _helper("test_us_asec_demographic_source")._repoint_published_households( + parent, attachment, monkeypatch, native_ids + ) + if person_changes: + _changed_parent(parent, attachment, monkeypatch, person_changes) + source = money.load_authenticated_current_money_source(parent, attachment) + for path in paths.values(): + + def change(rows): + rows[0].append("PRPERTYP") + for row in rows[1:]: + position = int(row[0]) - 1 + row.append(tokens[position % len(tokens)]) + if native_ids is not None: + row[1] = str(native_ids[position]) + if raw_ages is not None: + row[3] = str(raw_ages[position]) + if person_changes and "PERIDNUM" in person_changes: + row[0] = person_changes["PERIDNUM"][position] + + _rewrite(path, change) + _pin(paths, monkeypatch) + return source, paths + + +def _read(source, paths, **kwargs): + return coverage.authenticate_asec_coverage(source, member_paths=paths, **kwargs) + + +def test_closed_complete_source_exact_parent_and_defensive_views(tmp_path, monkeypatch): + source, paths = _fixtures(tmp_path, monkeypatch) + before = money._frame_signature(source.frame) + tables = {e: source.frame.table(e).copy(deep=True) for e in source.frame.entities} + result = _read(source, paths) + coverage.verify_asec_coverage_parent(result, source) + view = result.table() + assert view.PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + assert view.PRPERTYP_state.tolist() == ["observed_code"] * 3 + [ + "blank_unresolved", + "unlabelled_in_range", + "malformed_token", + ] + assert view.PERIDNUM.tolist() == list(source.scope.person_native_keys) + assert view.person_household_id.tolist() == [1, 1, 2, 2, 3, 3] + raw, original_receipt = literal.read_asec_person_coverage_source( + paths, person_roster=source.frame.person + ) + receipt = result.receipt + assert receipt["literal_reader_receipt"] == original_receipt + # Native key cells retain their exact strings; the artifact view declares + # one non-nullable vocabulary independently of the parent's string storage. + pd.testing.assert_frame_equal(view[list(raw)], raw.astype({"PERIDNUM": "string"})) + assert receipt["source_authenticated"] is True + assert receipt["named_parent_binding_authenticated"] is True + for name in ( + "release_eligible", + "domain_authority", + "period_harmonized", + "cross_survey_coverage_equivalence_established", + "original_design_weight_semantics_established", + "archive_bytes_read", + ): + assert receipt[name] is False + assert receipt["literal_reader_receipt"]["source_authenticated"] is False + assert ( + receipt["literal_reader_receipt"]["population_binding_authenticated"] is False + ) + assert receipt["age_relation"]["compared_rows"] == 6 + assert ( + receipt["age_relation"]["relation"] == "original_csv_A_AGE_equals_parent_A_AGE" + ) + assert receipt["household_partitions"]["households"] == 3 + assert [p["rows"] for p in receipt["sources"]] == [2, 2, 2] + assert str(tmp_path) not in json.dumps(receipt) + assert "0000000000000000000001" not in json.dumps(receipt) + pin = result.content_sha256 + view.loc[0, "PRPERTYP"] = "changed" + view.index = pd.Index([99, 1, 2, 3, 4, 5]) + receipt["parent"]["frame_sha256"] = "changed" + assert result.content_sha256 == pin and result.table().PRPERTYP.iloc[0] == "1" + assert money._frame_signature(source.frame) == before + for entity, expected in tables.items(): + pd.testing.assert_frame_equal( + source.frame.table(entity), expected, check_exact=True + ) + with pytest.raises(FrozenInstanceError): + result._body = b"changed" + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + coverage.AuthenticatedAsecCoverage(result._header, result._body) + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + replace(result) + + +@pytest.mark.parametrize( + "token,state", + [ + ("", "blank_unresolved"), + ("\0", "malformed_token"), + ("1\0x", "malformed_token"), + (" 1", "malformed_token"), + ("1.0", "malformed_token"), + ("NA", "malformed_token"), + ('a"b', "malformed_token"), + ("a,b\r\n\t😀", "malformed_token"), + ("-4", "unlabelled_in_range"), + ("4", "out_of_range"), + ("3", "observed_code"), + ], +) +def test_literal_utf8_roundtrip_without_coercion(tmp_path, monkeypatch, token, state): + source, paths = _fixtures(tmp_path, monkeypatch, (token,)) + result = _read(source, paths) + assert result.table().PRPERTYP.tolist() == [token] * 6 + assert result.table().PRPERTYP_state.tolist() == [state] * 6 + assert not result.table().PRPERTYP.isna().any() + + +@pytest.mark.parametrize("kind", ["frame", "receipt", "hash"]) +def test_no_plain_parent_can_issue_authority(tmp_path, monkeypatch, kind): + source, paths = _fixtures(tmp_path, monkeypatch) + candidate = { + "frame": source.frame, + "receipt": json.loads(source.source.identity), + "hash": money._frame_signature(source.frame), + }[kind] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="AUTHENTICATED_PARENT" + ): + _read(candidate, paths) + with pytest.raises(TypeError): + coverage.authenticate_asec_coverage(source, member_paths=paths, member_pins=()) + + +@pytest.mark.parametrize( + "column,value", + [ + ("A_AGE", "82"), + ("A_AGE", ""), + ("A_AGE", "55.0"), + ("A_AGE", "\0"), + ("PH_SEQ", "8"), + ("A_LINENO", "8"), + ("PERIDNUM", "9" * 22), + ], +) +def test_authenticated_raw_coordinate_conflicts_refuse( + tmp_path, monkeypatch, column, value +): + source, paths = _fixtures(tmp_path, monkeypatch) + _rewrite( + paths[2022], lambda rows: rows[1].__setitem__(rows[0].index(column), value) + ) + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + + +@pytest.mark.parametrize( + "column", + [ + "A_AGE", + "PERIDNUM", + "person_household_id", + "unrelated_raw_observation", + "WSAL_VAL", + ], +) +def test_same_ids_mutated_parent_cannot_reuse_binding(tmp_path, monkeypatch, column): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + source.frame.person.loc[:, column] = "9" * 22 if column == "PERIDNUM" else 1 + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_REFUSAL" + ): + coverage.verify_asec_coverage_parent(result, source) + + +def test_distinct_authenticated_parent_with_same_ids_cannot_reuse_artifact( + tmp_path, monkeypatch +): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + source, paths = _fixtures(first, monkeypatch) + result = _read(source, paths) + other, other_paths = _fixtures( + second, + monkeypatch, + person_changes={"unrelated_raw_observation": np.arange(6, dtype=np.int64) + 1}, + ) + assert other.scope.person_ids == source.scope.person_ids + # Restore the producer's fixture pins so this specifically checks parent + # identity rather than the independent source-pin implementation guard. + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_CHANGED" + ): + coverage.verify_asec_coverage_parent(result, other) + assert set(other_paths) == {2022, 2023, 2024} + + +def test_merged_native_households_refuse_even_with_matching_per_row_coordinates( + tmp_path, monkeypatch +): + # The current-money parent already refuses this shape, before coverage + # issuance. Keep that layering and also test the wrapper's own fence. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 8], + "person_household_id": [1, 1], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize("parent_age,raw_age", [(80, 80), (82, 80), (0, "")]) +def test_raw_age_identity_never_substitutes_model_age( + tmp_path, monkeypatch, parent_age, raw_age +): + source, paths = _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([parent_age, 14] * 3, dtype=np.int64), + }, + raw_ages=[raw_age, 14] * 3, + ) + before = money._frame_signature(source.frame) + if parent_age == raw_age: + result = _read(source, paths) + assert result.table().A_AGE.tolist() == [80, 14] * 3 + assert result.receipt["age_relation"]["model_age_used"] is False + else: + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + assert money._frame_signature(source.frame) == before + + +def test_parent_authority_refuses_model_age_before_source_issuance( + tmp_path, monkeypatch +): + # Canonical model age is already forbidden by the raw parent boundary. + # Keep that refusal; do not loosen the parent merely to exercise coverage. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([80, 14] * 3, dtype=np.int64), + "age": [82.0, 14.0] * 3, + }, + raw_ages=[80, 14] * 3, + ) + + +def test_same_native_person_keys_across_cohorts_remain_distinct(tmp_path, monkeypatch): + keys = [str(i).zfill(22) for i in (1, 2)] * 3 + source, paths = _fixtures(tmp_path, monkeypatch, person_changes={"PERIDNUM": keys}) + result = _read(source, paths) + assert result.table().PERIDNUM.tolist() == keys + assert result.table().PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + + +def test_partition_bijection_refuses_splits_and_cross_cohort_merges(): + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 7], + "person_household_id": [1, 2], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_SPLIT" + ): + coverage._partitions(person) + person.source_year = [2022, 2023] + person.person_household_id = [1, 1] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize( + "kind", + [ + "missing_cohort", + "duplicate_cohort", + "duplicate_row", + "missing_row", + "swapped_cohort", + ], +) +def test_complete_cohort_authority_refuses_incomplete_or_duplicate_inputs( + tmp_path, monkeypatch, kind +): + source, paths = _fixtures(tmp_path, monkeypatch) + if kind == "missing_cohort": + paths.pop(2023) + elif kind == "duplicate_cohort": + pins = coverage._MEMBER_PINS + monkeypatch.setattr(coverage, "_MEMBER_PINS", (pins[0], pins[0], pins[2])) + elif kind == "swapped_cohort": + paths[2022], paths[2023] = paths[2023], paths[2022] + else: + _rewrite( + paths[2022], + lambda rows: ( + rows.__setitem__(1, rows[2].copy()) + if kind == "duplicate_row" + else rows.pop() + ), + ) + _pin(paths, monkeypatch) + with pytest.raises(coverage.AsecCoverageAuthenticationError): + _read(source, paths) + + +def test_candidate_self_rehash_is_not_authority_and_reconstruction_runs_first( + tmp_path, monkeypatch +): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + candidate = tmp_path / "candidate.bin" + candidate.write_bytes(result.to_bytes()) + assert ( + _read(source, paths, candidate_path=candidate).to_bytes() == result.to_bytes() + ) + body = result._body.replace(b"observed_code", b"invented_code", 1) + header = json.loads(result._header) + header["body_sha256"] = coverage._sha(body) + header = coverage._json(header) + payload = coverage.MAGIC + struct.pack(" 1 + for changed_slice in ( + replace(first, columns=first.columns[:-1]), + replace(first, columns=first.columns[::-1]), + replace(first, columns=(*first.columns, "__extra_input__")), + # A declared column lets Node construction succeed; this direct + # contract check must reject non-ALL rows before any mask evaluation. + replace(first, rows=first.columns[0]), + ): + bad_node = replace(nodes[0], inputs=(changed_slice, *nodes[0].inputs[1:])) + with pytest.raises(ValueError, match="^SURVEY_HOST_INPUTS$"): + bridge._current_context_frame( + replace(context, node=bad_node), expanded.frame + ) + for field, values, reason in ( + ("tables", context.tables, "SURVEY_HOST_TABLES"), + ("weights", context.weights, "SURVEY_HOST_WEIGHTS"), + ): + missing = dict(values) + del missing[next(iter(missing))] + extra = {**values, "__extra_entity__": next(iter(values.values()))} + for changed_values in (missing, extra): + with pytest.raises(ValueError, match="^" + reason + "$"): + bridge._current_context_frame( + replace(context, **{field: changed_values}), expanded.frame + ) + + person = context.tables[expanded.frame.schema.person_entity] + changed_tables = dict(context.tables) + changed_tables[expanded.frame.schema.person_entity] = person.iloc[:, ::-1] + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_tables[expanded.frame.schema.person_entity] = person.rename_axis( + "__changed_columns__", axis="columns" + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_person = person.copy(deep=True) + changed_person.iloc[0, changed_person.columns.get_loc("age")] += 1 + changed_tables[expanded.frame.schema.person_entity] = changed_person + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_NATIVE_BITS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + + # Positive and negative zero are both valid weights; only their exact bits + # differ. This must reach the replay comparison rather than Frame refusal. + entity = next(e for e, w in context.weights.items() if np.any(w.values == 0)) + value = context.weights[entity] + changed_values = value.values.copy() + zero = int(np.flatnonzero(changed_values == 0)[0]) + changed_values[zero] = 0.0 if np.signbit(changed_values[zero]) else -0.0 + changed_weights = {**context.weights, entity: Weights(changed_values, value.kind)} + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_WEIGHT_BYTES$"): + bridge._current_context_frame( + replace(context, weights=changed_weights), expanded.frame + ) + with pytest.raises(ValueError, match="^SURVEY_HOST_STRATA_NAME$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename("__changed_name__")), + expanded.frame, + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename_axis("__changed_index__")), + expanded.frame, + ) + assert owner._frame_identity(expanded.frame) == before + + graph = replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + compiled = compile_graph(graph) + assert len(compiled.graph.nodes) == 6 + live.kernels.register( + bridge.CurrentSurveyHostProjectionKernel(preparation, allocated, expanded) + ) + live.kernels.register(bridge.CurrentSurveyMatrixKernel()) + outputs = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population: observed.__setitem__( + name, population + ), + ) + actual = [] + for node_id, name, type_, expected, kernel in ( + ( + bridge.CURRENT_PROJECTION_NODE, + "projection", + bridge.CURRENT_PROJECTION_TYPE, + projection, + bridge.CurrentSurveyHostProjectionKernel, + ), + ( + bridge.CURRENT_MATRIX_NODE, + "matrix", + bridge.model_input.RECIPIENT_MATRIX_TYPE, + matrix, + bridge.CurrentSurveyMatrixKernel, + ), + ): + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=expected, + capabilities=kernel.capabilities, + ) + receipt = manifest.node(node_id) + assert receipt.opaque_artifacts[name] == opaque_artifact_key( + receipt.key, name + ) + actual.append(live.store.load_bytes(receipt.opaque_artifacts[name])) + replay.same_replayed_population(expanded, observed[node_id]) + checked = bridge.verify_materialized_current_survey_host( + preparation, + allocated, + expanded, + population=observed[bridge.CURRENT_MATRIX_NODE], + projection=actual[0], + matrix=actual[1], + ) + assert checked == evidence + outputs.append(tuple(actual)) + assert outputs[0] == outputs[1] == (projection, matrix) + assert owner._frame_identity(expanded.frame) == before + wrong_pins = copy.deepcopy(pins) + wrong_pins["preparation"]["producer_key"] = "f" * 64 + bad_projection_node = replace( + nodes[0], + params={ + **dict(nodes[0].params), + "host_edges": codec.encode_json(wrong_pins).decode(), + }, + ) + bad_graph = replace( + graph, nodes=(*live.compiled.graph.nodes, bad_projection_node, nodes[1]) + ) + with pytest.raises(NodeRejected, match="SURVEY_HOST_EDGE_PIN"): + run_graph( + compile_graph(bad_graph), + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume="auto", + ) + + # A well-formed amount mutation can pass the pure value parser; the live + # materialized verifier must reject it against the retained current owner. + changed = copy.deepcopy(document) + changed["asec"]["rows"][0][3] = np.asarray([60001.0], dtype=" TargetRegistry: + """Fictional values and identities; these are not Census observations.""" + + return TargetRegistry( + [ + TargetSpec( + name=name, + entity="household", + measure=measure, + value=value, + period="2024", + source="Invented fixture; no published Census value", + family="acs.S0101", + metadata={ + "table": "S0101", + "reference_sha256": "a" * 64, + "geography": "0100000US", + "universe": "population", + "role": "calibration", + "evidence_scope": "invented", + }, + ) + for name, (measure, value) in FIXTURE_TARGETS.items() + ], + country="us", + ) + + +def fixture_frame(*, counted=False, kind=WeightKind.IMPORTANCE, scale=1.0) -> Frame: + person = pd.DataFrame( + { + "person_id": np.array([row[0] for row in PEOPLE], dtype="int64"), + "person_household_id": np.array([row[1] for row in PEOPLE], dtype="int64"), + "age": np.array([float(row[2]) for row in PEOPLE]), + } + ) + household = pd.DataFrame( + { + "household_id": np.array(HOUSEHOLD_IDS, dtype="int64"), + "is_group_quarters": np.array( + [identity == GROUP_QUARTERS_HOUSEHOLD for identity in HOUSEHOLD_IDS] + ), + } + ) + if counted: + for column in COLUMNS: + household[column] = np.array(EXPECTED[column], dtype="int64") + return Frame( + {"person": person, "household": household}, + SCHEMA, + {"household": Weights(np.array([1.0, 0.5, 1.5, 0.5, 1.0]) * scale, kind)}, + ) + + +def count_context(frame: Frame, node: Node) -> KernelContext: + return KernelContext( + node=node, + tables={ + "person": frame.table("person").loc[ + :, ["person_id", "person_household_id", "age"] + ], + "household": frame.table("household").loc[:, ["household_id"]], + }, + weights={"household": frame.weights_for("household")}, + strata=frame.strata, + params=node.params, + rng=np.random.default_rng(0), + ) + + +def counted(result: KernelResult) -> pd.DataFrame: + return pd.DataFrame( + {column: result.columns[("household", column)] for column in COLUMNS} + ) + + +# -------------------------------------------------------------------------- +# The declaration +# -------------------------------------------------------------------------- + + +def test_the_node_declares_exactly_its_inputs_and_owns_eighteen_int64_columns(): + node = national_age_count_node(population="assembled") + assert node.structural is StructuralDelta.NONE + assert node.population == "assembled" + assert node.base is None + assert node.sources == () + assert node.weights is None + assert node.artifact_inputs == () + # ``age`` is an owned cell and travels in a Slice; ids and memberships are + # structural, so they are declared normatively in the parameters instead. + assert {(slice_.entity, slice_.columns) for slice_ in node.inputs} == { + ("person", ("age",)) + } + assert all(slice_.rows == "all" for slice_ in node.inputs) + assert node.params["person_columns"] == ("age", "person_household_id") + assert node.params["household_columns"] == ("household_id",) + assert tuple(owned.column for owned in node.outputs) == COLUMNS + assert {owned.dtype for owned in node.outputs} == {"int64"} + assert {owned.entity for owned in node.outputs} == {"household"} + assert node.params["age_convention"] == SUPPORTED_AGE_CONVENTION + assert node.params["schema_version"] == AGE_COUNT_SCHEMA_VERSION + # No source channel, spine column, weight or model output is declared. + declared = { + *(column for slice_ in node.inputs for column in slice_.columns), + *node.params["person_columns"], + *node.params["household_columns"], + } + assert not any( + marker in column + for column in declared + for marker in ("spine", "support_channel", "weight") + ) + + +def test_an_altered_band_schema_changes_the_node_key(): + baseline = national_age_count_node(population="assembled") + moved = national_age_count_node( + population="assembled", + bands=( + replace(NATIONAL_AGE_ACTIVATION.bands[0], column="people_age_0_to_4"), + *NATIONAL_AGE_ACTIVATION.bands[1:], + ), + ) + assert moved.normative() != baseline.normative() + assert moved.params["bands"] != baseline.params["bands"] + assert tuple(owned.column for owned in moved.outputs) != COLUMNS + + +def test_the_kernel_refuses_a_node_that_differs_from_its_declaration(): + frame = fixture_frame() + node = national_age_count_node(population="assembled") + for tampered in ( + replace(node, inputs=(Slice("person", ("age", "is_group_quarters")),)), + replace(node, inputs=(*node.inputs, Slice("household", ("household_id",)))), + replace(node, outputs=node.outputs[:-1]), + replace(node, mass="free"), + replace(node, kernel="us.something_else@1"), + ): + with pytest.raises(ValueError, match="complete declaration"): + NationalAgeCountKernel().run(count_context(frame, tampered)) + convention = replace( + node, params={**node.params, "age_convention": "aged_to_target_period"} + ) + with pytest.raises(ValueError, match="implements only"): + NationalAgeCountKernel().run(count_context(frame, convention)) + version = replace(node, params={**node.params, "schema_version": 2}) + with pytest.raises(ValueError, match="schema version"): + NationalAgeCountKernel().run(count_context(frame, version)) + extra = replace(node, params={**node.params, "seed": 1}) + with pytest.raises(ValueError, match="exactly its band schema"): + NationalAgeCountKernel().run(count_context(frame, extra)) + widened = replace( + node, + params={ + **node.params, + "person_columns": ("age", "person_household_id", "is_group_quarters"), + }, + ) + with pytest.raises(ValueError, match="reads exactly"): + NationalAgeCountKernel().run(count_context(frame, widened)) + + +def test_the_kernel_refuses_a_household_view_beyond_its_declared_id(): + frame = fixture_frame() + node = national_age_count_node(population="assembled") + leaked = replace( + count_context(frame, node), + tables={ + "person": frame.table("person").loc[ + :, ["person_id", "person_household_id", "age"] + ], + "household": frame.table("household"), + }, + ) + with pytest.raises(ValueError, match="carry exactly"): + NationalAgeCountKernel().run(leaked) + + +@pytest.mark.parametrize( + "bands", + [ + # Overlapping bands would count a person twice. + json.dumps([["a", 0, 4], ["b", 3, 9], ["c", 10, None]]), + # A gap would drop people silently. + json.dumps([["a", 0, 4], ["b", 7, None]]), + # No open top band leaves the oldest uncovered. + json.dumps([["a", 0, 4], ["b", 5, 9]]), + # Two open bands cannot be disjoint. + json.dumps([["a", 0, None], ["b", 5, None]]), + # Duplicate columns would collide. + json.dumps([["a", 0, 4], ["a", 5, None]]), + # Not starting at zero would drop the youngest. + json.dumps([["a", 1, 4], ["b", 5, None]]), + "[]", + "not json", + ], +) +def test_an_incoherent_band_schema_is_refused(bands): + frame = fixture_frame() + node = national_age_count_node(population="assembled") + with pytest.raises(ValueError): + NationalAgeCountKernel().run( + count_context(frame, replace(node, params={**node.params, "bands": bands})) + ) + + +# -------------------------------------------------------------------------- +# What the operator is statically incapable of reading +# -------------------------------------------------------------------------- + + +OPERATOR_MODULE = Path(graph_national_age_counts.__file__) + + +def _operator_tree() -> ast.Module: + return ast.parse(OPERATOR_MODULE.read_text()) + + +def test_the_operator_never_touches_weights_sources_artifacts_or_randomness(): + """A static read of the kernel's own source, not a runtime sample.""" + + forbidden = {"weights", "sources", "artifacts", "rng", "strata"} + reads = { + node.attr + for node in ast.walk(_operator_tree()) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "context" + and node.attr in forbidden + } + assert not reads, f"the age-count operator reads context.{sorted(reads)}" + + +def test_the_operator_opens_no_file_and_imports_no_rules_engine(): + tree = _operator_tree() + imported = { + name.split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for name in (alias.name for alias in node.names) + } | { + (node.module or "").split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level == 0 + } + assert not imported & {"policyengine_us", "policyengine", "microdf", "urllib"} + called = { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, (ast.Name, ast.Attribute)) + } + assert not called & { + "open", + "read_bytes", + "read_text", + "read_csv", + "read_hdf", + "load", + "calculate", + "simulate", + } + + +# -------------------------------------------------------------------------- +# Counting +# -------------------------------------------------------------------------- + + +def test_every_person_is_counted_once_into_the_right_band(): + frame = fixture_frame() + node = national_age_count_node(population="assembled") + result = NationalAgeCountKernel().run(count_context(frame, node)) + table = counted(result) + assert list(table.index) == list(HOUSEHOLD_IDS) + assert table.index.name == "household_id" + assert set(table.dtypes) == {np.dtype("int64")} + for column in COLUMNS: + assert tuple(table[column]) == EXPECTED[column], column + assert int(table.to_numpy().sum()) == len(PEOPLE) + assert result.receipt["people_counted"] == len(PEOPLE) + assert result.receipt["households"] == len(HOUSEHOLD_IDS) + assert result.receipt["consumes_weights"] is False + assert result.receipt["release_eligible"] is False + assert result.weights is None + + +@pytest.mark.parametrize( + ("age", "column"), + [ + (0, "people_age_0_4"), + (4, "people_age_0_4"), + (5, "people_age_5_9"), + (9, "people_age_5_9"), + (79, "people_age_75_79"), + (80, "people_age_80_84"), + (84, "people_age_80_84"), + (85, "people_age_85_plus"), + (99, "people_age_85_plus"), + (115, "people_age_85_plus"), + (120, "people_age_85_plus"), + ], +) +def test_band_boundaries_and_topcoded_ages(age, column): + """One person, one band: the boundaries land where the publisher puts them.""" + + frame = Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([3], dtype="int64"), + "age": np.array([float(age)]), + } + ), + "household": pd.DataFrame({"household_id": np.array([3], dtype="int64")}), + }, + SCHEMA, + {"household": Weights(np.array([1.0]), WeightKind.IMPORTANCE)}, + ) + node = national_age_count_node(population="assembled") + table = counted(NationalAgeCountKernel().run(count_context(frame, node))) + assert int(table[column].iloc[0]) == 1 + assert int(table.to_numpy().sum()) == 1 + + +def test_rows_align_by_id_not_position(): + """Neither axis is assumed sorted, sequential, or shared between tables.""" + + household = pd.DataFrame({"household_id": np.array([90, 7, 22], dtype="int64")}) + person = pd.DataFrame( + { + "person_id": np.array([5, 2, 9, 4], dtype="int64"), + "person_household_id": np.array([22, 90, 22, 7], dtype="int64"), + "age": np.array([0.0, 85.0, 6.0, 40.0]), + } + ) + node = national_age_count_node(population="assembled") + context = KernelContext( + node=node, + tables={"person": person, "household": household}, + weights={ + "household": Weights(np.array([1.0, 1.0, 1.0]), WeightKind.IMPORTANCE) + }, + strata=pd.Series(["x"] * 4), + params=node.params, + rng=np.random.default_rng(0), + ) + table = counted(NationalAgeCountKernel().run(context)) + assert list(table.index) == [90, 7, 22] + assert list(table["people_age_0_4"]) == [0, 0, 1] + assert list(table["people_age_5_9"]) == [0, 0, 1] + assert list(table["people_age_40_44"]) == [0, 1, 0] + assert list(table["people_age_85_plus"]) == [1, 0, 0] + # Reordering the person rows cannot change any count. + shuffled = person.iloc[[3, 1, 0, 2]].reset_index(drop=True) + reshuffled = counted( + NationalAgeCountKernel().run( + replace(context, tables={"person": shuffled, "household": household}) + ) + ) + pd.testing.assert_frame_equal(table, reshuffled) + + +def test_group_quarters_people_are_counted_with_their_membership_intact(): + frame = fixture_frame() + node = national_age_count_node(population="assembled") + # The operator cannot even see the group-quarters flag: it is neither a + # declared input nor a column of the household view it is handed. + assert "is_group_quarters" not in { + *(column for slice_ in node.inputs for column in slice_.columns), + *node.params["person_columns"], + *node.params["household_columns"], + } + table = counted(NationalAgeCountKernel().run(count_context(frame, node))) + quarters = HOUSEHOLD_IDS.index(GROUP_QUARTERS_HOUSEHOLD) + assert int(table.loc[GROUP_QUARTERS_HOUSEHOLD, "people_age_85_plus"]) == 1 + assert int(table.loc[GROUP_QUARTERS_HOUSEHOLD].sum()) == sum( + EXPECTED[column][quarters] for column in COLUMNS + ) + assert int(table.to_numpy().sum()) == len(PEOPLE) + + +@pytest.mark.parametrize("age", [121, 200, 999]) +def test_ages_outside_the_declared_envelope_refuse(age): + node = national_age_count_node(population="assembled") + context = count_context(fixture_frame(), node) + person = context.tables["person"].copy() + person.loc[person.index[0], "age"] = age + with pytest.raises(ValueError, match="declared maximum"): + NationalAgeCountKernel().run( + replace(context, tables={**context.tables, "person": person}) + ) + + +def test_counts_do_not_read_weights(): + node = national_age_count_node(population="assembled") + baseline = counted( + NationalAgeCountKernel().run(count_context(fixture_frame(), node)) + ) + for kind in (WeightKind.DESIGN, WeightKind.IMPORTANCE, WeightKind.CALIBRATED): + for scale in (1.0, 137.0): + other = counted( + NationalAgeCountKernel().run( + count_context(fixture_frame(kind=kind, scale=scale), node) + ) + ) + pd.testing.assert_frame_equal(baseline, other) + + +@pytest.mark.parametrize( + ("age", "match"), + [ + (np.nan, "unknown"), + (-1.0, "negative"), + (4.5, "whole number"), + (np.inf, "finite"), + ], +) +def test_an_unusable_age_refuses_rather_than_dropping_the_person(age, match): + frame = fixture_frame() + person = frame.table("person").copy() + person.loc[0, "age"] = age + node = national_age_count_node(population="assembled") + context = replace( + count_context(frame, node), + tables={ + "person": person.loc[:, ["person_id", "person_household_id", "age"]], + "household": frame.table("household").loc[:, ["household_id"]], + }, + ) + with pytest.raises(ValueError, match=match): + NationalAgeCountKernel().run(context) + + +@pytest.mark.parametrize("dtype", ["string", "bool", "complex128"]) +def test_a_non_numeric_age_is_not_coerced(dtype): + frame = fixture_frame() + person = frame.table("person").copy() + person["age"] = person["age"].astype(dtype) + node = national_age_count_node(population="assembled") + context = replace( + count_context(frame, node), + tables={ + "person": person.loc[:, ["person_id", "person_household_id", "age"]], + "household": frame.table("household").loc[:, ["household_id"]], + }, + ) + with pytest.raises(ValueError, match="real numeric"): + NationalAgeCountKernel().run(context) + + +def test_unresolved_membership_refuses_rather_than_dropping_the_person(): + frame = fixture_frame() + person = frame.table("person").copy() + person.loc[0, "person_household_id"] = 999 + node = national_age_count_node(population="assembled") + context = replace( + count_context(frame, node), + tables={ + "person": person.loc[:, ["person_id", "person_household_id", "age"]], + "household": frame.table("household").loc[:, ["household_id"]], + }, + ) + with pytest.raises(ValueError, match="must resolve to a household row"): + NationalAgeCountKernel().run(context) + + +def test_a_duplicated_household_id_is_refused(): + node = national_age_count_node(population="assembled") + household = pd.DataFrame({"household_id": np.array([3, 3], dtype="int64")}) + person = pd.DataFrame( + { + "person_id": np.array([1, 2], dtype="int64"), + "person_household_id": np.array([3, 3], dtype="int64"), + "age": np.array([1.0, 2.0]), + } + ) + context = KernelContext( + node=node, + tables={"person": person, "household": household}, + weights={"household": Weights(np.array([1.0, 1.0]), WeightKind.IMPORTANCE)}, + strata=pd.Series(["x", "x"]), + params=node.params, + rng=np.random.default_rng(0), + ) + with pytest.raises(ValueError, match="unique"): + NationalAgeCountKernel().run(context) + + +def test_a_household_with_no_people_is_refused_explicitly(): + """The population contract requires every group row to have a member.""" + + node = national_age_count_node(population="assembled") + household = pd.DataFrame({"household_id": np.array([3, 8], dtype="int64")}) + person = pd.DataFrame( + { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([3], dtype="int64"), + "age": np.array([1.0]), + } + ) + context = KernelContext( + node=node, + tables={"person": person, "household": household}, + weights={"household": Weights(np.array([1.0, 1.0]), WeightKind.IMPORTANCE)}, + strata=pd.Series(["x"]), + params=node.params, + rng=np.random.default_rng(0), + ) + with pytest.raises(ValueError, match="at least one person"): + NationalAgeCountKernel().run(context) + + +@pytest.mark.parametrize("bad", [np.nan, 2.5, 3 + 2j, 1e20, np.uint64(2**63 + 3)]) +def test_a_broken_membership_column_is_refused(bad): + node = national_age_count_node(population="assembled") + household = pd.DataFrame({"household_id": np.array([3], dtype="int64")}) + person = pd.DataFrame( + { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([bad]), + "age": np.array([1.0]), + } + ) + context = KernelContext( + node=node, + tables={"person": person, "household": household}, + weights={"household": Weights(np.array([1.0]), WeightKind.IMPORTANCE)}, + strata=pd.Series(["x"]), + params=node.params, + rng=np.random.default_rng(0), + ) + with pytest.raises(ValueError, match="household membership"): + NationalAgeCountKernel().run(context) + + +@pytest.mark.parametrize("household_id", [-(2**63), 2**63 - 1]) +def test_int64_boundary_ids_keep_their_exact_household_identity(household_id): + node = national_age_count_node(population="assembled") + household = pd.DataFrame({"household_id": np.array([household_id], dtype="int64")}) + person = pd.DataFrame( + { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([household_id], dtype="int64"), + "age": np.array([1.0]), + } + ) + context = KernelContext( + node=node, + tables={"person": person, "household": household}, + weights={"household": Weights(np.array([1.0]), WeightKind.IMPORTANCE)}, + strata=pd.Series(["x"]), + params=node.params, + rng=np.random.default_rng(0), + ) + result = NationalAgeCountKernel().run(context) + count = result.columns[("household", "people_age_0_4")] + assert count.index.tolist() == [household_id] + assert count.tolist() == [1] + + +# -------------------------------------------------------------------------- +# Composition with the unchanged calibration node +# -------------------------------------------------------------------------- + + +def test_the_composition_wires_counts_into_the_existing_calibration_node(): + count, calibration = national_age_calibration_nodes( + base="assembled", + registry=fixture_registry(), + epochs=EPOCHS, + learning_rate=LEARNING_RATE, + max_weight_ratio=CAP, + max_initial_weight_ratio=CAP, + ) + assert count.id == "national.age_counts" + assert calibration.id == "national.demographic_calibration" + assert calibration.base == "assembled" + assert count.population == "assembled" + owned = tuple(item.column for item in count.outputs) + assert calibration.inputs[0].columns == owned + assert calibration.structural is StructuralDelta.REWEIGHT + assert calibration.weights == WeightTransition( + "household", "calibrated", mass="free" + ) + # No frame-context artifact is produced or relabelled by this composition: + # the diagnostics edge is the only artifact either node emits. + assert count.artifact_outputs == () + assert tuple(item.name for item in calibration.artifact_outputs) == ("diagnostics",) + + +def test_a_registry_that_disagrees_with_the_bands_is_refused(): + """A partial or reordered registry is refused, not half-wired.""" + + full = fixture_registry() + for broken in ( + TargetRegistry(full.specs[:4], country="us"), + TargetRegistry((full.specs[1], full.specs[0], *full.specs[2:]), country="us"), + ): + with pytest.raises(ValueError, match="exactly the declared band columns"): + national_age_calibration_nodes( + base="assembled", + registry=broken, + epochs=EPOCHS, + learning_rate=LEARNING_RATE, + max_weight_ratio=CAP, + max_initial_weight_ratio=CAP, + ) + + +# -------------------------------------------------------------------------- +# The real graph, and a fresh-process warm replay +# -------------------------------------------------------------------------- + + +class FixtureSource(KernelBase): + ref = "test.age_count_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.CREATE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context): + return KernelResult( + frame=load_source("frame-store", context.sources["fixture"]) + ) + + def implementation_hash(self): + # Pytest and the fresh child import this fixture under different module + # names. Bind its same actual file bytes, not that incidental name. + return hashlib.sha256( + Path(__file__).read_bytes() + source_hash(load_source).encode() + ).hexdigest() + + +class FixtureImportance(FixtureSource): + ref = "test.age_count_importance@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult( + weights=Weights( + context.weights["household"].values * context.params["factor"], + WeightKind.IMPORTANCE, + ) + ) + + +def graph_fixture(output, warm=False): + """Cold and warm run the same on-disk fixture and real production kernels.""" + + from microcosm.build.transfer_example import _source_frame_path + + output = Path(output) + store = ContentStore(output / "store") + if not warm: + frame = fixture_frame(kind=WeightKind.DESIGN, scale=2.0) + path = _source_frame_path(store, frame) + (output / "source-path.txt").write_text(str(path)) + path = Path((output / "source-path.txt").read_text()) + count, calibration = national_age_calibration_nodes( + base="assembled", + registry=fixture_registry(), + epochs=EPOCHS, + learning_rate=LEARNING_RATE, + max_weight_ratio=CAP, + max_initial_weight_ratio=CAP, + ) + graph = Graph( + "us", + (SourceRef("fixture", "frame-store"),), + ( + Node( + "source", + FixtureSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "age", "float64"), + Owned("household", "is_group_quarters", "bool"), + ), + ), + Node( + "assembled", + FixtureImportance.ref, + base="source", + inputs=(Slice("household", ("is_group_quarters",)),), + structural=StructuralDelta.REWEIGHT, + mass="free", + weights=WeightTransition("household", "importance", mass="free"), + params={"factor": 0.5}, + ), + count, + calibration, + ), + ) + registry = KernelRegistry() + for kernel in ( + FixtureSource(), + FixtureImportance(), + NationalAgeCountKernel(), + DemographicCalibrationKernel(), + ): + if warm: + + def forbidden(context): + raise AssertionError("A warm replay must execute no kernels") + + kernel.run = forbidden + registry.register(kernel) + result = run_graph( + compile_graph(graph), sources={"fixture": path}, store=store, kernels=registry + ) + population = result.population("national.demographic_calibration") + node = result.node("national.demographic_calibration") + payload = store.load_bytes(node.opaque_artifacts["diagnostics"]) + counts = population.table("household").loc[:, list(COLUMNS)] + return { + "key": result.key, + "diagnostics_sha256": hashlib.sha256(payload).hexdigest(), + "weight_bytes_sha256": hashlib.sha256( + population.weights_for("household").values.tobytes() + ).hexdigest(), + "counts": counts.to_dict(orient="list"), + "household_ids": population.table("household")["household_id"].tolist(), + "count_dtypes": sorted({str(dtype) for dtype in counts.dtypes}), + "weight_kind": population.weights_for("household").kind.value, + "diagnostics": json.loads(payload), + "hits": [n.hit for n in result.nodes.values()], + } + + +def test_the_real_graph_counts_and_calibrates_the_invented_population(tmp_path): + cold = graph_fixture(tmp_path) + assert cold["hits"] == [False, False, False, False] + assert cold["household_ids"] == list(HOUSEHOLD_IDS) + assert cold["count_dtypes"] == ["int64"] + for column, expected in EXPECTED.items(): + assert cold["counts"][column] == list(expected) + assert sum(sum(values) for values in cold["counts"].values()) == len(PEOPLE) + assert cold["weight_kind"] == "calibrated" + + # The graph result equals the direct solver over the same counted frame. + direct = calibrate( + fixture_frame(counted=True), + fixture_registry().to_target_set(), + weight_entity="household", + method="adam", + epochs=EPOCHS, + learning_rate=LEARNING_RATE, + max_weight_ratio=CAP, + mass="free", + seed=0, + ) + assert ( + cold["weight_bytes_sha256"] + == hashlib.sha256(direct.weights.tobytes()).hexdigest() + ) + assert cold["diagnostics"] == diagnostics_payload( + direct, + target_registry=fixture_registry(), + build={ + "solver_weight_anchor": "incoming_importance", + "solver_max_weight_ratio": CAP, + "executor_weight_anchor": "original_design", + "executor_max_weight_ratio": CAP, + }, + ) + assert cold["diagnostics"]["final_loss"] < cold["diagnostics"]["initial_loss"] + assert cold["diagnostics"]["skipped"] == [] + # Standard errors are absent from this invented registry, and the loss + # would ignore them either way. + assert DemographicCalibrationKernel.capabilities.consumes_se is False + + +def test_a_fresh_process_replays_warm_with_no_kernel_runs(tmp_path): + cold = graph_fixture(tmp_path) + assert cold["hits"] == [False, False, False, False] + # The child removes foreign editable namespace entries after site startup. + source_root = Path(__file__).resolve().parents[3] + code = """import sys, pathlib, importlib.util, json +w = pathlib.Path(sys.argv[1]) +sys.path[:] = [p for p in sys.path if not ('/packages/microcosm-' in p and p.endswith('/src'))] +sys.path[:0] = [str(p) for p in w.glob('packages/microcosm-*/src')] +spec = importlib.util.spec_from_file_location('age_count_graph_fixture', sys.argv[2]) +m = importlib.util.module_from_spec(spec); sys.modules[spec.name] = m; spec.loader.exec_module(m) +pathlib.Path(sys.argv[4]).write_text(json.dumps(m.graph_fixture(sys.argv[3], warm=True))) +""" + result_path = tmp_path / "fresh-warm.json" + subprocess.run( + [ + sys.executable, + "-c", + code, + str(source_root), + __file__, + str(tmp_path), + str(result_path), + ], + check=True, + timeout=180, + ) + warm = json.loads(result_path.read_text()) + assert warm["hits"] == [True, True, True, True] + assert {k: v for k, v in warm.items() if k != "hits"} == { + k: v for k, v in cold.items() if k != "hits" + } diff --git a/packages/microcosm-build/tests/test_us_puf_detail_transfer.py b/packages/microcosm-build/tests/test_us_puf_detail_transfer.py new file mode 100644 index 000000000..f2433c649 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf_detail_transfer.py @@ -0,0 +1,1026 @@ +"""Invented-only raw Schedule C transfer; no genuine source or tax calculation.""" + +import importlib.util +import json +import subprocess +import sys +import time +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_puf_detail_transfer as graph_detail +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import puf_growth_graph as growth_graph +from microcosm.build.us_runtime import puf_monetary_agi_projection as agi +from microcosm.build.us_runtime import puf_raw_source as raw +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_qrf import LegacyQRFTrainKernel, legacy_qrf_train_nodes +from microcosm.fit.model_input import decode_recipient_matrix +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import SourceCodecRegistry, load_raw_bytes +from microcosm.graph.population import Population, dtype_for_token + +_fixture_spec = importlib.util.spec_from_file_location( + "puf_detail_source_fixture", + Path(__file__).with_name("test_us_puf_price_baseline.py"), +) +source_fixture = importlib.util.module_from_spec(_fixture_spec) +sys.modules[_fixture_spec.name] = source_fixture +_fixture_spec.loader.exec_module(source_fixture) + +HERE = Path(__file__).resolve().parents[3] +BOUNDARY = "fixture_puf_detail.population" + + +def arm(channel, wage_delta=0): + if channel == "asec": + pids, households, tax_units = [1, 2, 3], [11, 11, 42], [7, 7, 15] + head, spouse, dependent = [True, False, True], [False] * 3, [False, True, False] + wages, statuses, hh_ids, weights = ( + [10.0 + wage_delta, 999.0, 20.0], + ["SURVIVING_SPOUSE", "SINGLE"], + [11, 42], + [13.0, 0.0], + ) + ages = [40, 12, 50] + else: + pids, households, tax_units = [1, 2, 4, 7], [11] * 4, [7, 7, 7, 8] + head, spouse, dependent = ( + [True, False, False, True], + [False, True, False, False], + [False, False, True, False], + ) + wages, statuses, hh_ids, weights = ( + [100.0 + wage_delta, 70.0, 800.0, 30.0], + ["JOINT", "SINGLE"], + [11], + [7.0], + ) + ages = [40, 39, 12, 21] + unique_units = list(dict.fromkeys(tax_units)) + person = pd.DataFrame( + { + "person_id": pids, + "person_household_id": households, + "person_tax_unit_id": tax_units, + "person_spm_unit_id": households, + "person_family_id": households, + "person_marital_unit_id": pids, + "age": ages, + "is_tax_unit_head": head, + "is_tax_unit_spouse": spouse, + "is_tax_unit_dependent": dependent, + "source_year": pd.Series( + ["2024"] * len(pids), dtype=dtype_for_token("string") + ), + ( + "asec_reported_wage_income_2024_price" + if channel == "asec" + else "employment_income_before_lsr" + ): wages, + } + ) + person[channel + "_source_only"] = pd.Series( + [pd.NA, *range(1, len(pids))], dtype="Int64" + ) + return Frame( + { + "person": person, + "household": pd.DataFrame( + {"household_id": hh_ids, "state_fips": [6] * len(hh_ids)} + ), + "tax_unit": pd.DataFrame( + { + "tax_unit_id": unique_units, + "filing_status_input": pd.Series( + statuses, dtype=dtype_for_token("string") + ), + } + ), + "spm_unit": pd.DataFrame({"spm_unit_id": hh_ids}), + "family": pd.DataFrame({"family_id": hh_ids}), + "marital_unit": pd.DataFrame({"marital_unit_id": pids}), + }, + US_SCHEMA, + {"household": Weights(np.array(weights), WeightKind.DESIGN)}, + pd.Series([channel] * len(pids), dtype=dtype_for_token("string")), + ) + + +def host(wage_delta=0): + result = assemble_spines( + {"asec": arm("asec", wage_delta), "acs": arm("acs", wage_delta)}, + household_mass_shares={"asec": 0.6, "acs": 0.4}, + ) + tables = {} + for e in result.entities: + table = result.table(e).copy() + for col in table: + if not pd.api.types.is_numeric_dtype(table[col]): + table[col] = table[col].astype(dtype_for_token("string")) + tables[e] = table + return Frame( + tables, + US_SCHEMA, + {"household": result.weights_for("household")}, + result.strata.astype(dtype_for_token("string")), + metadata=result.metadata, + ) + + +class Host(KernelBase): + """Only hard-coded invented households; this source accepts no external data.""" + + ref = "test.puf_detail.invented_host@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.CREATE, + numeric=Numeric.PLATFORM_BITWISE, + ) + + def run(self, context): + assert set(context.params) == {"wage_delta"} + assert context.params["wage_delta"] in (0, 1) + assert ( + context.sources["literal_host"].read_bytes() + == b"invented fixed host literals v1\n" + ) + frame = host(context.params["wage_delta"]) + scope = { + "scope": detail.SCOPE, + "native_content_sha256": detail.population_content(frame), + "source_fixture_sha256": codec.sha( + codec.encode_json( + { + "fixture": "literal-two-arm-households-v1", + "wage_delta": context.params["wage_delta"], + } + ) + ), + } + return KernelResult( + frame=frame, + artifacts={"fixture_host": codec.encode_json(scope)}, + receipt={ + "scope": detail.SCOPE, + "source": "code-owned invented literals only", + }, + ) + + +class RawScope(KernelBase): + ref = "test.puf_detail.raw_scope@1" + capabilities = Host.capabilities + + def run(self, context): + assert ( + context.sources["literal_scope"].read_bytes() + == b"invented fixed raw scope v1\n" + ) + return KernelResult( + frame=Frame( + { + "person": pd.DataFrame({"person_id": [1], "person_model_id": [1]}), + "model": pd.DataFrame({"model_id": [1], "fixture_anchor": [0.0]}), + }, + EntitySchema(group_entities=("model",)), + {"model": Weights(np.ones(1), WeightKind.DESIGN)}, + ) + ) + + +def fixture_sources(root, weight_delta=0): + root.mkdir(exist_ok=True) + ordinary = [ + source_fixture.main_record( + recid=str(11 + i), + mars=str(1 + i % 4), + s006=str(10 + i + (weight_delta if i == 0 else 0)), + amounts={ + "E00200": str(10 * i), + "E00900": str((-9, 0, 17)[i % 3]), + "E00100": str(4 * i - 11), + }, + ) + for i in range(16) + ] + # Permuted and interleaved aggregates retain delivered order, including decimals. + aggregates = [ + source_fixture.main_record( + recid=str(i), mars="0", s006="0", amounts={"E00900": "-1234.50"} + ) + for i in (999999, 999997, 999996, 999998) + ] + records = [ + aggregates[0], + *ordinary[:8], + aggregates[1], + *ordinary[8:], + *aggregates[2:], + ] + decoded, status, projection, definition = source_fixture.build_fixture(records) + main = source_fixture.csv_bytes(source_fixture.MAIN_HEADER, records) + demographic = source_fixture.csv_bytes( + source_fixture.DEMOGRAPHIC_HEADER, source_fixture.DEFAULT_DEMOGRAPHIC + ) + source_paths = {} + for name, body in [ + (definition.main.source_name, main), + (definition.demographic.source_name, demographic), + ("fixture_cpi", source_fixture.FIXTURE_CPI_RESOURCE_BYTES), + ]: + path = root / name + if path.exists(): + assert path.read_bytes() == body + else: + path.write_bytes(body) + source_paths[name] = path + status_bytes = raw.encode_return_status(status, definition) + projection_bytes = agi.encode_agi_projection( + decoded, projection, definition, status_artifact_sha256=codec.sha(status_bytes) + ) + payloads = { + "definition": definition.canonical, + "projection_definition": projection.canonical, + "status": status_bytes, + "projection": projection_bytes, + } + inputs = { + name: { + "path": str(root / (name + ".fixture")), + "bytes": len(body), + "sha256": codec.sha(body), + } + for name, body in payloads.items() + } + code_head = subprocess.check_output( + ["git", "-C", str(HERE), "rev-parse", "HEAD"], text=True + ).strip() + resource = { + k: v + for k, v in source_fixture.FIXTURE_RESOURCE.items() + if k != "resource_bytes" + } + compiled = source_fixture.compile_price_baseline( + detail.MONEY_FIELDS, **source_fixture.FIXTURE_RESOURCE + ) + params = { + "scope": detail.SCOPE, + "definition": definition.params_text, + "projection": projection.params_text, + "resource": codec.encode_json(resource).decode(), + "inputs": codec.encode_json(inputs).decode(), + "code_head": code_head, + } + return definition, projection, compiled, params, source_paths + + +class Calls(list): + def __init__(self): + super().__init__() + self.contexts = {} + self.timings = {} + + +def setup(root, *, wage_delta=0, weight_delta=0, forbid=False): + definition, projection, compiled_price, params, source_paths = fixture_sources( + root / ("source-" + str(weight_delta)), weight_delta + ) + for name, body in ( + ("literal_scope", b"invented fixed raw scope v1\n"), + ("literal_host", b"invented fixed host literals v1\n"), + ): + path = root / name + if path.exists(): + assert path.read_bytes() == body + else: + path.write_bytes(body) + source_paths[name] = path + registry = SourceCodecRegistry() + raw.register_us_puf_raw_source_codecs(registry, definition=definition) + # The raw CPI byte codec itself is the real graph source codec. + registry.register_bytes("raw-bytes-v1", load_raw_bytes) + store = ContentStore(root / "store", codecs=registry) + population = host(wage_delta) + columns = frame_column_declarations(population) + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base="fixture_host", source_channels=("acs", "asec") + ) + fits = legacy_qrf_train_nodes( + "fixture_puf_fit", + population="fixture_puf_donor", + entity="tax_unit", + predictors=detail.FEATURES, + targets=(detail.TARGET,), + seed=578, + n_estimators=2, + zero_atol=0.0, + phase=detail.SCOPE, + ) + source_fixture_sha = codec.sha( + codec.encode_json( + {"fixture": "literal-two-arm-households-v1", "wage_delta": wage_delta} + ) + ) + nodes = ( + Node( + "raw_scope", + RawScope.ref, + structural=StructuralDelta.CREATE, + sources=("literal_scope",), + outputs=(Owned("model", "fixture_anchor", "float64"),), + ), + raw.us_puf_raw_source_node(population="raw_scope", definition=definition), + agi.us_puf_monetary_agi_source_node( + population="raw_scope", definition=definition, projection=projection + ), + *graph_detail.price_nodes(params, compiled_price), + Node( + "fixture_host", + Host.ref, + structural=StructuralDelta.CREATE, + outputs=columns, + sources=("literal_host",), + params={"wage_delta": wage_delta}, + artifact_outputs=(ArtifactOutput("fixture_host", graph_detail.HOST_TYPE),), + ), + *clone_nodes, + *fits, + *graph_detail.host_nodes( + columns, + base=clone_nodes[0].id, + host_producer="fixture_host", + source_fixture_sha256=source_fixture_sha, + fit_nodes=fits, + ), + ) + graph = Graph( + "us", + ( + *raw.us_puf_raw_source_refs(definition), + SourceRef("fixture_cpi", "raw-bytes-v1"), + SourceRef("literal_host", "raw-bytes-v1"), + SourceRef("literal_scope", "raw-bytes-v1"), + ), + nodes, + ) + kernels = KernelRegistry() + raw.register_us_puf_raw_source_kernels( + kernels, definition=definition, source_codecs=registry + ) + agi.register_us_puf_monetary_agi_source_kernels( + kernels, definition=definition, projection=projection, source_codecs=registry + ) + growth_graph.register_us_puf_growth_kernels(kernels) + clone.register_us_combined_survey_clone_kernels(kernels) + for kernel in ( + Host(), + RawScope(), + graph_detail.FixturePriceAdaptKernel(), + graph_detail.FixturePriceExportKernel(), + graph_detail.FixtureDonorKernel(), + graph_detail.DetailBoundaryKernel(), + graph_detail.DetailMatrixKernel(), + graph_detail.DetailAttachKernel(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + calls = Calls() + for instance in kernels.as_mapping().values(): + original = instance.run + + def counted(context, fn=original): + calls.append(context.node.id) + calls.contexts[context.node.id] = context + if forbid: + raise AssertionError("WARM_KERNEL_CALLED") + started = time.monotonic() + try: + return fn(context) + finally: + calls.timings[context.node.id] = time.monotonic() - started + + instance.run = counted + return compile_graph(graph), store, kernels, source_paths, calls + + +def execute(root, *, forbid=False, wage_delta=0, weight_delta=0): + graph, store, kernels, sources, calls = setup( + root, forbid=forbid, wage_delta=wage_delta, weight_delta=weight_delta + ) + source_calls = [] + if forbid: + + def no_source(*args, **kwargs): + source_calls.append("graph source loader") + raise AssertionError("WARM_SOURCE_CODEC_CALLED") + + store.codecs.load = no_source + store.codecs.load_bytes = no_source + captured = {} + first_boundary = [] + + def observe(frame, event, value): + if ( + event == "return" + and isinstance(value, Population) + and value.version in (clone.COMBINED_CLONE_NODE, BOUNDARY) + ): + captured[value.version] = { + k: v.copy() for k, v in value.design_weights.items() + } + if ( + value.version == BOUNDARY + and value.mass_ledger + and value.mass_ledger[-1].node_id == BOUNDARY + and not first_boundary + ): + first_boundary.append(value) + + previous = sys.getprofile() + sys.setprofile(observe) + try: + manifest = run_graph( + graph, + sources=sources, + store=store, + kernels=kernels, + resume="require" if forbid else "auto", + ) + finally: + sys.setprofile(previous) + before, after = first_boundary[0].frame, manifest.population(BOUNDARY) + assert detail.population_content(before) == detail.population_content( + manifest.population(clone.COMBINED_CLONE_NODE) + ) + assert first_boundary[0].mass_ledger[:-1] == manifest.mass_ledger( + clone.COMBINED_CLONE_NODE + ) + placement = store.load_bytes( + manifest.node("fixture_puf_detail.matrix").opaque_artifacts["placement"] + ) + proof = graph_detail.verify_materialized_transfer( + before, + after, + placement, + before_design=first_boundary[0].design_weights, + after_design=captured[BOUNDARY], + before_ledger=first_boundary[0].mass_ledger, + after_ledger=manifest.mass_ledger(BOUNDARY), + boundary_node=BOUNDARY, + ) + evidence = { + "scope": detail.SCOPE, + "nodes": { + name: { + "key": receipt.key, + "hit": receipt.hit, + "implementation": receipt.kernel_impl_hash, + "artifacts": dict(receipt.opaque_artifacts), + } + for name, receipt in manifest.nodes.items() + }, + "opaque_payload_sha256": { + name + "/" + slot: codec.sha(store.load_bytes(key)) + for name, receipt in manifest.nodes.items() + for slot, key in receipt.opaque_artifacts.items() + }, + "materialized_content_sha256": detail.population_content(after), + "kernel_calls": list(calls), + "kernel_seconds": calls.timings, + "source_codec_calls_during_required_graph": source_calls, + "preservation": proof, + "setup_note": "Code-owned fixture definitions are constructed/decoded before run_graph, including warm setup.", + } + ( + root + / f"graph-evidence-{'require' if forbid else 'auto'}-{wage_delta}-{weight_delta}.json" + ).write_text(json.dumps(evidence, indent=2) + "\n") + return manifest, store, calls, captured, proof + + +def test_price_reader_refuses_unbound_bytes(): + from microcosm.build.us_runtime.puf_detail_transfer import decode_price_arrays + + with pytest.raises(ValueError, match="PRICE"): + decode_price_arrays(b"not an envelope", expected={}, expected_sha256="0" * 64) + + +def test_real_invented_transfer(cold): + manifest, store, calls, _, proof = cold[1] + assert len(calls) == len(manifest.nodes) + assert proof["all_nonowned_preserved"] + after = manifest.population(BOUNDARY) + assert after.n("household") == 6 and after.n("tax_unit") == 8 + matrix = decode_recipient_matrix( + store.load_bytes( + manifest.node("fixture_puf_detail.matrix").opaque_artifacts["matrix"] + ) + ) + assert matrix.features.iloc[:, 0].tolist() == [10.0, 20.0, 170.0, 30.0] + assert matrix.features.mars_2.tolist() == [1.0, 0.0, 1.0, 0.0] + raw_values = codec.read_raw_target( + store.load_bytes( + manifest.node("fixture_puf_detail.apply.000").opaque_artifacts["raw_draw"] + ), + target=detail.TARGET, + index=matrix.features.index, + ) + observed = ( + after.table("tax_unit") + .set_index("tax_unit_id") + .loc[matrix.entity_ids, detail.OUTPUT] + .to_numpy() + ) + np.testing.assert_array_equal(observed.view("uint64"), raw_values.view("uint64")) + donor = manifest.population("fixture_puf_donor") + assert donor.weights_for("tax_unit").kind is WeightKind.DESIGN + assert len(donor.table("tax_unit")) == 16 + assert set(np.sign(donor.table("tax_unit")[detail.TARGET])) == {-1, 0, 1} + + +@pytest.fixture(scope="module") +def cold(tmp_path_factory): + root = tmp_path_factory.mktemp("puf-detail-literals") + return root, execute(root) + + +def altered(frame, mutator): + tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + mutator(tables) + return Frame( + tables, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata, + ) + + +def attach_context(cold): + return cold[1][2].contexts["fixture_puf_detail.attach"] + + +def change_artifact(context, name, *, payload=None, producer=None): + from microcosm.graph.keys import opaque_artifact_key + + artifacts = dict(context.artifacts) + value = artifacts[name] + if payload is not None: + value = replace(value, payload=payload) + if producer is not None: + edge = next(e for e in context.node.artifact_inputs if e.name == name) + value = replace( + value, + producer_key=producer, + key=opaque_artifact_key(producer, edge.artifact), + ) + artifacts[name] = value + return replace(context, artifacts=artifacts) + + +@pytest.mark.parametrize( + "mutation,reason", + [ + ("payload", "PRICE_PAYLOAD_SHA"), + ("body", "PRICE_BODY_SHA"), + ("trailing", "PRICE_TRAILING_BYTES"), + ("truncated", "PRICE_BODY_TRUNCATED"), + ("header", "PRICE_IDENTITY"), + ], +) +def test_price_reader_refusals(cold, mutation, reason): + _, (manifest, store, _, _, _) = cold + receipt = manifest.node("fixture_puf_price.export") + body = store.load_bytes(receipt.opaque_artifacts["price"]) + binding = codec.decode_json( + store.load_bytes(receipt.opaque_artifacts["price_binding"]) + ) + expected, sha = dict(binding["header"]), binding["payload_sha256"] + if mutation in ("payload", "body"): + body = body[:-1] + bytes([body[-1] ^ 1]) + elif mutation == "trailing": + body += b"x" + elif mutation == "truncated": + body = body[:-1] + else: + expected["source_head"] = "f" * 40 + if mutation != "payload": + sha = codec.sha(body) + with pytest.raises(ValueError, match=reason): + detail.decode_price_arrays(body, expected=expected, expected_sha256=sha) + + +@pytest.mark.parametrize("alias", ["raw_draw", "matrix"]) +def test_sibling_producer_refusal(cold, alias): + context = change_artifact(attach_context(cold), alias, producer="f" * 64) + with pytest.raises(ValueError, match="DETAIL_SIBLING_PRODUCER"): + graph_detail.DetailAttachKernel().run(context) + + +@pytest.mark.parametrize( + "mutation,reason", + [ + ("matrix_key", "DETAIL_MATRIX_PRODUCER"), + ("seed", "DETAIL_RAW_STATE_BINDING"), + ("entity", "DETAIL_STATE_ENTITY_INDEX"), + ("target", "DETAIL_RAW_STATE_BINDING"), + ("raw_sha", "DETAIL_RAW_STATE_BINDING"), + ], +) +def test_apply_state_binding(cold, mutation, reason): + context = attach_context(cold) + packet = codec.decode_json(context.artifacts["apply_state"].payload) + if mutation == "matrix_key": + packet["matrix_producer_key"] = "f" * 64 + elif mutation == "seed": + packet["application"]["seed"] += 1 + elif mutation == "entity": + packet["application"]["state"]["entity"] = "household" + elif mutation == "target": + packet["application"]["state"]["predictors"] = list(reversed(detail.FEATURES)) + else: + packet["application"]["raw_targets"][0]["sha256"] = "f" * 64 + context = change_artifact(context, "apply_state", payload=codec.encode_json(packet)) + with pytest.raises(ValueError, match=reason): + graph_detail.DetailAttachKernel().run(context) + + +@pytest.mark.parametrize("mutation", ["order", "zero_weight_omission", "feature"]) +def test_matrix_exact_selected_rows(cold, mutation): + from microcosm.fit.model_input import encode_recipient_matrix + + context = attach_context(cold) + matrix = decode_recipient_matrix(context.artifacts["matrix"].payload) + features = matrix.features.copy() + if mutation == "order": + features = features.iloc[::-1] + elif mutation == "zero_weight_omission": + # The second selected unit is the retained zero-weight ASEC household. + assert context.weights["tax_unit"].values[5] == 0 + features = features.drop(features.index[1]) + else: + features.iloc[0, 0] += 1 + payload = encode_recipient_matrix( + features, entity="tax_unit", entity_ids=features.index.to_numpy() + ) + context = change_artifact(context, "matrix", payload=payload) + with pytest.raises(ValueError, match="DETAIL_PLACEMENT_BINDING"): + graph_detail.DetailAttachKernel().run(context) + + +def test_raw_id_order_even_with_resealed_application(cold): + context = attach_context(cold) + matrix = decode_recipient_matrix(context.artifacts["matrix"].payload) + values = codec.read_raw_target( + context.artifacts["raw_draw"].payload, + target=detail.TARGET, + index=matrix.features.index, + ) + raw_payload = codec.encode_raw_target( + values[::-1], target=detail.TARGET, index=matrix.features.index[::-1] + ) + state = codec.decode_json(context.artifacts["apply_state"].payload) + state["application"]["raw_targets"][0]["sha256"] = codec.sha(raw_payload) + context = change_artifact(context, "raw_draw", payload=raw_payload) + context = change_artifact(context, "apply_state", payload=codec.encode_json(state)) + with pytest.raises(ValueError, match="target/index/order"): + graph_detail.DetailAttachKernel().run(context) + + +def test_native_cell_ownership_refused(cold): + context = attach_context(cold) + node = replace(context.node, outputs=(Owned("tax_unit", detail.OUTPUT, "float64"),)) + with pytest.raises(ValueError, match="DETAIL_OWNERSHIP"): + graph_detail.DetailAttachKernel().run(replace(context, node=node)) + + +@pytest.mark.parametrize("kind", ["missing", "complex", "bool"]) +def test_missing_or_untyped_host_wage_refuses(cold, kind): + before = cold[1][0].population(clone.COMBINED_CLONE_NODE) + name = "asec_reported_wage_income_2024_price" + + def mutate(tables): + if kind == "missing": + tables["person"][name] = np.nan + elif kind == "complex": + tables["person"][name] = np.ones(len(tables["person"]), dtype=complex) * ( + 1 + 2j + ) + else: + tables["person"][name] = True + + with pytest.raises(ValueError, match="HOST_MISSING_FEATURE"): + detail.recipient_matrix(altered(before, mutate)) + + +@pytest.mark.parametrize( + "kind", ["role_type", "cohort", "source_channel", "membership"] +) +def test_clone_source_and_membership_refusals(cold, kind): + before = cold[1][0].population(clone.COMBINED_CLONE_NODE) + p = graph_detail.provenance + + def mutate(tables): + person = tables["person"] + role = person[p.support_clone_index_column("person")].eq(1) + if kind == "role_type": + person[p.support_clone_index_column("person")] = person[ + p.support_clone_index_column("person") + ].astype(float) + elif kind == "cohort": + person.loc[role, "source_year"] = "2023" + elif kind == "source_channel": + # Both candidate wage fields are finite, so lineage, not missing data, + # must reject this coherent role-1 whole-household source relabel. + person["asec_reported_wage_income_2024_price"] = 10.0 + person["employment_income_before_lsr"] = 10.0 + for e, table in tables.items(): + selected = table[p.support_clone_index_column(e)].eq(1) + col = p.support_channel_column(e) + table.loc[selected, col] = table.loc[selected, col].map( + {"asec": "acs", "acs": "asec"} + ) + else: + # Move a detail person across two same-role tax units. No weight fit. + selected = person.index[role] + person.loc[selected[-4], "person_tax_unit_id"] = person.loc[ + selected[-1], "person_tax_unit_id" + ] + + reason = ( + "HOST_CLONE_TYPE" + if kind == "role_type" + else "HOST_COPIED_SOURCE_CELL" + if kind != "membership" + else "HOST_CLONE_MEMBERSHIP" + ) + with pytest.raises(ValueError, match=reason): + detail.recipient_matrix(altered(before, mutate)) + + +def test_genuine_source_definition_refused(): + with pytest.raises(ValueError): + detail.fixture_sources( + raw.packaged_definition().params_text, + agi.packaged_agi_projection().params_text, + ) + + +def test_source_profile_rekeys_only_extension(monkeypatch): + kernel = graph_detail.DetailMatrixKernel() + before = kernel.implementation_hash() + original = raw.csv_acceptance_profile + monkeypatch.setattr( + raw, + "csv_acceptance_profile", + lambda: {**original(), "invented_test_profile": 1}, + ) + assert kernel.implementation_hash() != before + + +def test_host_change_reuses_donor_fit(cold): + root, (original, _, _, _, _) = cold + changed, _, calls, _, proof = execute(root, wage_delta=1) + assert ( + changed.node("fixture_puf_fit.000").key + == original.node("fixture_puf_fit.000").key + ) + assert changed.node("fixture_puf_fit.000").hit + assert "fixture_puf_fit.000" not in calls + for name in ( + "fixture_puf_detail.matrix", + "fixture_puf_detail.apply.000", + "fixture_puf_detail.attach", + ): + assert ( + changed.node(name).key != original.node(name).key + and not changed.node(name).hit + ) + assert proof["all_nonowned_preserved"] + + +def test_donor_weight_change_rekeys_fit_preserves_host(cold): + root, (original, _, _, _, _) = cold + changed, _, calls, _, proof = execute(root, weight_delta=1) + assert ( + changed.node("fixture_puf_fit.000").key + != original.node("fixture_puf_fit.000").key + ) + assert "fixture_puf_fit.000" in calls + assert ( + changed.node(clone.COMBINED_CLONE_NODE).key + == original.node(clone.COMBINED_CLONE_NODE).key + ) + assert changed.node(clone.COMBINED_CLONE_NODE).hit + np.testing.assert_array_equal( + changed.population(BOUNDARY).weights_for("household").values.view("uint64"), + original.population(BOUNDARY).weights_for("household").values.view("uint64"), + ) + assert proof["all_nonowned_preserved"] + + +@pytest.mark.parametrize( + "kind", ["anchor", "zero_anchor", "ledger", "omitted_column", "native_output"] +) +def test_full_materialized_population_refusals(cold, kind): + _, (manifest, store, _, _, _) = cold + before, after = ( + manifest.population(clone.COMBINED_CLONE_NODE), + manifest.population(BOUNDARY), + ) + placement = store.load_bytes( + manifest.node("fixture_puf_detail.matrix").opaque_artifacts["placement"] + ) + ledger = manifest.mass_ledger(BOUNDARY) + # Separate actual Population fixture: the main pilot has no incoming DESIGN + # anchor roster, so it cannot claim nonempty-history coverage by itself. + anchors = np.arange(before.n("household"), dtype="float64") + first = Population.from_frame( + before, BOUNDARY, mass_ledger=ledger, design_weights={"household": anchors} + ) + last = Population.from_frame( + after, BOUNDARY, mass_ledger=ledger, design_weights={"household": anchors} + ) + + def verify(a, b, payload=placement): + return graph_detail.verify_materialized_transfer( + a.frame, + b.frame, + payload, + before_design=a.design_weights, + after_design=b.design_weights, + before_ledger=a.mass_ledger, + after_ledger=b.mass_ledger, + boundary_node=BOUNDARY, + ) + + assert verify(first, last)["design_anchor_entities"] == 1 + if kind in ("anchor", "zero_anchor"): + changed = anchors.copy() + changed[0 if kind == "zero_anchor" else 2] += 1 + last = replace(last, design_weights={"household": changed}) + reason = "MATERIALIZED_DESIGN_CHANGED" + elif kind == "ledger": + last = replace( + last, mass_ledger=(*ledger[:-1], replace(ledger[-1], node_id="forged")) + ) + reason = "MATERIALIZED_GRAPH_LEDGER_CHANGED" + elif kind == "omitted_column": + # A projected kernel cannot see this omitted native source column. The + # materialized proof rejects its matching projection digest anyway. + projected = altered( + before, lambda t: t["person"].drop(columns="age", inplace=True) + ) + bound = codec.decode_json(placement) + bound["host_content_sha256"] = detail.population_content(projected) + placement = codec.encode_json(bound) + reason = "MATERIALIZED_HOST_BINDING" + else: + bad = altered( + after, lambda t: t["tax_unit"].loc.__setitem__((0, detail.OUTPUT), 123.0) + ) + last = replace(last, frame=bad) + reason = "MATERIALIZED_NATIVE_OUTPUT" + with pytest.raises(ValueError, match=reason): + verify(first, last, placement) + + +def test_donor_price_authority_and_original_metadata(cold): + context = cold[1][2].contexts["fixture_puf_donor"] + result = graph_detail.FixtureDonorKernel().run(context) + projection = detail.fixture_sources( + context.params["definition"], context.params["projection"] + )[1] + assert result.receipt["source_projection_document"] == projection.params_text + assert result.receipt["source_admission"] == detail.SOURCE_ADMISSION + binding = codec.decode_json(context.artifacts["price_binding"].payload) + binding["header"]["contract_sha256"] = "f" * 64 + changed = change_artifact( + context, "price_binding", payload=codec.encode_json(binding) + ) + with pytest.raises(ValueError, match="DONOR_PRICE_AUTHORITY"): + graph_detail.FixtureDonorKernel().run(changed) + + +def test_changed_host_authority_refused(cold): + context = attach_context(cold) + scope = codec.decode_json(context.artifacts["fixture_host"].payload) + scope["scope"] = "genuine" + changed = change_artifact(context, "fixture_host", payload=codec.encode_json(scope)) + with pytest.raises(ValueError, match="HOST_FIXTURE_AUTHORITY"): + graph_detail.DetailAttachKernel().run(changed) + + +@pytest.mark.parametrize( + "kind", + ["unknown", "joint_missing_spouse", "survivor_has_spouse", "negative", "nonfinite"], +) +def test_filing_structure_and_wage_refusals(cold, kind): + before = cold[1][0].population(clone.COMBINED_CLONE_NODE) + + def mutate(tables): + units = tables["tax_unit"] + if kind == "unknown": + units["filing_status_input"] = "UNKNOWN" + elif kind == "joint_missing_spouse": + units.loc[ + units.filing_status_input.eq("SURVIVING_SPOUSE"), "filing_status_input" + ] = "JOINT" + elif kind == "survivor_has_spouse": + units.loc[units.filing_status_input.eq("JOINT"), "filing_status_input"] = ( + "SURVIVING_SPOUSE" + ) + else: + tables["person"]["asec_reported_wage_income_2024_price"] = ( + -1.0 if kind == "negative" else np.inf + ) + + reason = ( + "HOST_FILING_STATUS" + if kind == "unknown" + else "HOST_FILING_STRUCTURE" + if "spouse" in kind + else "HOST_NONFINITE_FEATURE" + ) + with pytest.raises(ValueError, match=reason): + detail.recipient_matrix(altered(before, mutate)) + + +def test_negative_donor_wage_refuses_before_fit(cold): + context = cold[1][2].contexts["fixture_puf_donor"] + _, projection, status, decoded, _, _ = graph_detail.fixture_inputs(context) + binding = codec.decode_json(context.artifacts["price_binding"].payload) + arrays = detail.decode_price_arrays( + context.artifacts["price"].payload, + expected=binding["header"], + expected_sha256=binding["payload_sha256"], + ) + arrays["E00200"] = np.full_like(arrays["E00200"], -1.0) + with pytest.raises(ValueError, match="DONOR_WAGE_FEATURE"): + detail.donor_frame(arrays, status, decoded, projection) + + +def test_materialized_mask_cannot_move_to_person(cold): + manifest, store, _, anchors, _ = cold[1] + before = manifest.population(clone.COMBINED_CLONE_NODE) + after = manifest.population(BOUNDARY) + + def move_mask(tables): + tables["tax_unit"].drop(columns=detail.MASK, inplace=True) + tables["person"][detail.MASK] = False + + # This is a valid Frame: the name exists once, on the wrong entity. Merely + # adding a duplicate name would be rejected by Frame before this boundary. + moved = altered(after, move_mask) + placement = store.load_bytes( + manifest.node("fixture_puf_detail.matrix").opaque_artifacts["placement"] + ) + ledger = manifest.mass_ledger(BOUNDARY) + with pytest.raises(ValueError, match="MATERIALIZED_NONOWNED_CHANGED"): + graph_detail.verify_materialized_transfer( + before, + moved, + placement, + before_design=anchors[BOUNDARY], + after_design=anchors[BOUNDARY], + before_ledger=ledger, + after_ledger=ledger, + boundary_node=BOUNDARY, + ) + + +def test_context_strips_only_tax_unit_owned_names(cold): + context = attach_context(cold) + tables = {e: table.copy(deep=True) for e, table in context.tables.items()} + tables["tax_unit"].drop(columns=detail.MASK, inplace=True) + tables["person"][detail.MASK] = False + restored = graph_detail.context_frame(replace(context, tables=tables)) + assert detail.MASK in restored.person and detail.MASK not in restored.table( + "tax_unit" + ) diff --git a/packages/microcosm-build/tests/test_us_puf_price_baseline.py b/packages/microcosm-build/tests/test_us_puf_price_baseline.py new file mode 100644 index 000000000..befc7339e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf_price_baseline.py @@ -0,0 +1,1843 @@ +"""The developmental CPI-U price-restatement baseline, on invented bytes only. + +No PUF is read, no artifact store is touched and no country model is imported. +The delivery, the raw definition and the projection document are all built +from literals here and travel the explicit ``test_fixture`` route. The CPI +levels are the real published annual averages, because the point of the +developmental authority is that a public resource may say so about itself; the +resource body and URL below are fabricated fixtures, with the digest computed +from those exact invented bytes. A genuine run must supply its retained body. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import math +import struct +from pathlib import Path + +import numpy as np +import pytest + +from microcosm.build.us_runtime import puf_price_baseline as baseline +from microcosm.build.us_runtime.puf_growth import ( + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + FactorAuthority, + FieldRole, + GrownPufTable, + GrowthFactorTable, + GrowthKind, + GrowthRecipe, + GrowthRule, + PufGrowthRefusalError, + SignBranch, + apply_puf_design_weight_growth, + apply_puf_growth, + compile_puf_growth, + puf_growth_monetary_metadata, +) +from microcosm.build.us_runtime.puf_monetary_agi_projection import ( + AGI_FIELD, + fixture_agi_projection_document, + packaged_agi_projection, +) +from microcosm.build.us_runtime.puf_monetary_source import ( + AGGREGATE_SENTINEL, + PufMonetaryRefusalError, + PufMonetarySourceProjection, + decode_puf_monetary_source, + projection_document_json, +) +from microcosm.build.us_runtime.puf_price_baseline import ( + DESIGN_WEIGHT_IDENTITY_SERIES, + PAIRWISE_OBSERVATIONS, + PRICE_BASELINE_LEVELS, + PRICE_BASELINE_RATIO, + PRICE_BASELINE_SERIES_NAME, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + adapt_projection_to_money_view, + compile_price_baseline, + price_baseline_factor_document, + price_baseline_invariants, + price_baseline_recipe, + require_price_baseline_contract, +) +from microcosm.build.us_runtime.puf_raw_source import ( + PUF_AGGREGATE_RECIDS, + decode_puf_raw_source, + definition_document_json, + fixture_definition, + packaged_definition, +) + +PACKAGED_DEFINITION = packaged_definition() +PACKAGED_THIRTEEN = packaged_agi_projection() +MAIN_HEADER = list(PACKAGED_DEFINITION.main.delivered_header) +DEMOGRAPHIC_HEADER = list(PACKAGED_DEFINITION.demographic.delivered_header) + +#: A fabricated stand-in for the BLS flat file, in the publisher's own +#: tab-delimited, space-padded shape (see ``us-puf-price-baseline-cpi-pin.py`` +#: in the architecture-review lane, which extracted this exact row shape from +#: the genuine download). This is **not** the genuine ~2.7 MB distribution — +#: it is three invented lines carrying the two real published levels, and its +#: digest below is computed from these exact bytes, not asserted. +FIXTURE_CPI_RESOURCE_BYTES = ( + b"series_id \tyear\tperiod\t value\tfootnote_codes\r\n" + b"CUUR0000SA0 \t2015\tM13\t 237.017\t\r\n" + b"CUUR0000SA0 \t2024\tM13\t 313.689\t\r\n" +) + +#: A second fabricated body, distinguished from the one above only by an +#: extra row for a series this baseline does not read. Same two verified +#: records, a different whole-file digest — the shape +#: ``test_the_two_contracts_differ_only_in_identity`` needs, without simply +#: asserting a ``resource_sha256`` string no bytes back up. +FIXTURE_CPI_RESOURCE_BYTES_ALT = FIXTURE_CPI_RESOURCE_BYTES + ( + b"CUUS0000SA0 \t2024\tM13\t 312.332\t\r\n" +) + +#: Invented stand-ins for the pin a genuine run supplies. The URL is a +#: reserved non-resolvable domain; nothing here retrieved anything. +FIXTURE_RESOURCE = { + "source_url": "https://example.invalid/pub/time.series/cu/cu.data.1.AllItems", + "resource_sha256": hashlib.sha256(FIXTURE_CPI_RESOURCE_BYTES).hexdigest(), + "resource_bytes": FIXTURE_CPI_RESOURCE_BYTES, + "retrieved_utc": "2026-09-06T00:00:00+00:00", +} + + +def header_digest(header) -> str: + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def git_blob_sha1(payload: bytes) -> str: + return hashlib.sha1( + b"blob " + str(len(payload)).encode("ascii") + b"\0" + payload + ).hexdigest() + + +def csv_bytes(header, records) -> bytes: + lines = [",".join(header)] + lines.extend(",".join(record) for record in records) + return ("\n".join(lines) + "\n").encode("utf-8") + + +def main_record(*, recid, mars="1", s006="12345", flpdyr="2015", amounts=None): + values = { + "RECID": recid, + "MARS": mars, + "DSI": "0", + "S006": s006, + "FLPDYR": flpdyr, + "FLPDMO": "12", + **(amounts or {}), + } + return [values.get(name, "0") for name in MAIN_HEADER] + + +def demographic_record(*, recid): + values = { + "RECID": recid, + "AGEDP1": "0", + "AGEDP2": "0", + "AGEDP3": "0", + "AGERANGE": "2", + "EARNSPLIT": "0", + "GENDER": "1", + } + return [values[name] for name in DEMOGRAPHIC_HEADER] + + +#: Four invented returns and one publisher disclosure aggregate. Row 14 has a +#: qualified dividend above its ordinary dividend, so the pairwise observation +#: has a nonzero violation count to preserve rather than a vacuous zero. +DEFAULT_MAIN = [ + main_record( + recid="11", + s006="12345", + flpdyr="2015", + amounts={ + "E00100": "71000", + "E00200": "50000", + "E00300": "125", + "E00400": "40", + "E00600": "900", + "E00650": "700", + "E00900": "12000", + "E01000": "3400", + "E01500": "8000", + "E01700": "6400", + "E02100": "150", + "P22250": "1200", + "P23250": "2200", + }, + ), + main_record( + recid="12", + mars="2", + s006="9900", + flpdyr="2013", + amounts={ + "E00100": "-9300", + "E00200": "31000", + "E00900": "-2500", + "E01000": "-3000", + "E02100": "-780", + "P22250": "-3000", + "P23250": "0", + }, + ), + main_record(recid="13", mars="4", s006="700", flpdyr="2014"), + main_record( + recid="14", + s006="42", + flpdyr="2012", + amounts={"E00100": "79", "E00200": "77", "E00650": "2"}, + ), + main_record( + recid="15", + s006="1000", + flpdyr="2015", + amounts={ + "E00100": "600", + "E00200": "600", + "E00600": "500", + "E00650": "100", + }, + ), + main_record( + recid=str(PUF_AGGREGATE_RECIDS[0]), + mars="0", + s006="0", + amounts={ + "E00100": "-999999999.50", + "E00200": "999999999.12", + "E01000": "-99999999.75", + }, + ), +] +DEFAULT_DEMOGRAPHIC = [demographic_record(recid="11"), demographic_record(recid="13")] + + +def build_fixture(main_records=None): + mains = DEFAULT_MAIN if main_records is None else main_records + main = csv_bytes(MAIN_HEADER, mains) + demographic = csv_bytes(DEMOGRAPHIC_HEADER, DEFAULT_DEMOGRAPHIC) + document = definition_document_json(PACKAGED_DEFINITION) + document["route"] = "test_fixture" + document["authority"] = "invented_fixture_nonauthority" + for name, payload, header, records in ( + ("main", main, MAIN_HEADER, len(mains)), + ("demographic", demographic, DEMOGRAPHIC_HEADER, len(DEFAULT_DEMOGRAPHIC)), + ): + entry = document["sources"][name] + entry["bytes"] = len(payload) + entry["sha256"] = hashlib.sha256(payload).hexdigest() + entry["git_blob_sha1"] = git_blob_sha1(payload) + entry["delivered_header"] = list(header) + entry["delivered_header_width"] = len(header) + entry["header_record_canonical_sha256"] = header_digest(header) + entry["data_records"] = records + definition = fixture_definition(document) + projection_json = projection_document_json(PACKAGED_THIRTEEN) + projection_json["route"] = "test_fixture" + projection_json["authority"] = "invented_fixture_nonauthority" + projection_json["raw_source_definition"]["sha256"] = definition.sha256 + projection_json["raw_source_definition"]["bytes"] = len(definition.canonical) + for name, pin in ( + ("main", definition.main), + ("demographic", definition.demographic), + ): + entry = projection_json["sources"][name] + entry["sha256"] = pin.sha256 + entry["git_blob_sha1"] = pin.git_blob_sha1 + entry["bytes"] = pin.bytes + entry["header_record_canonical_sha256"] = pin.header_record_canonical_sha256 + entry["delivered_header_width"] = len(pin.delivered_header) + entry["data_records"] = pin.data_records + projection = fixture_agi_projection_document(projection_json, definition) + status = decode_puf_raw_source(main, demographic, definition) + decoded = decode_puf_monetary_source(main, status, projection, definition) + return decoded, status, projection, definition + + +@pytest.fixture +def fixture(): + return build_fixture() + + +@pytest.fixture +def contract(): + return compile_price_baseline(PACKAGED_THIRTEEN.fields, **FIXTURE_RESOURCE) + + +# -------------------------------------------------------------------------- +# The resource, the factor table and the recipe +# -------------------------------------------------------------------------- + + +def test_the_pinned_levels_are_the_published_annual_averages(): + assert PRICE_BASELINE_SOURCE_YEAR == 2015 + assert PRICE_BASELINE_TARGET_YEAR == 2024 + assert dict(PRICE_BASELINE_LEVELS) == {2015: "237.017", 2024: "313.689"} + assert PRICE_BASELINE_RATIO == (313689, 237017) + assert baseline.CPI_U_SERIES_ID == "CUUR0000SA0" + assert baseline.CPI_U_ANNUAL_AVERAGE_PERIOD == "M13" + + +def test_the_declared_factor_is_the_exact_rational_in_float64(contract): + """The level-text division and the exact rational agree bit for bit here.""" + numerator, denominator = PRICE_BASELINE_RATIO + exact = numerator / denominator + from_levels = float(PRICE_BASELINE_LEVELS[2024]) / float( + PRICE_BASELINE_LEVELS[2015] + ) + assert exact == from_levels + bits = struct.pack(" abs(losses["exact_source_sum"]) + assert losses["relative_difference"] <= losses["relative_difference_net_scaled"] + + +def test_a_design_weight_on_another_series_or_another_year_is_declined(fixture): + """The weight's own series and source year are part of the declaration.""" + series = { + PRICE_BASELINE_SERIES_NAME: price_series(2015, 2024), + DESIGN_WEIGHT_IDENTITY_SERIES: identity_weight_series(), + "some.other.identity": identity_weight_series(), + } + compiled = compiled_case( + baseline_rules(weight_series="some.other.identity"), series + ) + error = refusal_from_both_entries(fixture, compiled) + assert error.reason == "PRICE_BASELINE_DESIGN_WEIGHT_SERIES" + + # ``GrowthRecipe`` already pins the weight's target year to the money + # target year, so the open dimension is its source year. + rules = [ + rule for rule in baseline_rules().rules if str(rule.role) != "design_weight" + ] + rules.append( + GrowthRule( + "S006", + FieldRole.DESIGN_WEIGHT, + 2014, + 2024, + GrowthKind.DESIGN_WEIGHT_GROWTH, + SignBranch.ANY, + DESIGN_WEIGHT_IDENTITY_SERIES, + "case", + ) + ) + # Every series must span the table's whole year set, so the price series + # carries a 2014 column too. No money rule names it. + compiled = compiled_case( + GrowthRecipe("case_recipe_v1", tuple(rules)), + { + PRICE_BASELINE_SERIES_NAME: { + "kind": "nominal_price_restatement", + "levels": {"2014": "236.736", "2015": "237.017", "2024": "313.689"}, + }, + DESIGN_WEIGHT_IDENTITY_SERIES: { + "kind": "design_weight_growth", + "levels": {"2014": "1", "2015": "1", "2024": "1"}, + }, + }, + ) + error = refusal_from_both_entries(fixture, compiled) + assert error.reason == "PRICE_BASELINE_DESIGN_WEIGHT_YEAR" + + +def test_a_grown_table_missing_a_provenance_column_is_refused(fixture, contract): + adaptation, grown = grown_for(fixture, contract) + del grown.table[PROVENANCE_RECID_COLUMN] + with pytest.raises(PufGrowthRefusalError) as error: + price_baseline_invariants( + adaptation, grown, contract, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES + ) + assert error.value.reason == "PROVENANCE_COLUMN_MISSING" + assert error.value.field == PROVENANCE_RECID_COLUMN + + +def test_a_weight_side_that_is_not_one_dimensional_is_refused(fixture, contract): + """The before side is checked too, not only the array a caller passes.""" + adaptation, grown = grown_for(fixture, contract) + forged = dataclasses.replace( + adaptation, design_weight=adaptation.design_weight.reshape(-1, 1) + ) + with pytest.raises(PufGrowthRefusalError) as error: + price_baseline_invariants( + forged, grown, contract, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES + ) + assert error.value.reason == "DESIGN_WEIGHT_SHAPE" + + +def test_a_pairwise_column_present_on_only_one_side_is_refused(fixture): + """Absent on both sides is an observation not made; absent on one is a bug.""" + decoded, status, _, _ = fixture + subset = compile_price_baseline( + [f for f in PACKAGED_THIRTEEN.fields if f != "E00650"], **FIXTURE_RESOURCE + ) + adaptation = adapt_projection_to_money_view( + decoded, status, subset, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES + ) + grown = apply_puf_growth(adaptation.money_view, subset) + assert "E00650" not in adaptation.money_view.columns + grown.table["E00650"] = np.zeros(len(grown.table), dtype="float64") + with pytest.raises(PufGrowthRefusalError) as error: + price_baseline_invariants( + adaptation, grown, subset, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES + ) + assert error.value.reason == "PAIRWISE_COLUMN_ASYMMETRY" + assert error.value.field == "qualified_within_ordinary_dividends" + + +@pytest.mark.parametrize( + "text", + [ + b"313.6890", + b"313.6890000001", + b"3.13689e2", + b"+313.689", + b"NaN", + b"inf", + b"", + b"\xff", + ], +) +def test_source_level_text_must_match_exactly(text): + body = FIXTURE_CPI_RESOURCE_BYTES.replace(b"313.689", text) + with pytest.raises(PufGrowthRefusalError) as error: + baseline.verify_cpi_resource_bytes( + body, resource_sha256=hashlib.sha256(body).hexdigest() + ) + assert error.value.reason == "PRICE_BASELINE_RESOURCE_RECORD_VALUE" + assert error.value.field == "2024" + + +@pytest.mark.parametrize( + ("old", "new"), + [(b"CUUR0000SA0", b"CUUS0000SA0"), (b"2024", b"2023"), (b"M13", b"M12")], +) +def test_wrong_series_year_or_period_cannot_supply_a_required_record(old, new): + body = FIXTURE_CPI_RESOURCE_BYTES.replace(old, new) + with pytest.raises(PufGrowthRefusalError) as error: + baseline.verify_cpi_resource_bytes( + body, resource_sha256=hashlib.sha256(body).hexdigest() + ) + assert error.value.reason == "PRICE_BASELINE_RESOURCE_RECORD_MISSING" + + +@pytest.mark.parametrize("year,level", [(b"2015", b"237.017"), (b"2024", b"313.689")]) +def test_conflicting_or_repeated_required_records_are_refused(year, level): + body = ( + FIXTURE_CPI_RESOURCE_BYTES + b"CUUR0000SA0\t" + year + b"\tM13\t999.999\t\r\n" + ) + with pytest.raises(PufGrowthRefusalError) as error: + baseline.verify_cpi_resource_bytes( + body, resource_sha256=hashlib.sha256(body).hexdigest() + ) + assert error.value.reason == "PRICE_BASELINE_RESOURCE_DUPLICATE_RECORD" + assert error.value.field == year.decode("ascii") + + +def test_bls_field_padding_and_lf_or_crlf_do_not_change_the_record_text(): + body = FIXTURE_CPI_RESOURCE_BYTES.replace(b"\r\n", b"\n").replace(b" ", b" ") + receipt = baseline.verify_cpi_resource_bytes( + bytearray(body), resource_sha256=hashlib.sha256(body).hexdigest() + ) + assert receipt["records"]["2015"]["level_text"] == "237.017" + assert receipt["records"]["2024"]["level_text"] == "313.689" + + +@pytest.mark.parametrize( + "entry", ["factor_document", "compile", "gate", "adapt", "invariants"] +) +def test_every_baseline_entry_requires_resource_bytes(entry, fixture, contract): + decoded, status, _, _ = fixture + arguments = {k: v for k, v in FIXTURE_RESOURCE.items() if k != "resource_bytes"} + with pytest.raises(TypeError, match="resource_bytes"): + if entry == "factor_document": + price_baseline_factor_document(**arguments) + elif entry == "compile": + compile_price_baseline(PACKAGED_THIRTEEN.fields, **arguments) + elif entry == "gate": + require_price_baseline_contract(contract) + elif entry == "adapt": + adapt_projection_to_money_view(decoded, status, contract) + else: + price_baseline_invariants(object(), object(), contract) + + +@pytest.mark.parametrize( + "entry", ["factor_document", "compile", "gate", "adapt", "invariants"] +) +def test_every_baseline_entry_authenticates_the_named_digest(entry, fixture, contract): + decoded, status, _, _ = fixture + arguments = {**FIXTURE_RESOURCE, "resource_bytes": FIXTURE_CPI_RESOURCE_BYTES_ALT} + with pytest.raises(PufGrowthRefusalError) as error: + if entry == "factor_document": + price_baseline_factor_document(**arguments) + elif entry == "compile": + compile_price_baseline(PACKAGED_THIRTEEN.fields, **arguments) + elif entry == "gate": + require_price_baseline_contract( + contract, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES_ALT + ) + elif entry == "adapt": + adapt_projection_to_money_view( + decoded, status, contract, resource_bytes=FIXTURE_CPI_RESOURCE_BYTES_ALT + ) + else: + price_baseline_invariants( + object(), + object(), + contract, + resource_bytes=FIXTURE_CPI_RESOURCE_BYTES_ALT, + ) + assert error.value.reason == "PRICE_BASELINE_RESOURCE_DIGEST" + + +@pytest.mark.parametrize("entry", ["gate", "adapt", "invariants"]) +def test_generic_growth_compiler_cannot_bypass_record_verification(entry, fixture): + body = FIXTURE_CPI_RESOURCE_BYTES.replace(b"313.689", b"999.999") + document = json.loads(price_baseline_factor_document(**FIXTURE_RESOURCE)) + # The generic format accepts a declared resource digest. Baseline entry + # points must authenticate it even when their own factory was bypassed. + document["provenance"]["resource_sha256"] = hashlib.sha256(body).hexdigest() + compiled = compile_puf_growth( + price_baseline_recipe(PACKAGED_THIRTEEN.fields), + GrowthFactorTable.from_bytes( + json.dumps( + document, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + ), + ) + decoded, status, _, _ = fixture + with pytest.raises(PufGrowthRefusalError) as error: + if entry == "gate": + require_price_baseline_contract(compiled, resource_bytes=body) + elif entry == "adapt": + adapt_projection_to_money_view( + decoded, status, compiled, resource_bytes=body + ) + else: + price_baseline_invariants(object(), object(), compiled, resource_bytes=body) + assert error.value.reason == "PRICE_BASELINE_RESOURCE_RECORD_VALUE" diff --git a/packages/microcosm-build/tests/test_us_survey_age_activation.py b/packages/microcosm-build/tests/test_us_survey_age_activation.py new file mode 100644 index 000000000..3c6a24b69 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_age_activation.py @@ -0,0 +1,151 @@ +"""Invented S0101-only bytes exercise activation mechanics, never real evidence.""" + +import hashlib +import json +from dataclasses import replace + +import pytest + +from microcosm.build.us_runtime import survey_age_activation as activation +from microcosm.build.us_runtime import survey_age_sources as source + + +def invented_documents(): + """Obvious arithmetic fixture values, independent of all measured inputs.""" + row = {"GEO_ID": "0100000US", "NAME": "United States", "us": "1"} + variables = {} + bands = activation.age.NATIONAL_AGE_ACTIVATION.bands + for ordinal, (variable, label) in enumerate( + [("S0101_C01_001", "Estimate!!Total!!Total population")] + + [(band.variable, band.label) for band in bands] + ): + count = sum(range(1, 19)) if ordinal == 0 else ordinal + row.update( + { + variable + "E": str(count), + variable + "EA": None, + variable + "M": "16.45", + variable + "MA": None, + } + ) + variables[variable + "E"] = { + "group": "S0101", + "concept": "Age and Sex", + "predicateType": "int", + "label": label, + } + return {"variables": variables}, [list(row), list(row.values())] + + +def capture_fixture(root, *, metadata=None, data=None): + default_metadata, default_data = invented_documents() + documents = { + "metadata": default_metadata if metadata is None else metadata, + "data": default_data if data is None else data, + } + digests = {} + for request in source.survey_age_requests(): + payload = json.dumps(documents[request["kind"]], separators=(",", ":")).encode() + source.capture_survey_age_response(root, request, payload) + digests[request["kind"]] = hashlib.sha256(payload).hexdigest() + return activation.SurveyAgeActivation( + metadata_sha256=digests["metadata"], data_sha256=digests["data"] + ) + + +def test_invented_capture_yields_eighteen_ordered_terms_with_declared_source_semantics( + tmp_path, +): + declaration = capture_fixture(tmp_path) + registry = activation.activate_survey_age_targets(tmp_path, declaration=declaration) + assert [s.value for s in registry] == list(range(1, 19)) + assert [s.name for s in registry] == [f"S0101_C01_{i:03}" for i in range(2, 20)] + assert all( + s.metadata["reference_sha256"] == activation.activation_digest(declaration) + for s in registry + ) + assert all(s.se == pytest.approx(10) and s.period == "2024" for s in registry) + assert all( + "ASEC 2025" in s.notes and "income year 2024" in s.notes for s in registry + ) + # This is a source-documentation protocol exercised with invented values, + # not an independently authenticated real Census capture. + assert all(s.metadata["evidence_scope"] == "source_documented" for s in registry) + + +@pytest.mark.parametrize( + "change", ["estimate", "total", "duplicate", "geography", "foreign", "annotation"] +) +def test_malformed_data_refuses_instead_of_partial_activation(tmp_path, change): + metadata, data = invented_documents() + header, values = data + if change == "estimate": + values[header.index("S0101_C01_002E")] = "1.5" + elif change == "total": + values[header.index("S0101_C01_001E")] = "1" + elif change == "duplicate": + header.append(header[-1]) + values.append(values[-1]) + elif change == "geography": + values[header.index("us")] = "2" + elif change == "foreign": + header.append("UNSUPPORTED_001E") + values.append("invented-unconsumed") + else: + values[header.index("S0101_C01_002EA")] = "(X)" + declaration = capture_fixture(tmp_path, metadata=metadata, data=data) + with pytest.raises(ValueError): + activation.activate_survey_age_targets(tmp_path, declaration=declaration) + + +@pytest.mark.parametrize( + "field,value", + [ + ("group", "UNSUPPORTED"), + ("label", "Wrong age band"), + ("concept", "Other"), + ("predicateType", "float"), + ], +) +def test_publisher_definition_is_part_of_activation(tmp_path, field, value): + metadata, data = invented_documents() + metadata["variables"]["S0101_C01_002E"][field] = value + declaration = capture_fixture(tmp_path, metadata=metadata, data=data) + with pytest.raises(ValueError): + activation.activate_survey_age_targets(tmp_path, declaration=declaration) + + +def test_controlled_margin_keeps_unknown_standard_error(tmp_path): + metadata, data = invented_documents() + header, values = data + values[header.index("S0101_C01_002M")] = "-555555555" + values[header.index("S0101_C01_002MA")] = "*****" + declaration = capture_fixture(tmp_path, metadata=metadata, data=data) + registry = activation.activate_survey_age_targets(tmp_path, declaration=declaration) + assert registry.specs[0].se is None + + +def test_changed_pin_or_declaration_cannot_reuse_capture(tmp_path): + declaration = capture_fixture(tmp_path) + with pytest.raises(ValueError, match="PIN"): + activation.activate_survey_age_targets( + tmp_path, declaration=replace(declaration, data_sha256="0" * 64) + ) + binding = activation.activation_binding(declaration) + binding["period"] = "2025" + with pytest.raises(ValueError, match="DECLARATION"): + activation.declaration_from_binding(binding) + + +def test_registry_relabel_or_value_change_fails_exact_activation_binding(tmp_path): + from microcosm.calibrate.registry import TargetRegistry + + declaration = capture_fixture(tmp_path) + registry = activation.activate_survey_age_targets(tmp_path, declaration=declaration) + altered = TargetRegistry( + [replace(s, value=s.value + 1) for s in registry], country="us" + ) + with pytest.raises(ValueError, match="REGISTRY"): + activation.verify_survey_age_targets( + tmp_path, declaration=declaration, registry=altered + ) diff --git a/packages/microcosm-build/tests/test_us_survey_age_artifact.py b/packages/microcosm-build/tests/test_us_survey_age_artifact.py new file mode 100644 index 000000000..4e742e18b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_age_artifact.py @@ -0,0 +1,214 @@ +"""Invented measurement and actual graph replay; no source/target admission.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import test_us_national_age_counts as fixture + +from microcosm.build.us_runtime import graph_survey_age_artifact as artifact +from microcosm.frame import WeightKind +from microcosm.graph import ( + ContentStore, + Graph, + KernelRegistry, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def measured(): + frame = fixture.fixture_frame() + node = artifact.survey_age_count_artifact_node(population="invented") + original = fixture.count_context(frame, node) + context = replace(original, tables={"person": original.tables["person"]}) + result = artifact.SurveyAgeCountArtifactKernel().run(context) + return frame, context, result + + +def test_actual_count_kernel_preserves_population_and_emits_expected_counts(): + frame, context, result = measured() + assert not result.columns and result.frame is None and result.weights is None + values = artifact.decode_survey_age_counts(result.artifacts["counts"]) + np.testing.assert_array_equal(values.household_ids, fixture.HOUSEHOLD_IDS) + for column, expected in fixture.EXPECTED.items(): + np.testing.assert_array_equal(values.counts[column], expected) + assert values.people == 20 + assert values.population == "invented" + assert not set(fixture.COLUMNS) & set(frame.table("household")) + assert ( + result.receipt["counts_sha256"] + == hashlib.sha256(result.artifacts["counts"]).hexdigest() + ) + assert result.receipt["consumes_weights"] is False + assert context.node.outputs == () + assert context.node.inputs == (fixture.Slice("person", ("age",)),) + + +def test_counts_ignore_weights_and_return_defensive_decoded_values(): + _, context, result = measured() + changed = replace(context, weights={}) + assert ( + artifact.SurveyAgeCountArtifactKernel().run(changed).artifacts + == result.artifacts + ) + values = artifact.decode_survey_age_counts(result.artifacts["counts"]) + values.household_ids[0] = 999 + values.counts.iloc[0, 0] = 999 + fresh = artifact.decode_survey_age_counts(result.artifacts["counts"]) + assert fresh.household_ids[0] == 7 and fresh.counts.iloc[0, 0] == 1 + + +def test_actual_kernel_and_transport_preserve_full_signed_int64_ids(): + frame = fixture.fixture_frame() + ids = np.array([-(2**63), -7, 0, 5, 2**63 - 1], dtype=np.int64) + lookup = dict(zip(fixture.HOUSEHOLD_IDS, ids, strict=True)) + person = frame.table("person") + person["person_household_id"] = np.array( + [lookup[int(value)] for value in person.person_household_id], dtype=np.int64 + ) + frame.table("household")["household_id"] = ids.copy() + frame.revalidate() + node = artifact.survey_age_count_artifact_node(population="invented") + context = fixture.count_context(frame, node) + result = artifact.SurveyAgeCountArtifactKernel().run( + replace(context, tables={"person": context.tables["person"]}) + ) + values = artifact.decode_survey_age_counts(result.artifacts["counts"]) + np.testing.assert_array_equal(values.household_ids, ids) + for column, expected in fixture.EXPECTED.items(): + np.testing.assert_array_equal(values.counts[column], expected) + + +@pytest.mark.parametrize("age", [-1.0, 1.5, float("nan"), 121.0]) +def test_actual_kernel_age_refusals(age): + _, context, _ = measured() + person = context.tables["person"].copy(deep=True) + person.iloc[0, person.columns.get_loc("age")] = age + with pytest.raises(ValueError): + artifact.SurveyAgeCountArtifactKernel().run( + replace(context, tables={"person": person}) + ) + + +def rewrite_header(payload, **changes): + offset = len(artifact.MAGIC) + size = int.from_bytes(payload[offset : offset + 4], "big") + header = json.loads(payload[offset + 4 : offset + 4 + size]) + raw = artifact._json({**header, **changes}) + return ( + artifact.MAGIC + + len(raw).to_bytes(4, "big") + + raw + + payload[offset + 4 + size :] + ) + + +@pytest.mark.parametrize( + "change", + [ + {"rows": True}, + {"rows": 0}, + {"rows": 10**12}, + {"people": 0}, + {"people": 21}, + {"population": ""}, + {"columns": ["wrong"]}, + {"age_convention": "aged-forward"}, + {"protocol": "different"}, + ], +) +def test_transport_header_refusals(change): + payload = measured()[2].artifacts["counts"] + with pytest.raises(ValueError): + artifact.decode_survey_age_counts(rewrite_header(payload, **change)) + + +@pytest.mark.parametrize( + "change", + ["duplicate", "reversed", "negative_count", "trailing", "truncated", "magic"], +) +def test_transport_bytes_refuse(change): + payload = measured()[2].artifacts["counts"] + prefix = len(artifact.MAGIC) + 4 + offset = prefix + int.from_bytes(payload[len(artifact.MAGIC) : prefix], "big") + values = np.frombuffer(payload[offset:], dtype=" 0) + assert weights.tobytes() != np.array([100, 100, 100, 100, 0], dtype=float).tobytes() + bounds = stage.decode_numeric_survey_bounds(value.artifacts["bounds"].payload) + stage.check_numeric_survey_weights(bounds, weights) + diagnostics = json.loads(output.artifacts["diagnostics"]) + assert diagnostics and output.receipt["fixed_zero_rows"] == 1 + assert output.receipt["release_eligible"] is False + assert output.receipt["source_admission"] == "required_from_country_runner" + + +@pytest.mark.parametrize( + "change", + [ + {"protocol": "wrong"}, + {"budget_sha256": "z" * 64}, + {"population": ""}, + {"household_ids": [7, 15, 22, 40, True]}, + {"household_ids": [7, 15, 22, 40, 40]}, + {"household_ids": [90, 40, 22, 15, 7]}, + {"household_ids": [7, 15, 22, 40, 2**63]}, + {"group_indices": [0, 0, 1, 1, True]}, + {"group_indices": [0, 0, 1, 1, 3]}, + {"group_upper_hex": ["nan", "0x1.0p+10", "0x0.0p+0"]}, + {"group_upper_hex": ["0x1p+999999", "0x1.0p+10", "0x0.0p+0"]}, + {"row_upper_hex": [float(1).hex()] * 5}, + {"incoming_hex": [float(0).hex()] * 5}, + {"incoming_hex": [float(900).hex()] * 5}, + ], +) +def test_numeric_decoder_refuses_malformed_or_infeasible_bounds(change): + with pytest.raises(ValueError): + stage.decode_numeric_survey_bounds( + canonical_json({**numeric_document(), **change}) + ) + + +def test_numeric_values_are_defensive_and_noncanonical_bytes_refuse(): + raw = canonical_json(numeric_document()) + first = stage.decode_numeric_survey_bounds(raw) + first.incoming[0] = 999 + first.row_upper[0] = 0 + second = stage.decode_numeric_survey_bounds(raw) + assert second.incoming[0] == 100 and second.row_upper[0] == 1600 + with pytest.raises(ValueError, match="CANONICAL"): + stage.decode_numeric_survey_bounds(b" " + raw) + with pytest.raises(ValueError): + stage.decode_numeric_survey_bounds(raw[:-1]) + + +@pytest.mark.parametrize("change", ["ordered_ids", "population", "edge", "params"]) +def test_kernel_refuses_mismatched_inputs_before_solver(monkeypatch, change): + value = context() + if change == "ordered_ids": + document = {**numeric_document(), "household_ids": [8, 15, 22, 40, 90]} + value = replace( + value, + artifacts={ + **value.artifacts, + "bounds": artifact( + canonical_json(document), stage.BOUNDS_TYPE, "numeric_bounds" + ), + }, + ) + elif change == "population": + document = {**numeric_document(), "population": "other"} + value = replace( + value, + artifacts={ + **value.artifacts, + "bounds": artifact( + canonical_json(document), stage.BOUNDS_TYPE, "numeric_bounds" + ), + }, + ) + elif change == "edge": + value = replace( + value, + artifacts={ + **value.artifacts, + "bounds": replace(value.artifacts["bounds"], key="c" * 64), + }, + ) + else: + value = replace(value, params={**value.params, "grouped_preserve_zeros": False}) + calls = [] + monkeypatch.setattr(stage, "calibrate", lambda *a, **k: calls.append(True)) + with pytest.raises(ValueError): + stage.SurveyAgeCalibrationKernel().run(value) + assert not calls + + +@pytest.mark.parametrize("change", ["weights", "ids"]) +def test_late_diagnostics_mutation_is_refused(monkeypatch, change): + original = stage.diagnostics_payload + + def mutate(result, **kwargs): + payload = original(result, **kwargs) + if change == "ids": + result.frame.table("household").loc[0, "household_id"] = 999 + else: + object.__setattr__( + result.frame.weights_for("household"), "values", np.zeros(5) + ) + return payload + + monkeypatch.setattr(stage, "diagnostics_payload", mutate) + with pytest.raises(ValueError): + stage.SurveyAgeCalibrationKernel().run(context()) + + +def test_projection_can_preserve_the_full_signed_id_domain(): + document = {**numeric_document(), "household_ids": [-(2**63), -1, 0, 1, 2**63 - 1]} + assert stage.decode_numeric_survey_bounds( + canonical_json(document) + ).grouped.household_ids == tuple(document["household_ids"]) diff --git a/packages/microcosm-build/tests/test_us_survey_observed_age.py b/packages/microcosm-build/tests/test_us_survey_observed_age.py new file mode 100644 index 000000000..638603ac5 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_observed_age.py @@ -0,0 +1,120 @@ +"""Pure observed-age identity rules over invented numeric Series only.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import survey_observed_age as rule + + +@pytest.mark.parametrize("raw_dtype", ["int64", "Int64", "uint64"]) +@pytest.mark.parametrize("incumbent", ["absent", "missing", "matching", "partial"]) +def test_observed_identity_preserves_top_codes_and_inputs(raw_dtype, incumbent): + index = pd.Index([40, 30, 20, 10, 0], name="native_row") + raw = pd.Series([0, 14, 55, 80, 85], index=index, dtype=raw_dtype) + common = { + "absent": None, + "missing": pd.Series([pd.NA] * 5, index=index, dtype=object), + "matching": pd.Series([0, 14, 55, 80, 85], index=index, dtype="Int64"), + "partial": pd.Series([pd.NA, 14, pd.NA, 80, 85], index=index, dtype="Float64"), + }[incumbent] + raw_before = raw.copy(deep=True) + common_before = None if common is None else common.copy(deep=True) + result = rule.normalize_observed_age(raw, common) + assert result.dtype == np.dtype("float64") + assert result.tolist() == [0.0, 14.0, 55.0, 80.0, 85.0] + assert result.index.identical(index) and result.name == "age" + pd.testing.assert_series_equal(raw, raw_before) + if common is not None: + pd.testing.assert_series_equal(common, common_before) + result.iloc[0] = 100 + result.index.name = "owned_output" + pd.testing.assert_series_equal(raw, raw_before) + if common is not None: + pd.testing.assert_series_equal(common, common_before) + + +@pytest.mark.parametrize( + "raw, reason", + [ + (pd.Series([pd.NA], dtype="Int64"), "RAW_INTEGER_REQUIRED"), + (pd.Series([True]), "RAW_INTEGER_REQUIRED"), + (pd.Series([55.0]), "RAW_INTEGER_REQUIRED"), + (pd.Series(["55"]), "RAW_INTEGER_REQUIRED"), + (pd.Series([-1]), "RAW_REPRESENTATION"), + (pd.Series([2**53 + 1]), "RAW_REPRESENTATION"), + (pd.Series([2**64 - 1], dtype="uint64"), "RAW_REPRESENTATION"), + ], +) +def test_missing_or_nonliteral_raw_age_refuses(raw, reason): + with pytest.raises(ValueError, match=reason): + rule.normalize_observed_age(raw) + + +@pytest.mark.parametrize( + "common, reason", + [ + (pd.Series([56]), "INCUMBENT_CONFLICT"), + (pd.Series([55.5]), "INCUMBENT_CONFLICT"), + (pd.Series([True]), "INCUMBENT_NUMERIC_REQUIRED"), + (pd.Series(["55"]), "INCUMBENT_NUMERIC_REQUIRED"), + (pd.Series([55 + 0j]), "INCUMBENT_NUMERIC_REQUIRED"), + (pd.Series([float("inf")]), "INCUMBENT_REPRESENTATION"), + (pd.Series([-1]), "INCUMBENT_REPRESENTATION"), + ], +) +def test_nonmissing_common_age_must_match_without_coercion(common, reason): + with pytest.raises(ValueError, match=reason): + rule.normalize_observed_age(pd.Series([55]), common) + + +@pytest.mark.parametrize("incumbent", [2**53 + 1, 2**64 - 1]) +def test_integer_incumbent_cannot_round_into_equality(incumbent): + raw = pd.Series([2**53], dtype="uint64") + common = pd.Series([incumbent], dtype="uint64") + with pytest.raises(ValueError, match="INCUMBENT_REPRESENTATION"): + rule.normalize_observed_age(raw, common) + + +def test_ordered_index_and_exact_representation_boundary(): + raw = pd.Series([0, 2**53], index=pd.Index([2, 1], name="row")) + common = raw.astype(float) + with pytest.raises(ValueError, match="ORDERED_INDEX"): + rule.normalize_observed_age(raw, common.iloc[::-1]) + common.index = common.index.rename("different") + with pytest.raises(ValueError, match="ORDERED_INDEX"): + rule.normalize_observed_age(raw, common) + result = rule.normalize_observed_age(raw) + assert result.tolist() == [0.0, float(2**53)] + + +def test_nonseries_and_bound_refuse_before_numeric_allocation(monkeypatch): + with pytest.raises(ValueError, match="RAW_SERIES"): + rule.normalize_observed_age([55]) + with pytest.raises(ValueError, match="INCUMBENT_SERIES"): + rule.normalize_observed_age(pd.Series([55]), [55]) + monkeypatch.setattr(rule, "MAX_ROWS", 2) + raw = pd.Series([1, 2, 3]) + with monkeypatch.context() as local: + local.setattr( + pd.Series, + "to_numpy", + lambda *a, **kw: pytest.fail("Allocation occurred before row bound"), + ) + with pytest.raises(ValueError, match="ROW_BOUND"): + rule.normalize_observed_age(raw) + with pytest.raises(ValueError, match="ROW_BOUND"): + rule.normalize_observed_age(pd.Series([], dtype="int64")) + + +def test_known_negative_zero_and_rule_document_are_preserved(): + raw = pd.Series([0, 55]) + common = pd.Series([-0.0, 55.0]) + result = rule.normalize_observed_age(raw, common) + assert result.to_numpy().tobytes() == common.to_numpy().tobytes() + document = rule.rule_document() + assert document["relation"] == "numeric_identity" + assert document["temporal_adjustment"] is False + assert document["top_code_replacement"] is False + document["relation"] = "changed" + assert rule.rule_document()["relation"] == "numeric_identity" diff --git a/packages/microcosm-build/tests/test_us_survey_origin_budget.py b/packages/microcosm-build/tests/test_us_survey_origin_budget.py new file mode 100644 index 000000000..cc7c45817 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_origin_budget.py @@ -0,0 +1,562 @@ +"""Invented sources only; no calibration solver, targets or genuine inputs.""" + +import copy +import json +import sys +from fractions import Fraction + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_survey_population as graph +from microcosm.build.us_runtime import survey_origin_budget as owner +from microcosm.frame import WeightKind, Weights +from microcosm.graph import KernelResult, Node, StructuralDelta, WeightTransition +from microcosm.graph.population import PopulationError, patch + + +@pytest.mark.parametrize("d", [Fraction(0), Fraction(1, 10), Fraction(63803, 25)]) +@pytest.mark.parametrize("p", [Fraction(1), Fraction(1, 2), Fraction(3, 7)]) +@pytest.mark.parametrize("s", [Fraction(1), Fraction(1, 2)]) +def test_reference_exact_and_float_conventions(d, p, s): + a, b = d * s / p, d / p + value = owner._reference(d, p, s, float(a)) + assert value["d"] == [d.numerator, d.denominator] + assert value["b"] == [b.numerator, b.denominator] + assert value["a"] == [a.numerator, a.denominator] + assert float.fromhex(value["actual_incoming_float64_hex"]) == float(a) + assert float.fromhex(value["upper_float64_hex"]) == 4.0 * float(a) + if d == 0: + assert float.fromhex(value["upper_float64_hex"]) == 0 + + +def test_reference_refuses_staged_float_product_ulp(): + d, p, s = Fraction(1, 10), Fraction(3, 7), Fraction(1, 2) + staged = float(d) * float(s / p) + assert staged != float(d * s / p) + with pytest.raises( + owner.SurveyOriginBudgetError, match="ALLOCATION_OPERATION_ORDER" + ): + owner._reference(d, p, s, staged) + + +@pytest.mark.parametrize("bad", [True, 1, 0.5, "1", None]) +def test_reference_requires_exact_fraction(bad): + with pytest.raises(owner.SurveyOriginBudgetError, match="EXACT_FRACTION_REQUIRED"): + owner._reference(bad, Fraction(1), Fraction(1), 1.0) + + +@pytest.mark.parametrize( + "p,s", + [ + (Fraction(0), Fraction(1)), + (Fraction(2), Fraction(1)), + (Fraction(1), Fraction(0)), + (Fraction(1), Fraction(3, 4)), + ], +) +def test_reference_refuses_outside_declaration(p, s): + with pytest.raises(owner.SurveyOriginBudgetError, match="REFERENCE_DOMAIN"): + owner._reference(Fraction(1), p, s, 1.0) + + +@pytest.mark.parametrize( + "cls", [owner.SamplingOriginBudget, owner.SamplingOriginSuccessor] +) +def test_no_public_capsule_constructor(cls): + with pytest.raises(owner.SurveyOriginBudgetError, match="NO_PUBLIC"): + cls(b"{}") + + +def test_transport_exact_boundary_before_oversize_encoding(monkeypatch): + value = {"a": "x" * 32} + encoded = owner._json(value) + monkeypatch.setattr(owner, "MAX_PAYLOAD_BYTES", len(encoded)) + assert owner._json(value) == encoded + monkeypatch.setattr(owner, "MAX_PAYLOAD_BYTES", len(encoded) - 1) + with pytest.raises(graph.SurveyPopulationGraphError, match="TRANSPORT_LIMIT"): + owner._json(value) + + +def test_nullable_storage_distinguishes_hidden_values(): + left = pd.Series([1, pd.NA], dtype="Int64") + right = left.copy(deep=True) + right.array._data[1] += 19 + pd.testing.assert_series_equal(left, right) + assert not owner.storage_equal(left, right) + + +def test_refinement_refuses_before_source_borrow(): + with pytest.raises(owner.SurveyOriginBudgetError, match="UNSUPPORTED_REFINEMENT"): + owner.admit_survey_weight_only_population( + None, previous=None, current=None, previous_binding=object() + ) + + +@pytest.mark.parametrize("kind", ["prescription", "callable"]) +def test_producer_refuses_changed_loaded_contract(monkeypatch, kind): + if kind == "prescription": + monkeypatch.setattr(owner, "PRESCRIPTION", (*owner.PRESCRIPTION, "changed")) + else: + monkeypatch.setattr(owner, "_reference", lambda *args: {}) + with pytest.raises(owner.SurveyOriginBudgetError, match="PRODUCER_CHANGED"): + owner._producer() + + +def _actual(tmp_path, monkeypatch, *, fraction=Fraction(1), zero=True): + from test_us_survey_population_preparation import fixture + + arguments = fixture(tmp_path, monkeypatch, fraction=fraction, zero=zero) + return _run_actual(arguments, store_root=tmp_path / "store")[0] + + +def _run_actual(arguments, *, store_root, resume="auto"): + captured = {} + codes = { + id(graph._checked_preparation.__code__): "preparation", + id(graph._check_population_state.__code__): "population", + } + + def profile(frame, event, arg): + if event != "call": + return + kind = codes.get(id(frame.f_code)) + if kind == "preparation": + value = frame.f_locals["preparation"] + assert captured.get("preparation", value) is value + captured["preparation"] = value + elif kind == "population": + value = frame.f_locals["population"] + if value.version == graph.ALLOCATION_NODE: + captured["allocated_population"] = value + elif ( + value.version == owner.clone.COMBINED_CLONE_NODE + and owner.clone.COMBINED_CLONE_CLAIM_NODE in value.owners.values() + ): + captured["clone_population"] = value + + previous = sys.getprofile() + sys.setprofile(profile) + try: + manifest = graph.run_authenticated_survey_population( + **arguments, store_root=store_root, resume=resume + ) + finally: + sys.setprofile(previous) + assert set(captured) == {"preparation", "allocated_population", "clone_population"} + assert len(manifest.nodes) == 4 + return captured, manifest + + +def _advance(population, values, version): + # Apply the real generic REWEIGHT contract, including the unchanged original + # DESIGN scalar guard. No solver, target or replacement source authority. + node = Node( + version, + "invented.weight-only@1", + base=population.version, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={"max_weight_ratio": 8.0, "weight_anchor": "design"}, + ) + return patch( + population, + node, + KernelResult( + weights=Weights(np.asarray(values, dtype=np.float64), WeightKind.CALIBRATED) + ), + ) + + +def test_actual_initial_budget_and_zero(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + view = budget.checked_view() + assert view.preparation is arguments["preparation"] + assert view.initial_population is arguments["clone_population"] + doc = view.document + assert doc["group_count"] == 6 and len(doc["household_ids"]) == 12 + assert {row["source"] for row in doc["origins"]} == {"acs", "asec"} + assert all(len(row["members"]) == 2 for row in doc["origins"]) + zeros = [row for row in doc["origins"] if row["d"] == [0, 1]] + assert len(zeros) == 1 and zeros[0]["a"] == zeros[0]["b"] == [0, 1] + assert float.fromhex(zeros[0]["upper_float64_hex"]) == 0 + for row in doc["origins"]: + assert row["p"] == [1, 1] + assert row["upper_float64_hex"] == row["incoming_bound_float64_hex"] + values = arguments["clone_population"].frame.weights_for("household").values + view.grouped_bounds.check(values) + with pytest.raises(owner.SurveyOriginBudgetError, match="UNISSUED"): + owner.verify_survey_origin_budget(copy.copy(budget)) + doc["origins"][0]["a"] = [999, 1] + assert json.loads(budget.payload)["origins"][0]["a"] != [999, 1] + (tmp_path / "budget-metrics.json").write_text( + json.dumps( + { + "budget_bytes": len(budget.payload), + "groups": 6, + "households": 12, + "persons": 18, + "release_eligible": False, + "source_mode": "invented_only", + "calibration": "not_run", + } + ) + ) + + +def test_actual_fraction_is_catalogue_n_over_n(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch, fraction=Fraction(2, 3), zero=False) + budget = owner.freeze_survey_origin_budget(**arguments) + doc = json.loads(budget.payload) + asec = [row for row in doc["origins"] if row["source"] == "asec"] + assert len(asec) == 1 and asec[0]["p"] == [1, 2] + assert Fraction(*asec[0]["b"]) == 2 * Fraction(*asec[0]["d"]) + assert asec[0]["source_year"] == 2024 and asec[0]["survey_year"] == 2025 + + +def test_actual_rehashed_candidate_refuses(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + for field in ("p", "s", "b"): + doc = json.loads(budget.payload) + doc["origins"][0][field] = [1, 9] + changed = owner._json(doc) + with pytest.raises( + owner.SurveyOriginBudgetError, match="CANDIDATE_RECONSTRUCTION" + ): + owner.freeze_survey_origin_budget(**arguments, candidate=changed) + + +def test_actual_warm_candidate_reconstructs_fresh_source(tmp_path, monkeypatch): + from test_us_survey_population_preparation import fixture + + arguments = fixture(tmp_path, monkeypatch) + cold, cold_manifest = _run_actual(arguments, store_root=tmp_path / "store") + budget = owner.freeze_survey_origin_budget(**cold) + warm, warm_manifest = _run_actual( + arguments, store_root=tmp_path / "store", resume="require" + ) + assert warm["preparation"] is not cold["preparation"] + assert warm["clone_population"] is not cold["clone_population"] + assert warm_manifest.key == cold_manifest.key + assert len(warm_manifest.nodes) == 4 and all( + row.store_hit for row in warm_manifest.nodes.values() + ) + reconstructed = owner.freeze_survey_origin_budget(**warm, candidate=budget.payload) + assert reconstructed is not budget and reconstructed.payload == budget.payload + + +def test_actual_successor_keeps_zero_and_unequal_pair_weights(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + values = initial.frame.weights_for("household").values.copy() + values[:6] *= 1.25 + values[6:] *= 0.75 + current = _advance(initial, values, "invented.calibration.one") + binding = owner.admit_survey_weight_only_population( + budget, previous=initial, current=current + ) + checked = binding.checked_view() + assert checked.current is current and checked.previous is initial + assert checked.payload == binding.payload and checked.previous_binding is None + with pytest.raises(owner.SurveyOriginBudgetError, match="UNISSUED"): + owner.verify_survey_weight_only_successor(copy.copy(binding)) + assert json.loads(binding.payload)["budget_sha256"] == owner._sha(budget.payload) + assert np.count_nonzero(current.frame.weights_for("household").values == 0) == 2 + with pytest.raises(PopulationError, match="must move forward"): + _advance(current, values * 1.1, "invented.unsupported.refinement") + with pytest.raises(owner.SurveyOriginBudgetError, match="UNSUPPORTED_REFINEMENT"): + owner.admit_survey_weight_only_population( + budget, previous=current, current=current + ) + with pytest.raises(owner.SurveyOriginBudgetError, match="UNSUPPORTED_REFINEMENT"): + owner.admit_survey_weight_only_population( + budget, previous=initial, current=current, previous_binding=binding + ) + # Independent first-calibration branches from the same issued initial clone + # are supported by the actual graph and retain the same frozen budget. + next_population = _advance(initial, values * 1.1, "invented.calibration.two") + second = owner.admit_survey_weight_only_population( + budget, previous=initial, current=next_population + ) + assert json.loads(second.payload)["previous_binding_sha256"] is None + assert ( + json.loads(second.payload)["budget_sha256"] + == json.loads(binding.payload)["budget_sha256"] + ) + + +def test_actual_strict_group_bound_with_real_patch(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + # Six times the initial pair total exceeds U=4a, but each member remains + # below the unchanged 8d scalar graph guard. The real patch therefore runs. + values = initial.frame.weights_for("household").values * 6 + current = _advance(initial, values, "invented.over.group.bound") + with pytest.raises(ValueError, match="group weights exceed frozen absolute bounds"): + owner.admit_survey_weight_only_population( + budget, previous=initial, current=current + ) + zeros = np.flatnonzero(initial.frame.weights_for("household").values == 0) + assert len(zeros) == 2 + positive_zero = initial.frame.weights_for("household").values.copy() + positive_zero[zeros[0]] = 1 + with pytest.raises(PopulationError, match="design"): + _advance(initial, positive_zero, "invented.positive.zero") + + +def test_actual_view_decode_mutation_is_checked_before_return(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + current = _advance( + initial, initial.frame.weights_for("household").values.copy(), "invented.view" + ) + binding = owner.admit_survey_weight_only_population( + budget, previous=initial, current=current + ) + cases = ((budget, initial), (binding, current)) + for handle, population in cases: + table = population.frame.table("household") + column = table.columns.get_loc("household_id") + original = table.iloc[0, column] + injected = [] + checked_code = type(handle).checked_view.__code__ + decode_code = json.loads.__code__ + + def profile( + frame, + event, + arg, + *, + decode_code=decode_code, + checked_code=checked_code, + table=table, + column=column, + original=original, + population=population, + injected=injected, + ): + if ( + event == "return" + and frame.f_code is decode_code + and frame.f_back.f_code is checked_code + ): + table.iloc[0, column] = int(original) + 1234 + assert ( + population.frame.table("household").iloc[0, column] + == int(original) + 1234 + ) + injected.append(True) + + prior = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(owner.SurveyOriginBudgetError, match="POPULATION"): + handle.checked_view() + finally: + sys.setprofile(prior) + table.iloc[0, column] = original + assert injected == [True] + + +def _masked_numeric(frame): + for entity in frame.entities: + for column in frame.table(entity): + series = frame.table(entity)[column] + data, mask = ( + getattr(series.array, "_data", None), + getattr(series.array, "_mask", None), + ) + if ( + isinstance(data, np.ndarray) + and isinstance(mask, np.ndarray) + and mask.any() + and data.dtype.kind in "iuf" + ): + return series, int(np.flatnonzero(mask)[0]) + raise AssertionError("Invented fixture lacks a numeric nullable storage control") + + +def test_actual_nullable_backing_refuses_before_admission(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + current = _advance( + initial, initial.frame.weights_for("household").values.copy(), "invented.masked" + ) + series, position = _masked_numeric(current.frame) + before = series.copy(deep=True) + series.array._data[position] += 1 + pd.testing.assert_series_equal(before, series) + with pytest.raises(owner.SurveyOriginBudgetError, match="SUCCESSOR_NONWEIGHT"): + owner.admit_survey_weight_only_population( + budget, previous=initial, current=current + ) + + +def test_actual_retained_population_mutations_refuse(tmp_path, monkeypatch): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + for entity, column in ( + ("household", "household_id"), + ("person", "person_household_id"), + ): + table = initial.frame.table(entity) + column_position = table.columns.get_loc(column) + original = table.iloc[0, column_position] + table.iloc[0, column_position] = int(original) + 98765 + assert ( + initial.frame.table(entity).iloc[0, column_position] + == int(original) + 98765 + ) + with pytest.raises(owner.SurveyOriginBudgetError, match="POPULATION"): + owner.verify_survey_origin_budget(budget) + table.iloc[0, column_position] = original + assert initial.frame.table(entity).iloc[0, column_position] == original + weights = initial.frame.weights_for("household") + original_values = weights.values + changed_values = original_values.copy() + changed_values[0] += 1 + object.__setattr__(weights, "values", changed_values) + assert initial.frame.weights_for("household").values[0] == original_values[0] + 1 + with pytest.raises(owner.SurveyOriginBudgetError, match="POPULATION"): + owner.verify_survey_origin_budget(budget) + object.__setattr__(weights, "values", original_values) + original_design = initial.design_weights + with pytest.raises(ValueError, match="WRITEABLE"): + original_design["household"].flags.writeable = True + changed_design = original_design["household"].copy() + changed_design[0] += 1 + object.__setattr__(initial, "design_weights", {"household": changed_design}) + assert initial.design_weights["household"][0] == original_design["household"][0] + 1 + with pytest.raises(owner.SurveyOriginBudgetError, match="POPULATION"): + owner.verify_survey_origin_budget(budget) + object.__setattr__(initial, "design_weights", original_design) + for attribute, changed in (("owners", {}), ("mass_ledger", ())): + original = getattr(initial, attribute) + object.__setattr__(initial, attribute, changed) + with pytest.raises(owner.SurveyOriginBudgetError, match="POPULATION"): + owner.verify_survey_origin_budget(budget) + object.__setattr__(initial, attribute, original) + + +def _assert_final_mutation_refused(helper, operation, mutate, restore): + calls = [] + + def profile(frame, event, arg): + if event == "return" and frame.f_code is helper.__code__: + calls.append(True) + if len(calls) == 2: + mutate() + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises( + ( + owner.SurveyOriginBudgetError, + owner.source.SurveyPopulationPreparationError, + ) + ): + operation() + finally: + sys.setprofile(previous) + restore() + assert len(calls) == 2 + assert operation() is not None + + +@pytest.mark.parametrize("operation_kind", ["issuance", "borrow"]) +@pytest.mark.parametrize("changed_owner", ["preparation", "attached"]) +def test_actual_final_preparation_owner_mutation_refuses( + tmp_path, monkeypatch, operation_kind, changed_owner +): + arguments = _actual(tmp_path, monkeypatch) + preparation = arguments["preparation"] + budget = owner.freeze_survey_origin_budget(**arguments) + value = ( + preparation + if changed_owner == "preparation" + else owner.source._ISSUED[id(preparation)][2].native[1] + ) + original = value.payload + changed = original + b"\n" + + def mutate(): + object.__setattr__(value, "payload", changed) + assert object.__getattribute__(value, "payload") == changed + + def restore(): + object.__setattr__(value, "payload", original) + assert value.payload == original + + operation = ( + (lambda: owner.freeze_survey_origin_budget(**arguments)) + if operation_kind == "issuance" + else budget.checked_view + ) + _assert_final_mutation_refused(owner._initial, operation, mutate, restore) + + +@pytest.mark.parametrize("operation_kind", ["admission", "borrow"]) +def test_actual_final_budget_mutation_refuses_successor( + tmp_path, monkeypatch, operation_kind +): + arguments = _actual(tmp_path, monkeypatch) + budget = owner.freeze_survey_origin_budget(**arguments) + initial = arguments["clone_population"] + current = _advance( + initial, initial.frame.weights_for("household").values.copy(), "invented.final" + ) + + def admit(): + return owner.admit_survey_weight_only_population( + budget, previous=initial, current=current + ) + + binding = admit() + original = budget.payload + changed = original + b"\n" + + def mutate(): + object.__setattr__(budget, "payload", changed) + assert object.__getattribute__(budget, "payload") == changed + + def restore(): + object.__setattr__(budget, "payload", original) + assert budget.payload == original + + _assert_final_mutation_refused( + owner._checked_successor_document, + admit if operation_kind == "admission" else binding.checked_view, + mutate, + restore, + ) + + +def test_actual_replaced_preparation_entry_refuses_current_budget_borrow( + tmp_path, monkeypatch +): + arguments = _actual(tmp_path, monkeypatch) + preparation = arguments["preparation"] + budget = owner.freeze_survey_origin_budget(**arguments) + original = owner.source._ISSUED[id(preparation)] + copied = tuple(list(original)) + assert copied == original and copied is not original + + def mutate(): + owner.source._ISSUED[id(preparation)] = copied + assert owner.source._ISSUED[id(preparation)] is copied + + def restore(): + owner.source._ISSUED[id(preparation)] = original + assert owner.source._ISSUED[id(preparation)] is original + + _assert_final_mutation_refused(owner._initial, budget.checked_view, mutate, restore) diff --git a/packages/microcosm-build/tests/test_us_survey_population_preparation.py b/packages/microcosm-build/tests/test_us_survey_population_preparation.py new file mode 100644 index 000000000..b03ebefb8 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_population_preparation.py @@ -0,0 +1,636 @@ +"""Actual complete source catalogues and native issuers over invented bytes.""" + +import copy +import json +import os +import shutil +import sys +from fractions import Fraction +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_acs_person_coverage_authentication import ( + Member, + _csv, + _household, + _person, + build_fixture, +) +from test_us_asec_2024_native_population import _fixture as asec_fixture + +from microcosm.build.us_runtime import survey_population_preparation as owner +from microcosm.frame import WeightKind + + +def fixture( + tmp_path, + monkeypatch, + fraction=Fraction(1), + seed=41, + *, + zero=True, + zero_key=None, + missing_asec_money=None, +): + asec_root = tmp_path / "asec-original" + asec_root.mkdir() + paths = asec_fixture( + asec_root, + monkeypatch, + extra_household=True, + missing_money=missing_asec_money, + tokens=("2", "1", "2", "1", "2", "1"), + household_rows=None + if zero and zero_key is None + else [ + [ + "00007", + "1", + "0" if zero_key == "00007" else "000255212", + "6", + "1", + "2", + ], + [ + "00008", + "1", + "0" if zero_key == "00008" else "000010000", + "6", + "1", + "2", + ], + ], + ) + monkeypatch.setattr( + owner.acs_native.housing.shutil, + "disk_usage", + lambda _: SimpleNamespace(free=64 * 1024**3), + ) + households = [ + _household("2024HU0000001", NP=2), + _household("2024HU0000002", NP=1), + _household("2024GQ0000001", NP=1, TYPEHUGQ=2, WGTP=0, TEN=""), + _household("2024GQ0000002", NP=1, TYPEHUGQ=3, WGTP=0, TEN=""), + _household("2024HU0000003", NP=0, TEN=""), + ] + people = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4"), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + _person("2024GQ0000001", 1, 37, AGEP=40, MIL="4", ESR="6", PWGTP=77), + _person("2024GQ0000002", 1, 38, AGEP=50, MIL="4", ESR="6", PWGTP=78), + ] + acs = build_fixture( + tmp_path / "acs-original", + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(households)),), + person_members=(Member("psam_pusa.csv", _csv(people)),), + ) + source = tmp_path / "combined-source" + source.mkdir() + (source / "acs").mkdir() + (source / "asec").mkdir() + for name in ("csv_hus.zip", "csv_pus.zip"): + shutil.copyfile(acs.source_dir / name, source / "acs" / name) + for argument, name in ( + ("parent_path", "parent.h5"), + ("household_attachment_path", "household-attachment.h5"), + ("person_income_attachment_path", "person-income-attachment.h5"), + ("household_member_path", "hhpub25.csv"), + ): + shutil.copyfile(paths[argument], source / "asec" / name) + for year, path in paths["person_member_paths"].items(): + shutil.copyfile(path, source / "asec" / f"pppub{year - 1999}.csv") + request = { + "protocol": owner.REQUEST_PROTOCOL, + "declaration": owner.domains.DECLARATION, + "fraction": [fraction.numerator, fraction.denominator], + "seed": seed, + } + (source / "selection-request.json").write_bytes( + json.dumps(request, sort_keys=True, separators=(",", ":")).encode() + ) + snapshots = tmp_path / "captures" + snapshots.mkdir() + return dict( + source_dir=source, snapshot_root=snapshots, fraction=fraction, seed=seed + ) + + +def test_real_sources_prepare_once_without_allocation(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + result = owner.prepare_authenticated_survey_population(**arguments) + view = result.checked_view() + frame = view.frame + assert frame.weights_for("household").kind is WeightKind.DESIGN + assert frame.n("household") == 6 and frame.n("person") == 9 + assert not view.receipt["release_eligible"] + assert len(view.receipt["origins"]["households"]) == 6 + assert any( + row["original_anchor"] == [0, 1] + for row in view.receipt["origins"]["households"] + ) + assert view.receipt["catalogues"]["acs"]["counts"]["vacancies"] == 1 + assert view.selection_plan.source_authenticated is False + assert view.receipt["protocol"] == "microcosm.us.survey-population-preparation.v2" + assert owner.REQUEST_PROTOCOL == "microcosm.us.survey-population-request.v1" + assert owner.SOURCE_CODEC == "us-survey-population-source-v1" + state = owner._ISSUED[id(result)][2] + native_acs, native_asec = state.source_frames + known_acs_ages = { + ("2024HU0000001", "1"): 30, + ("2024HU0000001", "2"): 15, + ("2024HU0000002", "1"): 80, + ("2024GQ0000001", "1"): 40, + ("2024GQ0000002", "1"): 50, + } + acs_households = native_acs.table("household").set_index("household_id") + acs_serials = native_acs.person["person_household_id"].map( + acs_households["SERIALNO"] + ) + assert not acs_serials.isna().any() + acs_keys = list( + zip( + acs_serials.map(str), + native_acs.person["SPORDER"].map(str), + strict=True, + ) + ) + assert len(acs_keys) == len(known_acs_ages) and set(acs_keys) == set(known_acs_ages) + expected_acs = [known_acs_ages[key] for key in acs_keys] + known_asec_ages = { + str(5).zfill(22): 55, + str(6).zfill(22): 14, + str(7).zfill(22): 55, + str(8).zfill(22): 14, + } + asec_keys = native_asec.person["PERIDNUM"].map(str).tolist() + assert len(asec_keys) == len(known_asec_ages) and set(asec_keys) == set( + known_asec_ages + ) + expected_asec = [known_asec_ages[key] for key in asec_keys] + assert native_acs.person["AGEP"].map(str).tolist() == list(map(str, expected_acs)) + assert native_acs.person["A_AGE"].tolist() == expected_acs + assert native_asec.person["A_AGE"].tolist() == expected_asec + assert frame.person["A_AGE"].tolist() == expected_acs + expected_asec + assert frame.person["age"].tolist() == expected_acs + expected_asec + assert not frame.person["age"].isna().any() + assert "age" not in native_asec.person # The original ASEC owner stays raw. + evidence = view.receipt["origins"]["observed_age_normalization"] + assert evidence["rule"] == owner.observed_age.rule_document() + assert evidence["sources"]["acs"]["common_age_preexisting"] is True + assert evidence["sources"]["asec"]["common_age_preexisting"] is False + for index, channel in enumerate(("acs", "asec")): + assert ( + evidence["sources"][channel]["native_frame_sha256"] + == state.source_frame_seals[index] + == view.receipt["native"][channel]["frame_sha256"] + ) + + # Copy comparisons use the real issued source Frame, with no new source read. + normalized = owner._normalized_source_copy(native_asec) + owner._verify_normalized_copy(native_asec, normalized) + normalized.person["A_AGE"] = normalized.person["A_AGE"] + 1 + assert normalized.person["A_AGE"].iloc[0] == 56 + assert native_asec.person["A_AGE"].iloc[0] == 55 + with pytest.raises( + owner.SurveyPopulationPreparationError, match="NORMALIZED_SOURCE_STORAGE" + ): + owner._verify_normalized_copy(native_asec, normalized) + normalized = owner._normalized_source_copy(native_asec) + normalized.person["age"] = normalized.person["age"] + 1 + assert normalized.person["age"].iloc[0] == 56 + with pytest.raises( + owner.SurveyPopulationPreparationError, match="NORMALIZED_AGE_IDENTITY" + ): + owner._verify_normalized_copy(native_asec, normalized) + for change in ("column_name", "column_dtype", "strata_name"): + normalized = owner._normalized_source_copy(native_asec) + if change == "column_name": + normalized.person.columns = normalized.person.columns.rename("changed") + elif change == "column_dtype": + dtype = "string" if normalized.person.columns.dtype == object else object + normalized.person.columns = pd.Index(normalized.person.columns, dtype=dtype) + else: + normalized.strata.name = "changed" + with pytest.raises(owner.SurveyPopulationPreparationError, match="NORMALIZED_"): + owner._verify_normalized_copy(native_asec, normalized) + + # A temporary rule/callable change cannot borrow this issued preparation. + assert owner.observed_age in owner._modules() + assert owner.graph_population in owner._modules() + for module, name, changed in ( + (owner.observed_age, "RULE", "changed"), + (owner.observed_age, "AGE_CONVENTION", "changed"), + (owner.observed_age, "MAX_ROWS", 1), + (owner.observed_age, "MAX_EXACT_FLOAT64_INTEGER", 1), + (owner.observed_age, "normalize_observed_age", lambda *args: None), + (owner.graph_population, "_storage_parts", lambda *args: None), + ): + with monkeypatch.context() as local: + local.setattr(module, name, changed) + with pytest.raises( + owner.SurveyPopulationPreparationError, match="AUTHORITY_CHANGED" + ): + owner.verify_survey_population_preparation(result) + owner.verify_materialized_survey_population(result, frame) + + +def test_single_selection_uses_complete_n_and_exact_fraction(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch, fraction=Fraction(2, 3), zero=False) + calls = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is owner.survey_domain_sample.select_domain_households.__code__ + ): + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = owner.prepare_authenticated_survey_population(**arguments) + view = result.checked_view() + owner.verify_survey_population_preparation(result) + finally: + sys.setprofile(previous) + assert len(calls) == 1 + cell = next( + c + for c in view.selection_plan.cells + if c.source is owner.domains.Source.ASEC + and c.domain is owner.domains.Domain.SHARED_HOUSING + ) + assert ( + cell.eligible_households, + cell.selected_households, + cell.inclusion_probability, + ) == (2, 1, Fraction(1, 2)) + assert view.frame.n("household") == 5 + assert view.receipt["native"]["asec"]["households"] == 1 + assert view.receipt["native"]["asec"]["persons"] == 2 + asec_people = [ + r for r in view.receipt["origins"]["persons"]["rows"] if r[1] == "asec" + ] + assert len(asec_people) == 2 and len({r[-1] for r in asec_people}) == 1 + asec = next( + r for r in view.receipt["origins"]["households"] if r["source"] == "asec" + ) + assert asec["raw_native_id"] in {"00007", "00008"} + assert all(r["source_year"] == 2024 for r in view.receipt["origins"]["households"]) + + +def test_replay_reconstructs_sources_and_rejects_changed_candidate( + tmp_path, monkeypatch +): + arguments = fixture(tmp_path, monkeypatch) + first = owner.prepare_authenticated_survey_population(**arguments) + payload = first.to_bytes() + assert ( + json.loads(payload)["protocol"] + == "microcosm.us.survey-population-preparation.v2" + ) + replay = owner.prepare_authenticated_survey_population( + **arguments, candidate=payload + ) + assert replay is not first and replay.to_bytes() == payload + document = json.loads(payload) + document["selection"]["cells"][0]["share"] = [999, 1] + calls = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is owner.asec_native.load_authenticated_asec_2024_native_population.__code__ + ): + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.SurveyPopulationPreparationError, match="CANDIDATE_MISMATCH" + ): + owner.prepare_authenticated_survey_population( + **arguments, candidate=owner._encode(document) + ) + finally: + sys.setprofile(previous) + assert calls == [True] + + +def test_selected_zero_support_refuses_without_redraw_or_native_issuance( + tmp_path, monkeypatch +): + # Direct the invented source weights before pinning. The fixed seed is not + # searched or retried; this fixture-design draw is outside source issuance. + fraction, seed = Fraction(2, 3), 41 + domain = owner.domains.Domain.SHARED_HOUSING.value + control = owner.survey_domain_sample.select_domain_households( + row_ids=("00007", "00008"), + source_channels=("asec", "asec"), + domain_keys=(domain, domain), + cells=((domain, "asec"),), + fraction=fraction, + seed=seed, + )[0] + assert control.eligible_households == 2 and len(control.positions) == 1 + selected_key = ("00007", "00008")[control.positions[0]] + arguments = fixture(tmp_path, monkeypatch, fraction, seed, zero_key=selected_key) + calls = {"sampler": 0, "acs_native": 0, "asec_native": 0} + codes = { + owner.survey_domain_sample.select_domain_households.__code__: "sampler", + owner.acs_native.issue_acs_native_coverage.__code__: "acs_native", + owner.asec_native.load_authenticated_asec_2024_native_population.__code__: "asec_native", + } + refusals = [] + + def profile(frame, event, arg): + if event == "call" and frame.f_code in codes: + calls[codes[frame.f_code]] += 1 + + def trace(frame, event, arg): + if event == "exception" and frame.f_code is owner.selection._require.__code__: + refusals.append(str(arg[1])) + return trace + + old_profile, old_trace = sys.getprofile(), sys.gettrace() + sys.setprofile(profile) + sys.settrace(trace) + try: + with pytest.raises(owner.SurveyPopulationPreparationError): + owner.prepare_authenticated_survey_population(**arguments) + finally: + sys.setprofile(old_profile) + sys.settrace(old_trace) + assert "SELECTED_CELL_HAS_NO_POSITIVE_ANCHOR" in refusals + assert calls == {"sampler": 1, "acs_native": 0, "asec_native": 0} + + +@pytest.mark.parametrize("kind", ["wrapped_code", "closure", "default", "kwdefault"]) +def test_callable_configuration_drift_refuses_before_producer_io(monkeypatch, kind): + if kind == "wrapped_code": + monkeypatch.setattr( + owner._regular_reader.__wrapped__, "__code__", (lambda p, n: None).__code__ + ) + elif kind == "closure": + monkeypatch.setattr( + owner._regular_reader.__closure__[0], "cell_contents", lambda p, n: None + ) + elif kind == "default": + monkeypatch.setattr(owner._check_scalars, "__defaults__", (999,)) + else: + monkeypatch.setitem( + owner.prepare_authenticated_survey_population.__kwdefaults__, + "candidate", + b"substitution", + ) + calls = [] + + def profile(frame, event, arg): + if event == "call" and frame.f_code is owner._code_bytes.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises( + owner.SurveyPopulationPreparationError, match="PRODUCER_CHANGED" + ): + owner._producer() + finally: + sys.setprofile(previous) + assert calls == [] + + +def test_defensive_view_forgery_and_materialized_copy(tmp_path, monkeypatch): + result = owner.prepare_authenticated_survey_population( + **fixture(tmp_path, monkeypatch) + ) + view = result.checked_view() + view.receipt["release_eligible"] = True + assert result.checked_view().receipt["release_eligible"] is False + for forged in (copy.copy(result), view, view.receipt, view.payload): + with pytest.raises(owner.SurveyPopulationPreparationError): + owner.verify_survey_population_preparation(forged) + materialized = owner._copy_source(view.frame) + materialized.weights_for("household").values.setflags(write=True) + owner.verify_materialized_survey_population(result, materialized) + materialized.person.iloc[0, materialized.person.columns.get_loc("age")] += 1 + with pytest.raises( + owner.SurveyPopulationPreparationError, match="MATERIALIZED_FRAME_CHANGED" + ): + owner.verify_materialized_survey_population(result, materialized) + object.__setattr__(result, "payload", result.payload + b" ") + for borrow in ( + lambda: result.frame, + lambda: result.context, + lambda: result.selection_plan, + lambda: result.receipt, + result.to_bytes, + result.checked_view, + lambda: owner.verify_survey_population_preparation(result), + lambda: owner.verify_materialized_survey_population(result, view.frame), + ): + with pytest.raises(owner.SurveyPopulationPreparationError): + borrow() + + +@pytest.mark.parametrize( + "kind", + ["frame", "plan", "payload", "nested_acs", "nested_asec", "source", "source_extra"], +) +def test_late_mutation_at_final_producer_return_refuses_current_borrow( + tmp_path, monkeypatch, kind +): + arguments = fixture(tmp_path, monkeypatch) + result = owner.prepare_authenticated_survey_population(**arguments) + state = owner._ISSUED[id(result)][2] + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is owner._producer.__code__ + and frame.f_back.f_code is owner._validate.__code__ + and not fired + ): + fired.append(True) + if kind == "source_extra": + (arguments["source_dir"] / "asec" / "extra.txt").write_text("invented") + elif kind == "frame": + state.frame.person.iloc[ + 0, state.frame.person.columns.get_loc("age") + ] += 1 + elif kind == "plan": + object.__setattr__(state.plan.selected[0], "share", Fraction(99)) + elif kind == "payload": + object.__setattr__(result, "payload", result.payload + b" ") + elif kind == "nested_acs": + literal = owner.acs_native._owned(state.native[0]).literal + object.__setattr__(literal, "payload", literal.payload + b" ") + elif kind == "nested_asec": + nested = owner.asec_native._ISSUED[id(state.native[1])][2] + object.__setattr__( + nested.coverage, "_body", nested.coverage._body + b"x" + ) + else: + path = arguments["source_dir"] / "selection-request.json" + path.write_bytes(path.read_bytes() + b" ") + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(owner.SurveyPopulationPreparationError): + result.checked_view() + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize("change", ["extra", "symlink", "missing"]) +def test_exact_source_roster_refuses_before_issuers(tmp_path, monkeypatch, change): + arguments = fixture(tmp_path, monkeypatch) + source = arguments["source_dir"] + if change == "extra": + (source / "extra.json").write_bytes(b"{}") + elif change == "missing": + (source / "asec/hhpub25.csv").unlink() + else: + path = source / "asec/hhpub25.csv" + target = source.parent / "member.csv" + path.rename(target) + path.symlink_to(target) + calls = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code is owner.acs_catalogue.issue_acs_source_catalogue.__code__ + ): + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(owner.SurveyPopulationPreparationError): + owner.prepare_authenticated_survey_population(**arguments) + finally: + sys.setprofile(previous) + assert not calls + + +def test_request_changed_between_decode_and_roster_binding_refuses( + tmp_path, monkeypatch +): + arguments = fixture(tmp_path, monkeypatch) + fired = [] + + def profile(frame, event, arg): + if ( + event == "return" + and frame.f_code is owner._request.__code__ + and frame.f_back.f_code + is owner.prepare_authenticated_survey_population.__code__ + and not fired + ): + fired.append(True) + path = arguments["source_dir"] / "selection-request.json" + document = json.loads(path.read_bytes()) + document["seed"] += 1 + path.write_bytes(owner._encode(document)) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(owner.SurveyPopulationPreparationError): + owner.prepare_authenticated_survey_population(**arguments) + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize( + "change", ["duplicate", "whitespace", "unreduced", "bool_seed", "extra", "oversize"] +) +def test_request_is_canonical_and_bounded(tmp_path, change): + request = { + "protocol": owner.REQUEST_PROTOCOL, + "declaration": owner.domains.DECLARATION, + "fraction": [1, 2], + "seed": 41, + } + if change == "unreduced": + request["fraction"] = [2, 4] + elif change == "bool_seed": + request["seed"] = True + elif change == "extra": + request["extra"] = 0 + payload = json.dumps(request, sort_keys=True, separators=(",", ":")).encode() + if change == "duplicate": + payload = payload.replace(b'"seed":41', b'"seed":41,"seed":41') + elif change == "whitespace": + payload += b"\n" + elif change == "oversize": + payload = b"x" * (owner.MAX_REQUEST_BYTES + 1) + (tmp_path / "selection-request.json").write_bytes(payload) + with pytest.raises(owner.SurveyPopulationPreparationError): + owner.read_survey_population_request(tmp_path) + + +@pytest.mark.parametrize("change", ["fifo", "symlink"]) +def test_regular_reader_refuses_replacement_without_blocking( + tmp_path, monkeypatch, change +): + path = tmp_path / "input.txt" + path.write_bytes(b"invented") + target = tmp_path / "other.txt" + target.write_bytes(b"different") + actual_open = os.open + fired = [] + + def replace_at_open(value, flags, *args, **kwargs): + if value == path and not fired: + fired.append(True) + path.unlink() + if change == "fifo": + os.mkfifo(path) + else: + path.symlink_to(target) + return actual_open(value, flags, *args, **kwargs) + + monkeypatch.setattr(os, "open", replace_at_open) + with pytest.raises((owner.SurveyPopulationPreparationError, OSError)): + with owner._regular_reader(path, 100) as (stream, _info): + pytest.fail("Replacement must be refused before reading") + assert fired == [True] + + +def test_transport_bound_precedes_append(): + rows = [] + budget = [owner.MAX_PAYLOAD_BYTES - 1] + with pytest.raises(owner.SurveyPopulationPreparationError, match="ORIGIN_LIMIT"): + owner._bounded_append(rows, [1, "acs", 2], budget) + assert rows == [] + with pytest.raises(owner.SurveyPopulationPreparationError, match="PAYLOAD_LIMIT"): + owner._encode({"rows": ["invented"] * 100}, 30) + with pytest.raises(owner.SurveyPopulationPreparationError, match="SCALAR_LIMIT"): + owner._encode({"nested": ["x" * (owner._MAX_SCALAR_BYTES + 1)]}) + + +def test_semantic_identity_includes_column_and_strata_indexes(tmp_path, monkeypatch): + result = owner.prepare_authenticated_survey_population( + **fixture(tmp_path, monkeypatch) + ) + view = result.checked_view() + candidate = owner._copy_source(view.frame) + expected = owner._frame_identity(candidate) + candidate.person.columns.name = "different" + assert owner._frame_identity(candidate) != expected + candidate.person.columns.name = None + candidate.strata.index.name = "different" + assert owner._frame_identity(candidate) != expected diff --git a/packages/microcosm-build/tests/test_us_survey_population_replay.py b/packages/microcosm-build/tests/test_us_survey_population_replay.py new file mode 100644 index 000000000..86d025ddd --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_population_replay.py @@ -0,0 +1,454 @@ +"""Invented, source-independent tests of the directional cache replay boundary.""" + +from __future__ import annotations + +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime.survey_population_replay import ( + same_replayed_frame, + same_replayed_population, +) +from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph.population import MassRecord, Population +from microcosm.graph.store import ContentStore + +_ERROR = "^SURVEY_POPULATION_REPLAY_" + + +def _frame(): + index = pd.Index([101, 103, 107], dtype="int64", name="source_row") + person = pd.DataFrame( + { + "person_id": np.array([1, 2, 3], dtype=np.int64), + "person_household_id": np.array([10, 10, 20], dtype=np.int64), + "person_tax_unit_id": np.array([30, 30, 40], dtype=np.int64), + "nullable_integer": pd.arrays.IntegerArray( + np.array([11, 71, 73], dtype=np.int64), + np.array([False, True, True]), + ), + "nullable_boolean": pd.arrays.BooleanArray( + np.array([True, True, True]), np.array([False, True, True]) + ), + "native_float": np.array( + [0x8000000000000000, 0x7FF8000000000011, 0x3FF4000000000000], + dtype=np.uint64, + ).view(np.float64), + "text": pd.array(["", "invented\x00label", pd.NA], dtype="string"), + "object_value": pd.Series( + ["invented object label", b"invented bytes", None], + index=index, + dtype=object, + ), + }, + index=index, + ) + return Frame( + { + "person": person, + "household": pd.DataFrame({"household_id": [10, 20]}), + "tax_unit": pd.DataFrame({"tax_unit_id": [30, 40]}), + }, + EntitySchema(group_entities=("household", "tax_unit")), + {"household": Weights(np.array([1.5, 2.5]), WeightKind.IMPORTANCE)}, + pd.Series(["urban", "urban", "rural"], index=index, dtype=object), + mass_log=(MassChangeRecord("household", 2.0, 4.0, 2.0, "invented"),), + metadata={"source": {"arm": "invented", "members": (1, 2)}, "zero": -0.0}, + ) + + +def _rebuild(frame, **changes): + arguments = { + "tables": {entity: frame.table(entity) for entity in frame.entities}, + "schema": frame.schema, + "weights": { + entity: frame.weights_for(entity) for entity in frame.weighted_entities + }, + "strata": frame.strata, + "mass_log": frame.mass_log, + "metadata": frame.metadata, + } + arguments.update(changes) + return Frame(**arguments) + + +def _with_column(frame, name, values): + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"][name] = values + return _rebuild(frame, tables=tables) + + +def _population(frame): + return Population.from_frame( + frame, + "allocated", + mass_ledger=( + MassRecord( + "allocation", + "reweight", + "declared", + 12.0, + 4.0, + (("invented", 12.0),), + (("invented", 4.0),), + entity="household", + ), + ), + design_weights={"household": np.array([5.0, 7.0])}, + ) + + +def test_actual_store_roundtrip_accepts_canonical_nulls_and_typed_objects(tmp_path): + expected = _frame() + before_integer = expected.person["nullable_integer"].array._data.tobytes() + before_boolean = expected.person["nullable_boolean"].array._data.tobytes() + store = ContentStore(tmp_path / "invented-store") + store.put_frame("a" * 64, expected) + actual = store.load_frame("a" * 64) + + assert expected.person["nullable_integer"].array._data[1:].tolist() == [71, 73] + assert actual.person["nullable_integer"].array._data[1:].tolist() == [0, 0] + assert actual.person["nullable_boolean"].array._data[1:].tolist() == [False, False] + assert actual.person["native_float"].to_numpy().tobytes() == ( + expected.person["native_float"].to_numpy().tobytes() + ) + assert ( + actual.person["object_value"].iloc[0] == expected.person["object_value"].iloc[0] + ) + assert ( + actual.person["object_value"].iloc[0] + is not expected.person["object_value"].iloc[0] + ) + assert same_replayed_frame(expected, actual) is None + original_population = _population(expected) + assert ( + same_replayed_population( + original_population, replace(original_population, frame=actual) + ) + is None + ) + # Acceptance is read-only and must not canonicalize the original in place. + assert expected.person["nullable_integer"].array._data.tobytes() == before_integer + assert expected.person["nullable_boolean"].array._data.tobytes() == before_boolean + + +def test_unchanged_noncanonical_hidden_storage_is_accepted(): + assert same_replayed_frame(_frame(), _frame()) is None + + +@pytest.mark.parametrize("column", ["nullable_integer", "nullable_boolean"]) +@pytest.mark.parametrize("change", ["partial_canonical", "noncanonical", "reverse"]) +def test_hidden_storage_exception_is_directional_and_whole_column(column, change): + expected, actual = _frame(), _frame() + if change == "reverse": + expected.person[column].array._data[1:] = 0 + elif change == "partial_canonical": + actual.person[column].array._data[1] = 0 + elif column == "nullable_integer": + actual.person[column].array._data[1:] = [81, 83] + else: + expected.person[column].array._data[1:] = [False, True] + actual.person[column].array._data[1:] = [True, False] + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize("change", ["value", "mask", "dtype", "membership", "text"]) +def test_observed_values_masks_and_dtypes_are_exact(change): + expected = _frame() + actual = _frame() + if change == "value": + actual.person["nullable_integer"].array._data[0] += 1 + elif change == "mask": + actual.person["nullable_integer"].array._mask[1] = False + elif change == "dtype": + actual = _with_column( + actual, + "nullable_integer", + actual.person["nullable_integer"].astype("Int32"), + ) + elif change == "membership": + actual = _with_column( + actual, "person_household_id", np.array([10, 20, 20], dtype=np.int64) + ) + else: + actual = _with_column( + actual, + "text", + pd.array(["changed", "invented\x00label", pd.NA], dtype="string"), + ) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize( + "change", + [ + "rows", + "columns", + "index_name", + "index_dtype", + "column_axis_name", + "strata", + "schema_order", + ], +) +def test_axes_schema_and_strata_are_exact(change): + expected = _frame() + tables = {entity: expected.table(entity).copy() for entity in expected.entities} + arguments = {"tables": tables} + if change == "rows": + tables["person"] = tables["person"].iloc[[2, 1, 0]] + arguments["strata"] = expected.strata.iloc[[2, 1, 0]] + elif change == "columns": + tables["person"] = tables["person"].loc[ + :, list(reversed(tables["person"].columns)) + ] + elif change == "index_name": + tables["household"].index = tables["household"].index.rename("other_axis") + elif change == "index_dtype": + tables["household"].index = pd.Index([0, 1], dtype="int32") + elif change == "column_axis_name": + tables["household"].columns = tables["household"].columns.rename( + "other_columns" + ) + elif change == "strata": + arguments["strata"] = pd.Series( + ["rural", "urban", "rural"], index=expected.person.index, dtype=object + ) + else: + arguments["schema"] = EntitySchema(group_entities=("tax_unit", "household")) + actual = _rebuild(expected, **arguments) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize( + "change", + [ + "nested_metadata", + "metadata_scalar_type", + "metadata_signed_zero", + "mass_log", + "weight_bits", + "weight_signed_zero", + "weight_kind", + ], +) +def test_frame_context_and_exact_weight_contract_are_preserved(change): + expected = _frame() + if change == "nested_metadata": + actual = _rebuild( + expected, + metadata={"source": {"arm": "changed", "members": (1, 2)}, "zero": -0.0}, + ) + elif change == "metadata_scalar_type": + expected = _rebuild(expected, metadata={"flag": True}) + actual = _rebuild(expected, metadata={"flag": 1}) + elif change == "metadata_signed_zero": + actual = _rebuild( + expected, + metadata={"source": {"arm": "invented", "members": (1, 2)}, "zero": 0.0}, + ) + elif change == "mass_log": + actual = _rebuild( + expected, mass_log=(replace(expected.mass_log[0], reason="changed"),) + ) + elif change == "weight_signed_zero": + expected = _rebuild( + expected, + weights={ + "household": Weights(np.array([-0.0, 2.5]), WeightKind.IMPORTANCE) + }, + ) + actual = _rebuild( + expected, + weights={"household": Weights(np.array([0.0, 2.5]), WeightKind.IMPORTANCE)}, + ) + else: + values = expected.weights_for("household").values.copy() + kind = WeightKind.IMPORTANCE + if change == "weight_bits": + values[0] = np.nextafter(values[0], np.inf) + else: + kind = WeightKind.CALIBRATED + actual = _rebuild(expected, weights={"household": Weights(values, kind)}) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize( + "dtype,bits", + [ + ("float64", (0x8000000000000000, 0x7FF8000000000011, 0x3FF4000000000000)), + ("float32", (0x80000000, 0x7FC00011, 0x3FA00000)), + ], +) +@pytest.mark.parametrize("slot", [0, 1]) +def test_native_signed_zero_and_nan_payload_bits_cannot_change(dtype, bits, slot): + unsigned = "uint64" if dtype == "float64" else "uint32" + values = np.array(bits, dtype=unsigned) + expected = _with_column(_frame(), "native_float", values.view(dtype)) + changed = values.copy() + changed[slot] = 0 if slot == 0 else int(changed[slot]) + 1 + actual = _with_column(expected, "native_float", changed.view(dtype)) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize( + "before,after", + [ + (np.int64(7), 7), + (np.bool_(True), True), + (np.float32(1.25), 1.25), + (np.bytes_(b"x"), b"x"), + ], +) +def test_object_scalar_equivalence_uses_actual_store_types(before, after): + frame = _frame() + expected = _with_column( + frame, + "object_value", + pd.Series([before, None, None], index=frame.person.index, dtype=object), + ) + actual = _with_column( + expected, + "object_value", + pd.Series([after, None, None], index=expected.person.index, dtype=object), + ) + assert same_replayed_frame(expected, actual) is None + + +@pytest.mark.parametrize( + "before,after", + [(True, 1), (1, 1.0), (None, pd.NA), (pd.NaT, None), ("x", b"x"), (-0.0, 0.0)], +) +def test_object_scalar_types_null_sentinels_and_float_bits_cannot_change(before, after): + frame = _frame() + expected = _with_column( + frame, + "object_value", + pd.Series([before, None, None], index=frame.person.index, dtype=object), + ) + actual = _with_column( + frame, + "object_value", + pd.Series([after, None, None], index=frame.person.index, dtype=object), + ) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +def test_object_nan_payload_bits_cannot_change(): + frame = _frame() + payloads = np.array([0x7FF8000000000011, 0x7FF8000000000012], dtype=np.uint64).view( + np.float64 + ) + expected = _with_column( + frame, + "object_value", + pd.Series([payloads[0], None, None], index=frame.person.index, dtype=object), + ) + actual = _with_column( + frame, + "object_value", + pd.Series([payloads[1], None, None], index=frame.person.index, dtype=object), + ) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +@pytest.mark.parametrize("kind", ["object", "category", "nullable_float"]) +def test_equal_unsupported_store_values_still_refuse(kind, tmp_path): + frame = _frame() + if kind == "object": + values = pd.Series( + [{"unsupported": 1}, None, None], index=frame.person.index, dtype=object + ) + elif kind == "category": + values = pd.Categorical(["a", "b", "a"]) + else: + values = pd.array([1.0, pd.NA, 2.0], dtype="Float64") + unsupported = _with_column(frame, "object_value", values) + with pytest.raises(TypeError): + ContentStore(tmp_path / "unsupported-store").put_frame("b" * 64, unsupported) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(unsupported, unsupported) + + +@pytest.mark.parametrize("axis", ["index", "columns"]) +@pytest.mark.parametrize("before,after", [(True, 1), (-0.0, 0.0), ((True,), (1,))]) +def test_axis_names_preserve_store_scalar_types(axis, before, after): + expected, actual = _frame(), _frame() + getattr(expected.table("household"), axis).name = before + getattr(actual.table("household"), axis).name = after + # Pandas' axis equality alone treats each pair as equal. + assert getattr(expected.table("household"), axis).identical( + getattr(actual.table("household"), axis) + ) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(expected, actual) + + +def test_multiindex_outside_reviewed_store_profile_refuses(): + frame = _frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"].index = pd.MultiIndex.from_tuples([("a", 0), ("b", 1)]) + unsupported = _rebuild(frame, tables=tables) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_frame(unsupported, unsupported) + + +@pytest.mark.parametrize( + "change", + [ + "version", + "owner", + "design_value", + "design_signed_zero", + "missing_design", + "ledger", + "weight_kind", + ], +) +def test_population_provenance_and_original_design_bytes_are_exact(change): + expected = _population(_frame()) + if change == "version": + actual = replace(expected, version="another_version") + elif change == "owner": + owners = dict(expected.owners) + owners[("person", "nullable_integer")] = "another_owner" + actual = replace(expected, owners=owners) + elif change == "design_value": + actual = replace( + expected, + design_weights={"household": np.array([5.0, np.nextafter(7.0, np.inf)])}, + ) + elif change == "design_signed_zero": + expected = replace( + expected, design_weights={"household": np.array([-0.0, 7.0])} + ) + actual = replace(expected, design_weights={"household": np.array([0.0, 7.0])}) + elif change == "missing_design": + actual = replace(expected, design_weights={}) + elif change == "ledger": + actual = replace( + expected, + mass_ledger=( + replace(expected.mass_ledger[0], node_id="another_allocation"), + ), + ) + else: + frame = _rebuild( + expected.frame, + weights={"household": Weights(np.array([1.5, 2.5]), WeightKind.CALIBRATED)}, + ) + actual = replace( + expected, frame=frame, weight_kind={"household": WeightKind.CALIBRATED} + ) + with pytest.raises(ValueError, match=_ERROR): + same_replayed_population(expected, actual) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py b/packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py new file mode 100644 index 000000000..5e80c7731 --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py @@ -0,0 +1,185 @@ +"""Strict, stable-ID household group bounds shared by solver and admission. + +This is a numerical constraint, not authority for an origin or budget artifact. +Callers must authenticate its complete population and frozen origin bindings. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Sequence +from dataclasses import dataclass, field +from numbers import Integral + +import numpy as np + + +def _id(value: object) -> int | str: + if isinstance(value, bool): + raise ValueError("household IDs must be integers or nonempty strings") + if isinstance(value, Integral): + return int(value) + if isinstance(value, str) and value: + return value + raise ValueError("household IDs must be integers or nonempty strings") + + +@dataclass(frozen=True, eq=False) +class GroupedUpperBounds: + """An exhaustive, disjoint aligned map and absolute float64 upper bounds.""" + + household_ids: Sequence[int | str] + group_indices: np.ndarray + absolute_bounds: np.ndarray + constraint_digest: str = field(init=False) + _members: tuple[np.ndarray, ...] = field(init=False, repr=False) + + def __post_init__(self) -> None: + ids = tuple(_id(x) for x in self.household_ids) + if not ids or len(set(ids)) != len(ids): + raise ValueError("household IDs must be nonempty and unique") + groups = np.asarray(self.group_indices) + if groups.shape != (len(ids),) or groups.dtype.kind not in "iu": + raise ValueError( + "group indices must be integer and aligned to household IDs" + ) + bounds = np.asarray(self.absolute_bounds, dtype=np.float64) + if bounds.ndim != 1 or not bounds.size: + raise ValueError("absolute bounds must be a nonempty vector") + if not np.isfinite(bounds).all() or (bounds < 0).any(): + raise ValueError("absolute bounds must be finite and nonnegative") + if (groups < 0).any() or (groups >= len(bounds)).any(): + raise ValueError("unknown group index") + if set(groups.tolist()) != set(range(len(bounds))): + raise ValueError("group membership must cover every bound") + groups = np.frombuffer(groups.astype(" int: + return len(self.absolute_bounds) + + @property + def digest(self) -> str: + """Canonical ordered-ID/map/absolute-bound identity.""" + return self.constraint_digest + + def _weights(self, weights: np.ndarray, *, positive: bool = False) -> np.ndarray: + values = np.asarray(weights, dtype=np.float64) + if values.shape != (len(self.household_ids),): + raise ValueError("weights must align with all household IDs") + if not np.isfinite(values).all() or (values < 0).any(): + raise ValueError("group weights must be finite and nonnegative") + if positive and ((values <= 0).any() or (self.absolute_bounds <= 0).any()): + raise ValueError( + "grouped log-weight solver requires strictly positive weights and bounds" + ) + return values + + def totals(self, weights: np.ndarray) -> np.ndarray: + values = self._weights(weights) + try: + result = np.asarray( + [ + math.fsum(float(values[i]) for i in members) + for members in self._members + ], + dtype=np.float64, + ) + except OverflowError as exc: + raise ValueError("nonfinite group total") from exc + if not np.isfinite(result).all(): + raise ValueError("nonfinite group total") + return result + + def check(self, weights: np.ndarray, *, positive: bool = False) -> np.ndarray: + """Return exact ordered fsum totals; never forgive an overage.""" + values = self._weights(weights, positive=positive) + totals = self.totals(values) + if (totals > self.absolute_bounds).any(): + raise ValueError("group weights exceed frozen absolute bounds") + return totals + + def project( + self, weights: np.ndarray, *, positive: bool = True + ) -> tuple[np.ndarray, int]: + """Project directly from one materialized vector; no log/exp correction. + + For an overage, at most 64 downward float64 factor corrections recompute + from the unchanged candidate. A feasible input is a byte-preserving no-op. + """ + candidate = self._weights(weights, positive=positive) + totals = self.totals(candidate) + accepted = candidate.copy() + corrected = 0 + for group in np.flatnonzero(totals > self.absolute_bounds): + corrected += 1 + members = self._members[int(group)] + original = candidate[members] + factor = np.float64(self.absolute_bounds[group] / totals[group]) + for correction in range(65): + projected = original * factor + total = math.fsum(float(x) for x in projected) + if total <= self.absolute_bounds[group]: + accepted[members] = projected + break + if correction == 64: + raise ValueError( + "group projection exceeded 64 rounding corrections" + ) + factor = np.nextafter(factor, np.float64(0)) + else: # pragma: no cover + raise AssertionError("unreachable projection state") + self.check(accepted, positive=positive) + return accepted, corrected + + def diagnostics( + self, weights: np.ndarray, last_corrected_count: int = 0 + ) -> dict[str, object]: + if ( + isinstance(last_corrected_count, bool) + or not isinstance(last_corrected_count, Integral) + or not 0 <= last_corrected_count <= self.group_count + ): + raise ValueError( + "last corrected group count must be an integer from zero to group count" + ) + totals = self.check(weights) + return { + "constraint_digest": self.constraint_digest, + "group_count": self.group_count, + "binding_group_count": int( + np.count_nonzero(totals == self.absolute_bounds) + ), + "last_corrected_group_count": int(last_corrected_count), + } diff --git a/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py b/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py new file mode 100644 index 000000000..16b116747 --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_apply.py @@ -0,0 +1,139 @@ +"""One shared legacy application body for Slice and typed-matrix adapters.""" + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import qrf +from microcosm.fit.qrf_target import ( + LEGACY_QRF_TARGET_TYPE, + LegacyQRFTargetArtifact, + apply_target, +) +from microcosm.graph import ArtifactOutput, KernelResult + + +def run_application( + context, + *, + entity, + predictors, + table, + state_type=codec.APPLY_STATE_TYPE, + state_decoder=codec.read_application, + state_encoder=codec.encode_json, + extra_inputs=frozenset(), +): + if set(context.params) != {"target", "seed", "phase"}: + raise ValueError( + "Legacy apply requires exact target, seed and phase parameters." + ) + seed = context.params["seed"] + if type(seed) is not int or seed < 0: + raise ValueError("Legacy apply seed must be nonnegative.") + if context.node.artifact_outputs != ( + ArtifactOutput("raw_draw", codec.RAW_TARGET_TYPE), + ArtifactOutput("apply_state", state_type), + ): + raise ValueError( + "Legacy apply owns one raw target and a small application state." + ) + model_value = codec.artifact(context, "model", LEGACY_QRF_TARGET_TYPE) + training_value = codec.artifact( + context, "training_state", codec.TRAINING_STATE_TYPE + ) + input_by_name = {item.name: item for item in context.node.artifact_inputs} + if ( + input_by_name["model"].producer != input_by_name["training_state"].producer + or model_value.producer_key != training_value.producer_key + ): + raise ValueError("Legacy model and training state must share one fit producer.") + training, next_training = codec.read_training(training_value.payload) + target = context.params["target"] + if ( + not training["models"] + or training["models"][-1]["target"] != target + or training["models"][-1]["sha256"] != codec.sha(model_value.payload) + ): + raise ValueError( + "Legacy model bytes/target disagree with the training checkpoint." + ) + # Expected SHA is taken from the independently verified sibling output. + # The graph store/producer is trusted; a digest cannot make foreign pickle safe. + fitted = LegacyQRFTargetArtifact.from_trusted_bytes( + model_value.payload, expected_sha256=training["models"][-1]["sha256"] + ) + if ( + fitted.next_training_state != next_training + or fitted.training_id != training["models"][-1]["training_id"] + ): + raise ValueError("Legacy model training envelope differs from its checkpoint.") + before = fitted.training_state.to_dict() + if entity != before["entity"] or predictors != tuple(before["predictors"]): + raise ValueError( + "Legacy recipient Slice differs from the model entity/predictors." + ) + prior_targets = before["completed_targets"] + expected_inputs = {"model", "training_state"} | set(extra_inputs) + if prior_targets: + expected_inputs.add("apply_state") + expected_inputs.update(f"prior_{i:03d}" for i in range(len(prior_targets))) + if ( + set(input_by_name) != expected_inputs + or set(context.artifacts) != expected_inputs + ): + raise ValueError( + "Legacy apply inputs must contain the exact ordered raw prefix." + ) + if prior_targets: + previous, state = state_decoder( + codec.artifact(context, "apply_state", state_type).payload + ) + if previous["seed"] != seed or previous["models"] != training["models"][:-1]: + raise ValueError("Legacy application seed/model history changed.") + raw_history = previous["raw_targets"] + else: + # Exactly start_chain's second SeedSequence stream, not keyed draws. + _, draw_seed = np.random.SeedSequence(seed).spawn(2) + state = qrf.QRFChainState.from_dict( + { + **before, + "recipient_index": None, + "draw_rng_state": np.random.default_rng(draw_seed).bit_generator.state, + } + ) + raw_history = [] + raw = pd.DataFrame(index=table.index) + for i, prior in enumerate(prior_targets): + value = codec.artifact(context, f"prior_{i:03d}", codec.RAW_TARGET_TYPE) + if codec.sha(value.payload) != raw_history[i]["sha256"]: + raise ValueError("Legacy raw prior bytes differ from application history.") + raw[prior] = codec.read_raw_target( + value.payload, target=prior, index=table.index + ) + result = apply_target(fitted, table, raw, state=state) + payload = codec.encode_raw_target(result.raw_draw, target=target, index=table.index) + next_state = state_encoder( + { + "schema_version": 1, + "state": result.state.to_dict(), + "seed": seed, + "models": training["models"], + "raw_targets": [ + *raw_history, + {"target": target, "sha256": codec.sha(payload)}, + ], + } + ) + return KernelResult( + artifacts={"raw_draw": payload, "apply_state": next_state}, + receipt={ + "phase": context.params["phase"], + "target": target, + "recipient_rows": len(table), + "entity": entity, + "model_sha256": codec.sha(model_value.payload), + "raw_sha256": codec.sha(payload), + "regime": result.regime, + }, + ) diff --git a/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py b/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py new file mode 100644 index 000000000..ec4b8208d --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/_graph_legacy_qrf.py @@ -0,0 +1,197 @@ +"""Strict local artifact codecs for the legacy per-target graph protocol.""" + +from __future__ import annotations + +import hashlib +import json + +import numpy as np +import pandas as pd + +from microcosm.fit import qrf +from microcosm.fit.qrf_target import LegacyQRFTrainingState +from microcosm.frame import EntitySchema, Frame +from microcosm.graph import ROWS_ALL, ArtifactType, Numeric, platform_fingerprint + +TRAINING_STATE_TYPE = ArtifactType("microcosm.fit.qrf.legacy_training_state", 1) +APPLY_STATE_TYPE = ArtifactType("microcosm.fit.qrf.legacy_apply_state", 1) +RAW_TARGET_TYPE = ArtifactType("microcosm.fit.qrf.legacy_raw_target", 1) +_RAW_MAGIC = b"microcosm.fit.qrf.legacy_raw_target/1\n" + + +def sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def encode_json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def decode_json(payload: bytes) -> dict: + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Repeated legacy graph checkpoint JSON key.") + result[key] = value + return result + + def nonfinite(_): + raise ValueError("Nonfinite legacy graph checkpoint JSON number.") + + value = json.loads(payload, object_pairs_hook=pairs, parse_constant=nonfinite) + if not isinstance(value, dict) or encode_json(value) != payload: + raise ValueError("Legacy graph checkpoint must be a canonical JSON object.") + return value + + +def names(value, label): + if ( + not isinstance(value, tuple) + or not value + or any(not isinstance(name, str) or not name for name in value) + or len(set(value)) != len(value) + ): + raise ValueError(f"{label} must be a nonempty tuple of unique names.") + return value + + +def table(context, ref): + node = context.node + if node.kernel != ref or len(node.inputs) != 1 or node.inputs[0].rows != ROWS_ALL: + raise ValueError("Legacy QRF requires one all-row encoded input Slice.") + if node.outputs or node.sources: + raise ValueError( + "Legacy QRF target kernels own artifacts only and read declared slices." + ) + declared = node.inputs[0] + result = context.tables[declared.entity] + if not set(declared.columns).issubset(result): + raise ValueError("Legacy QRF table lacks declared input columns.") + if not isinstance(context.params.get("phase"), str) or not context.params["phase"]: + raise ValueError("Legacy QRF node requires its explicit outer phase.") + return declared, result + + +def model_frame(context, declared, values): + """Retain exact table index and resolved weight kind, with minimal links.""" + entity = declared.entity + table = values.loc[:, list(declared.columns)].copy() + ids = values[f"{entity}_id"].to_numpy(copy=True) + table.insert(0, f"{entity}_id", ids) + weights = context.weights[entity] + if len(weights) != len(table): + raise ValueError("Legacy QRF donor weights must align to input rows.") + if entity == "person": + table.insert(1, "person_model_unit_id", ids) + schema = EntitySchema(group_entities=("model_unit",)) + tables = {"person": table, "model_unit": pd.DataFrame({"model_unit_id": ids})} + else: + schema = EntitySchema(group_entities=(entity,)) + tables = { + entity: table, + "person": pd.DataFrame({"person_id": ids, f"person_{entity}_id": ids}), + } + return Frame(tables, schema, {entity: weights}) + + +def artifact(context, name, type_): + value = context.artifacts[name] + declared = [item for item in context.node.artifact_inputs if item.name == name] + if len(declared) != 1 or declared[0].type != type_ or value.type != type_: + raise ValueError(f"Wrong typed legacy QRF artifact: {name}.") + if ( + value.numerics.numeric is not Numeric.PLATFORM_BITWISE + or value.numerics.platform != platform_fingerprint() + ): + raise ValueError( + f"Legacy QRF {name} artifact requires this platform's bitwise scope." + ) + return value + + +def _hash(value): + return ( + isinstance(value, str) + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def history(value, targets, *, models): + if not isinstance(value, list) or len(value) != len(targets): + raise ValueError("Legacy QRF history length differs from target prefix.") + fields = {"target", "sha256", "training_id"} if models else {"target", "sha256"} + for record, target in zip(value, targets, strict=True): + if ( + not isinstance(record, dict) + or set(record) != fields + or record["target"] != target + ): + raise ValueError( + "Legacy QRF history must bind the exact ordered target prefix." + ) + if any(not _hash(record[key]) for key in fields - {"target"}): + raise ValueError("Legacy QRF history has an invalid digest.") + return value + + +def read_training(payload): + packet = decode_json(payload) + if ( + set(packet) != {"schema_version", "state", "models"} + or type(packet["schema_version"]) is not int + or packet["schema_version"] != 1 + ): + raise ValueError("Invalid legacy QRF graph training checkpoint.") + state = LegacyQRFTrainingState.from_dict(packet["state"]) + history(packet["models"], state.to_dict()["completed_targets"], models=True) + return packet, state + + +def read_application(payload): + packet = decode_json(payload) + if ( + set(packet) != {"schema_version", "state", "seed", "models", "raw_targets"} + or type(packet["schema_version"]) is not int + or packet["schema_version"] != 1 + ): + raise ValueError("Invalid legacy QRF graph application checkpoint.") + if type(packet["seed"]) is not int or packet["seed"] < 0: + raise ValueError("Invalid legacy QRF application seed.") + state = qrf.QRFChainState.from_dict(packet["state"]) + history(packet["models"], state.completed_targets, models=True) + history(packet["raw_targets"], state.completed_targets, models=False) + return packet, state + + +def encode_raw_target(values, *, target, index): + values = np.asarray(values, dtype=" np.ndarray: + """Validate raw target/index binding and return readonly exact float64 bits. + + The caller must obtain bytes from its authenticated graph artifact edge. + These raw draws are never the decoded or calibrated production columns. + """ + if not isinstance(payload, bytes) or not payload.startswith(_RAW_MAGIC): + raise ValueError("Invalid raw legacy target envelope.") + metadata, separator, body = payload[len(_RAW_MAGIC) :].partition(b"\n") + expected = {"target": target, "index": qrf._index_identity(index).to_dict()} + if ( + not separator + or decode_json(metadata) != expected + or len(body) != len(index) * 8 + ): + raise ValueError("Raw legacy target/index/order binding changed.") + result = np.frombuffer(body, dtype=" dict: + """Validate the wrapper and unchanged legacy packet for a downstream owner. + + A finalizer must additionally bind the matrix SHA and actual producer key + to its placement edge and verify the last raw SHA against its raw artifact. + This parser alone does not grant population or placement authority. + """ + packet = codec.decode_json(payload) + if ( + set(packet) + != {"schema_version", "matrix_sha256", "matrix_producer_key", "application"} + or type(packet["schema_version"]) is not int + or packet["schema_version"] != 1 + or not codec._hash(packet["matrix_sha256"]) + or not codec._hash(packet["matrix_producer_key"]) + ): + raise ValueError("Invalid matrix application checkpoint.") + _, state = codec.read_application(codec.encode_json(packet["application"])) + if not state.completed_targets: + raise ValueError("Matrix application checkpoint has no completed target.") + return packet + + +class LegacyQRFApplyMatrixKernel(KernelBase): + ref = "fit.qrf.legacy_target.apply_matrix@1" + capabilities = Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + dependencies=FIT_QRF_DEPENDENCIES, + ) + + def implementation_hash(self): + return source_hash( + type(self), + shared, + codec, + model_input, + qrf_target, + qrf, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + node = context.node + if node.kernel != self.ref or node.inputs or node.outputs or node.sources: + raise ValueError( + "Matrix apply reads typed artifacts only, with no Slice or source." + ) + if ( + not isinstance(context.params.get("phase"), str) + or not context.params["phase"] + ): + raise ValueError("Matrix apply requires its explicit outer phase.") + value = codec.artifact(context, "matrix", model_input.RECIPIENT_MATRIX_TYPE) + matrix = model_input.decode_recipient_matrix(value.payload) + binding = { + "matrix_sha256": codec.sha(value.payload), + "matrix_producer_key": value.producer_key, + } + + def read_state(payload): + packet = decode_matrix_apply_state(payload) + if any(packet[key] != expected for key, expected in binding.items()): + raise ValueError( + "Matrix application input bytes/producer changed within the chain." + ) + return codec.read_application(codec.encode_json(packet["application"])) + + def write_state(application): + return codec.encode_json( + {"schema_version": 1, **binding, "application": application} + ) + + result = shared.run_application( + context, + entity=matrix.entity, + predictors=tuple(matrix.features.columns), + table=matrix.features, + state_type=MATRIX_APPLY_STATE_TYPE, + state_decoder=read_state, + state_encoder=write_state, + extra_inputs=frozenset({"matrix"}), + ) + return KernelResult( + artifacts=result.artifacts, receipt={**result.receipt, **binding} + ) diff --git a/packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py b/packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py new file mode 100644 index 000000000..1bce184ac --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/graph_legacy_qrf.py @@ -0,0 +1,186 @@ +"""Declarations for explicit legacy donor-fit and recipient-apply target chains. + +Callers own scientifically reviewed preparation, donor support, encoding, and +finalization. Donor populations must be independent of recipient source ancestry +to reuse fits when recipient rungs change. Table indices remain the existing +pandas indices; graph entity IDs are never substituted as QRF checkpoint indices. +""" + +from __future__ import annotations + +from dataclasses import replace + +from microcosm.fit._graph_legacy_qrf import ( + APPLY_STATE_TYPE, + RAW_TARGET_TYPE, + TRAINING_STATE_TYPE, + names, + read_raw_target, +) +from microcosm.fit.graph_legacy_apply import LegacyQRFApplyKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.fit.qrf import DEFAULT_N_ESTIMATORS, DEFAULT_ZERO_ATOL +from microcosm.fit.qrf_target import LEGACY_QRF_TARGET_TYPE +from microcosm.graph import ArtifactInput, ArtifactOutput, Node, Slice + +__all__ = [ + "LegacyQRFTrainKernel", + "LegacyQRFApplyKernel", + "TRAINING_STATE_TYPE", + "APPLY_STATE_TYPE", + "RAW_TARGET_TYPE", + "legacy_qrf_train_nodes", + "legacy_qrf_apply_nodes", + "legacy_qrf_apply_matrix_nodes", + "read_raw_target", +] + + +def legacy_qrf_train_nodes( + prefix, + *, + population, + entity, + predictors, + targets, + seed, + phase, + n_estimators=DEFAULT_N_ESTIMATORS, + zero_atol=DEFAULT_ZERO_ATOL, + max_samples_leaf=None, +): + """Declare independent per-target fits; each successor reads small state only.""" + predictors = names(predictors, "predictors") + targets = names(targets, "targets") + nodes = [] + for i, target in enumerate(targets): + nodes.append( + Node( + f"{prefix}.{i:03d}", + LegacyQRFTrainKernel.ref, + population=population, + inputs=(Slice(entity, (*predictors, *targets)),), + params={ + "predictors": predictors, + "targets": targets, + "target": target, + "seed": seed, + "n_estimators": n_estimators, + "zero_atol": zero_atol, + "max_samples_leaf": max_samples_leaf, + "phase": phase, + }, + artifact_inputs=() + if i == 0 + else ( + ArtifactInput( + "training_state", + nodes[-1].id, + "training_state", + TRAINING_STATE_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("model", LEGACY_QRF_TARGET_TYPE), + ArtifactOutput("training_state", TRAINING_STATE_TYPE), + ), + ) + ) + return tuple(nodes) + + +def legacy_qrf_apply_nodes(prefix, *, population, fit_nodes, seed, phase): + """Declare recipient-only draws with immutable ordered raw target dependencies.""" + if not isinstance(fit_nodes, tuple) or not fit_nodes: + raise ValueError("fit_nodes must be a nonempty ordered tuple.") + first = fit_nodes[0] + expected_targets = first.params["targets"] + if len(fit_nodes) != len(expected_targets): + raise ValueError("fit_nodes must contain the complete target chain.") + nodes = [] + for i, fit in enumerate(fit_nodes): + if ( + fit.kernel != LegacyQRFTrainKernel.ref + or fit.params["target"] != expected_targets[i] + or fit.params["targets"] != expected_targets + or fit.inputs != first.inputs + or fit.population != first.population + ): + raise ValueError("fit_nodes must describe one ordered donor chain.") + edges = [ + ArtifactInput("model", fit.id, "model", LEGACY_QRF_TARGET_TYPE), + ArtifactInput( + "training_state", fit.id, "training_state", TRAINING_STATE_TYPE + ), + ] + if nodes: + edges.append( + ArtifactInput( + "apply_state", nodes[-1].id, "apply_state", APPLY_STATE_TYPE + ) + ) + edges.extend( + ArtifactInput(f"prior_{j:03d}", node.id, "raw_draw", RAW_TARGET_TYPE) + for j, node in enumerate(nodes) + ) + nodes.append( + Node( + f"{prefix}.{i:03d}", + LegacyQRFApplyKernel.ref, + population=population, + inputs=(Slice(fit.inputs[0].entity, fit.params["predictors"]),), + params={"target": fit.params["target"], "seed": seed, "phase": phase}, + artifact_inputs=tuple(edges), + artifact_outputs=( + ArtifactOutput("raw_draw", RAW_TARGET_TYPE), + ArtifactOutput("apply_state", APPLY_STATE_TYPE), + ), + ) + ) + return tuple(nodes) + + +def legacy_qrf_apply_matrix_nodes( + prefix, + *, + population, + fit_nodes, + matrix_producer, + matrix_artifact="matrix", + seed, + phase, +): + """Declare one fixed recipient-matrix producer for the complete target chain.""" + from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ) + from microcosm.fit.model_input import RECIPIENT_MATRIX_TYPE + + nodes = legacy_qrf_apply_nodes( + prefix, population=population, fit_nodes=fit_nodes, seed=seed, phase=phase + ) + matrix = ArtifactInput( + "matrix", matrix_producer, matrix_artifact, RECIPIENT_MATRIX_TYPE + ) + return tuple( + replace( + node, + kernel=LegacyQRFApplyMatrixKernel.ref, + inputs=(), + artifact_inputs=( + *( + replace(edge, type=MATRIX_APPLY_STATE_TYPE) + if edge.name == "apply_state" + else edge + for edge in node.artifact_inputs + ), + matrix, + ), + artifact_outputs=( + ArtifactOutput("raw_draw", RAW_TARGET_TYPE), + ArtifactOutput("apply_state", MATRIX_APPLY_STATE_TYPE), + ), + ) + for node in nodes + ) diff --git a/packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py b/packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py new file mode 100644 index 000000000..c7a494fa2 --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/graph_legacy_train.py @@ -0,0 +1,147 @@ +"""Donor-only fitted legacy QRF targets with no recipient graph dependency.""" + +from __future__ import annotations + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model as weight_module +from microcosm.fit import qrf, qrf_target +from microcosm.fit.kernels import FIT_QRF_DEPENDENCIES +from microcosm.fit.qrf_target import ( + LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainingState, + fit_target, +) +from microcosm.graph import ( + ArtifactOutput, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Numeric, + SeedSource, + source_hash, +) + + +class LegacyQRFTrainKernel(KernelBase): + ref = "fit.qrf.legacy_target.train@1" + capabilities = Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + dependencies=FIT_QRF_DEPENDENCIES, + ) + + def implementation_hash(self): + return source_hash( + type(self), + codec, + qrf_target, + qrf, + weight_module, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + declared, table = codec.table(context, self.ref) + expected_params = { + "predictors", + "targets", + "target", + "seed", + "n_estimators", + "zero_atol", + "max_samples_leaf", + "phase", + } + if set(context.params) != expected_params: + raise ValueError( + "Legacy QRF training parameters must be explicit and exact." + ) + predictors = codec.names(context.params["predictors"], "predictors") + targets = codec.names(context.params["targets"], "targets") + if set(predictors) & set(targets) or declared.columns != ( + *predictors, + *targets, + ): + raise ValueError( + "Legacy training Slice must contain predictors then disjoint ordered targets." + ) + if context.node.artifact_outputs != ( + ArtifactOutput("model", LEGACY_QRF_TARGET_TYPE), + ArtifactOutput("training_state", codec.TRAINING_STATE_TYPE), + ): + raise ValueError( + "Legacy training owns model and small training-state artifacts." + ) + names = {item.name for item in context.node.artifact_inputs} + if names not in (set(), {"training_state"}) or set(context.artifacts) != names: + raise ValueError( + "Legacy training accepts only the prior training checkpoint." + ) + seed, trees = context.params["seed"], context.params["n_estimators"] + if type(seed) is not int or seed < 0 or type(trees) is not int or trees < 1: + raise ValueError( + "Legacy fit requires nonnegative seed and positive tree count." + ) + model = qrf.RegimeGatedQRF( + seed=seed, + n_estimators=trees, + zero_atol=context.params["zero_atol"], + max_samples_leaf=context.params["max_samples_leaf"], + ) + donor = codec.model_frame(context, declared, table) + weight_kind = context.weights[declared.entity].kind.value + if names: + packet, state = codec.read_training( + codec.artifact( + context, "training_state", codec.TRAINING_STATE_TYPE + ).payload + ) + models = packet["models"] + else: + state = LegacyQRFTrainingState.from_chain( + model.start_chain( + donor, list(predictors), list(targets), weights=weight_kind + ) + ) + models = [] + state_values = state.to_dict() + if ( + tuple(state_values["predictors"]) != predictors + or tuple(state_values["targets"]) != targets + or state.next_target != context.params["target"] + ): + raise ValueError( + "Legacy training state/target order differs from declaration." + ) + fitted = fit_target(model, donor, state=state, weights=weight_kind) + payload = fitted.to_bytes() + next_models = [ + *models, + { + "target": fitted.target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + }, + ] + next_state = codec.encode_json( + { + "schema_version": 1, + "state": fitted.next_training_state.to_dict(), + "models": next_models, + } + ) + return KernelResult( + artifacts={"model": payload, "training_state": next_state}, + receipt={ + "phase": context.params["phase"], + "target": fitted.target, + "training_id": fitted.training_id, + "model_sha256": codec.sha(payload), + "donor_rows": len(table), + "entity": declared.entity, + "weight_kind": weight_kind, + "regime": fitted.regime, + }, + ) diff --git a/packages/microcosm-fit/src/microcosm/fit/model_input.py b/packages/microcosm-fit/src/microcosm/fit/model_input.py new file mode 100644 index 000000000..b6d8a93b2 --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/model_input.py @@ -0,0 +1,140 @@ +"""Finite model features with an exact, separate entity-ID row index. + +V1 is deliberately narrower than a pandas serializer: ordinary int64 Index, +None/string name, positive row count and little-endian float64 columns only. +The producer owns scientific encoding; this codec never casts source values. +""" + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.qrf import _index_identity +from microcosm.graph import ArtifactType + +RECIPIENT_MATRIX_TYPE = ArtifactType("microcosm.fit.legacy_recipient_matrix", 1) +MATRIX_HEADER_MAX_BYTES = 64 * 1024 +_MAGIC = b"microcosm.fit.legacy_recipient_matrix/1\n" +_FIELDS = { + "schema_version", + "entity", + "rows", + "columns", + "index_name", + "index_identity", +} + + +@dataclass(frozen=True) +class PreparedRecipientMatrix: + entity: str + features: pd.DataFrame + entity_ids: np.ndarray + + +def _index(ids, name): + if name is not None and not isinstance(name, str): + raise ValueError("Recipient matrix index name must be None or a string.") + if ( + not isinstance(ids, np.ndarray) + or ids.dtype != np.dtype(" bytes: + if not isinstance(features, pd.DataFrame): + raise TypeError("Recipient matrix features must be a DataFrame.") + if not isinstance(entity, str) or not entity or "." in entity: + raise ValueError("Recipient matrix entity must be a nonempty entity name.") + if type(features.index) is not pd.Index or features.index.dtype != np.dtype( + "int64" + ): + raise ValueError( + "Recipient matrix V1 requires an ordinary int64 Index, without casts." + ) + index = _index(entity_ids, features.index.name) + identity = _index_identity(index).to_dict() + if _index_identity(features.index).to_dict() != identity: + raise ValueError("Recipient matrix index and ordered entity IDs differ.") + columns = codec.names(tuple(features.columns), "matrix columns") + if any(dtype != np.dtype(" MATRIX_HEADER_MAX_BYTES: + raise ValueError("Recipient matrix header exceeds its size bound.") + return ( + _MAGIC + + len(header).to_bytes(4, "big") + + header + + entity_ids.tobytes(order="C") + + values.tobytes(order="C") + ) + + +def decode_recipient_matrix(payload: bytes) -> PreparedRecipientMatrix: + if not isinstance(payload, bytes) or not payload.startswith(_MAGIC): + raise ValueError("Invalid recipient matrix envelope.") + offset = len(_MAGIC) + if len(payload) < offset + 4: + raise ValueError("Truncated recipient matrix header length.") + length = int.from_bytes(payload[offset : offset + 4], "big") + start = offset + 4 + if length == 0 or length > MATRIX_HEADER_MAX_BYTES or len(payload) < start + length: + raise ValueError("Recipient matrix header is truncated or exceeds its bound.") + header = codec.decode_json(payload[start : start + length]) + if ( + set(header) != _FIELDS + or type(header["schema_version"]) is not int + or header["schema_version"] != 1 + ): + raise ValueError("Unsupported recipient matrix schema.") + entity, rows = header["entity"], header["rows"] + if ( + not isinstance(entity, str) + or not entity + or "." in entity + or type(rows) is not int + or rows < 1 + ): + raise ValueError("Recipient matrix entity/row count is invalid.") + if not isinstance(header["columns"], list): + raise ValueError("Recipient matrix columns must be an ordered list.") + columns = codec.names(tuple(header["columns"]), "matrix columns") + body = memoryview(payload)[start + length :] + if len(body) != rows * 8 + rows * len(columns) * 8: + raise ValueError("Recipient matrix body length differs from its dimensions.") + ids = np.frombuffer(body[: rows * 8], dtype=" str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _is_sha(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _implementation_binding() -> dict[str, object]: + """Conservative same-implementation, same-platform cache/load boundary.""" + return { + "numeric": Numeric.PLATFORM_BITWISE.value, + "platform": platform_fingerprint(), + "python": sys.version, + "versions": { + package: version(package) + for package in ( + "microcosm-fit", + "microcosm-frame", + "microcosm-graph", + "numpy", + "pandas", + "scipy", + "scikit-learn", + "quantile-forest", + ) + }, + "qrf_sha256": _sha(Path(qrf.__file__).read_bytes()), + "weight_resolution_sha256": _sha(Path(weight_resolution.__file__).read_bytes()), + "target_protocol_sha256": _sha(Path(__file__).read_bytes()), + } + + +@dataclass(frozen=True) +class LegacyQRFTrainingState: + """Immutable donor/config/order/fit-RNG state, with no recipient/draw fields. + + Use ``from_chain`` once at chain initialization; subsequent training targets + consume ``artifact.next_training_state`` and never depend on apply output. + ``to_dict``/``from_dict`` support JSON checkpoints between local workers. + """ + + _canonical_json: str + + def __post_init__(self) -> None: + value = json.loads(self._canonical_json) + if not isinstance(value, dict) or set(value) != _TRAINING_FIELDS: + raise ValueError("Invalid legacy QRF training state fields.") + if _canonical(value) != self._canonical_json: + raise ValueError("Legacy QRF training state must use canonical JSON.") + self._chain() + + def _chain(self) -> qrf.QRFChainState: + # Reuse the authoritative legacy validators. This fixed neutral draw + # state exists only inside validation, never in training serialization + # or identity, and never reaches application. + return qrf.QRFChainState.from_dict( + { + **self.to_dict(), + "recipient_index": None, + "draw_rng_state": np.random.PCG64(0).state, + } + ) + + def to_dict(self) -> dict[str, object]: + return json.loads(self._canonical_json) + + @classmethod + def from_dict(cls, value: Mapping[str, object]) -> LegacyQRFTrainingState: + return cls(_canonical(dict(value))) + + @classmethod + def from_chain(cls, state: qrf.QRFChainState) -> LegacyQRFTrainingState: + if not isinstance(state, qrf.QRFChainState): + raise TypeError("state must be a QRFChainState.") + value = state.to_dict() + del value["recipient_index"] + del value["draw_rng_state"] + return cls.from_dict(value) + + @property + def is_complete(self) -> bool: + return self._chain().is_complete + + @property + def next_target(self) -> str | None: + return self._chain().next_target + + +def _training_id(state, donor_sha256, implementation) -> str: + return _sha( + _canonical( + { + "type": LEGACY_QRF_TARGET_TYPE.name, + "schema_version": LEGACY_QRF_TARGET_TYPE.schema_version, + "training_state": state.to_dict(), + "donor_sha256": donor_sha256, + "implementation": implementation, + } + ).encode() + ) + + +def _consumed_values_sha256(table: pd.DataFrame, columns: tuple[str, ...]) -> str: + """Hash the exact float64 estimator inputs without a whole-table copy.""" + digest = hashlib.sha256(_canonical([list(columns), len(table)]).encode()) + for column in columns: + values = np.ascontiguousarray(table[column].to_numpy(dtype=" str: + target = self.training_state.next_target + if target is None: + raise ValueError("Model artifact training state is already complete.") + return target + + @property + def regime(self) -> str: + return self._target_model.regime + + @property + def platform(self) -> str: + return json.loads(self._implementation_json)["platform"] + + def _validate(self) -> None: + implementation = json.loads(self._implementation_json) + _validate_training_metadata( + self.training_state, + self.next_training_state, + self.donor_sha256, + self.training_id, + implementation, + ) + before = self.training_state._chain() + columns = (*before.predictors, *before.completed_targets) + if ( + type(self._target_model) is not qrf._TargetModel + or self._target_model.columns != columns + or self.regime not in _REGIMES + ): + raise ValueError("Legacy QRF fitted model does not match its metadata.") + + def to_bytes(self) -> bytes: + """Encode a trusted-local versioned envelope without mutating the model.""" + self._validate() + payload = pickle.dumps(self._target_model, protocol=pickle.HIGHEST_PROTOCOL) + metadata = { + "schema_version": 1, + "type": LEGACY_QRF_TARGET_TYPE.name, + "training_state": self.training_state.to_dict(), + "next_training_state": self.next_training_state.to_dict(), + "donor_sha256": self.donor_sha256, + "training_id": self.training_id, + "implementation": json.loads(self._implementation_json), + "target": self.target, + "regime": self.regime, + "pickle_sha256": _sha(payload), + } + header = _canonical(metadata).encode() + return _MAGIC + len(header).to_bytes(8, "big") + header + payload + + @classmethod + def from_trusted_bytes( + cls, data: bytes, *, expected_sha256: str + ) -> LegacyQRFTargetArtifact: + """Load only trusted producer bytes; a hash is not pickle authentication.""" + if not _is_sha(expected_sha256) or _sha(data) != expected_sha256: + raise ValueError("Legacy QRF artifact content digest mismatch.") + offset = len(_MAGIC) + if not data.startswith(_MAGIC) or len(data) < offset + 8: + raise ValueError("Invalid legacy QRF model envelope.") + size = int.from_bytes(data[offset : offset + 8], "big") + offset += 8 + if not 0 < size < len(data) - offset: + raise ValueError("Invalid legacy QRF model header length.") + metadata = json.loads(data[offset : offset + size]) + expected_fields = { + "schema_version", + "type", + "training_state", + "next_training_state", + "donor_sha256", + "training_id", + "implementation", + "target", + "regime", + "pickle_sha256", + } + if ( + not isinstance(metadata, dict) + or set(metadata) != expected_fields + or type(metadata["schema_version"]) is not int + or metadata["schema_version"] != 1 + or metadata["type"] != LEGACY_QRF_TARGET_TYPE.name + ): + raise ValueError("Invalid legacy QRF model metadata/schema version.") + if metadata["implementation"] != _implementation_binding(): + raise ValueError("Legacy QRF artifact implementation/platform mismatch.") + payload = data[offset + size :] + if _sha(payload) != metadata["pickle_sha256"]: + raise ValueError("Legacy QRF model pickle digest mismatch.") + before = LegacyQRFTrainingState.from_dict(metadata["training_state"]) + after = LegacyQRFTrainingState.from_dict(metadata["next_training_state"]) + _validate_training_metadata( + before, + after, + metadata["donor_sha256"], + metadata["training_id"], + metadata["implementation"], + ) + if ( + metadata["target"] != before.next_target + or metadata["regime"] not in _REGIMES + ): + raise ValueError("Legacy QRF artifact training identity mismatch.") + # Integrity, schema, implementation, and platform were checked above. + # This is still arbitrary-code-capable pickle: trust is a caller duty. + model = pickle.loads(payload) # noqa: S301 - trusted local artifacts only + artifact = cls( + before, + after, + metadata["donor_sha256"], + metadata["training_id"], + _canonical(metadata["implementation"]), + model, + ) + artifact._validate() + if artifact.regime != metadata["regime"]: + raise ValueError("Legacy QRF model regime metadata mismatch.") + return artifact + + +def fit_target( + model: qrf.RegimeGatedQRF, + donors: Frame | pd.DataFrame, + *, + state: LegacyQRFTrainingState, + weights: WeightSpec = DESIGN_WEIGHTS, +) -> LegacyQRFTargetArtifact: + """Fit exactly the next legacy target using observed donor prior targets.""" + if not isinstance(state, LegacyQRFTrainingState): + raise TypeError("state must be a LegacyQRFTrainingState.") + before = state._chain() + model._validate_chain_config(before) + if before.is_complete: + raise ValueError("Legacy QRF training chain is complete.") + resolved = qrf._resolve_qrf_fit_input( + donors, list(before.predictors), list(before.targets), weights + ) + model._validate_chain_donor(before, resolved) + columns = (*before.predictors, *before.completed_targets) + target = before.next_target + donor_sha256 = _consumed_values_sha256(resolved.table, (*columns, target)) + implementation = _implementation_binding() + rng = qrf._rng_from_state_json(before.fit_rng_state_json, stream="fit") + fitted = model._fit_target( + features=resolved.table.loc[:, list(columns)].to_numpy(dtype=np.float64), + y=resolved.table[target].to_numpy(dtype=np.float64), + columns=columns, + weights=resolved.weights, + rng=rng, + ) + after = replace( + before, + completed_targets=(*before.completed_targets, target), + fit_rng_state_json=qrf._rng_state_json(rng), + ) + return LegacyQRFTargetArtifact( + state, + LegacyQRFTrainingState.from_chain(after), + donor_sha256, + _training_id(state, donor_sha256, implementation), + _canonical(implementation), + fitted, + ) + + +def apply_target( + artifact: LegacyQRFTargetArtifact, + recipient_predictors: Frame | pd.DataFrame, + raw_prior_draws: pd.DataFrame, + *, + state: qrf.QRFChainState, +) -> qrf.QRFChainStepResult: + """Draw one fitted target, preserving raw conditioning and legacy RNG order.""" + if not isinstance(artifact, LegacyQRFTargetArtifact): + raise TypeError("artifact must be a LegacyQRFTargetArtifact.") + if not isinstance(state, qrf.QRFChainState): + raise TypeError("state must be a QRFChainState.") + if state.is_complete: + raise ValueError("Legacy QRF application chain is complete.") + artifact._validate() + if LegacyQRFTrainingState.from_chain(state) != artifact.training_state: + raise ValueError("Legacy QRF artifact/application training state mismatch.") + # Match fit_draw_next's explicit configured worker-count guard on resume. + if state.fit_n_jobs != qrf._fit_n_jobs(): + raise ValueError("QRF chain model configuration changed: fit worker count.") + recipient = qrf.RegimeGatedQRF._chain_recipient_table(recipient_predictors, state) + identity = qrf._index_identity(recipient.index) + if state.recipient_index is not None and state.recipient_index != identity: + raise ValueError("QRF chain recipient index/order changed since first draw.") + qrf.RegimeGatedQRF._validate_raw_prior_draws( + raw_prior_draws, recipient.index, state + ) + augmented = recipient.loc[:, list(state.predictors)].copy() + for prior in state.completed_targets: + augmented[prior] = raw_prior_draws[prior].to_numpy(copy=False) + rng = qrf._rng_from_state_json(state.draw_rng_state_json, stream="draw") + raw = np.asarray( + qrf._draw_target_with_rng(augmented, artifact._target_model, rng), + dtype=np.float64, + ) + raw.setflags(write=False) + advanced = replace( + state, + completed_targets=(*state.completed_targets, artifact.target), + recipient_index=identity, + fit_rng_state_json=artifact.next_training_state._chain().fit_rng_state_json, + draw_rng_state_json=qrf._rng_state_json(rng), + ) + return qrf.QRFChainStepResult( + target=artifact.target, + raw_draw=raw, + state=advanced, + regime=artifact.regime, + weight_kind=state.weight_kind, + ) diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py b/packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py index d9ce9f2cc..224214d25 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py @@ -1725,9 +1725,7 @@ def _scan_class_attributes( owner_name=owner.name, frame=frame, receiver_entity=self._definitions[owner.name].metadata.entity, - aggregation_entity=( - self._definitions[owner.name].metadata.entity - ), + aggregation_entity=(self._definitions[owner.name].metadata.entity), ) elif ( name == "defined_for" @@ -2427,30 +2425,10 @@ def _dedupe_flows( return list(unique.values()) -def _index_policyengine_us_sources( - variables_root: Path, - *, - parameters_root: Path | None = None, -) -> _PolicyEngineUSSourceIndex: - """Build the shared source index from one parsed module inventory. - - Consumer interpretation is layered on this exact IR; this checkpoint keeps - the metadata surface live while the sequential consumer pass is added. - """ - - if not variables_root.is_dir(): - raise RuntimeError( - "The installed PolicyEngine-US variable source tree is unavailable " - f"at {variables_root}." - ) - modules = _inventory_modules(variables_root) - resolver = _ParameterListResolver( - parameters_root - if parameters_root is not None - else variables_root.parent / "parameters" - ) - module_envs = _module_environments(modules, resolver) - +def _definitions_from_modules( + modules: Mapping[str, _ModuleIR], variables_root: Path +) -> Mapping[str, _SourceVariableDefinition]: + """Shared declaration parser; ownership does not interpret consumer formulas.""" definitions: dict[str, _SourceVariableDefinition] = {} for module_name in sorted(modules): module = modules[module_name] @@ -2470,6 +2448,28 @@ def _index_policyengine_us_sources( raise RuntimeError( f"No PolicyEngine variable classes found below {variables_root}." ) + return MappingProxyType(definitions) + + +def _index_policyengine_us_sources( + variables_root: Path, + *, + parameters_root: Path | None = None, +) -> _PolicyEngineUSSourceIndex: + """Build full metadata and consumer evidence from one parsed module inventory.""" + if not variables_root.is_dir(): + raise RuntimeError( + "The installed PolicyEngine-US variable source tree is unavailable " + f"at {variables_root}." + ) + modules = _inventory_modules(variables_root) + resolver = _ParameterListResolver( + parameters_root + if parameters_root is not None + else variables_root.parent / "parameters" + ) + module_envs = _module_environments(modules, resolver) + definitions = _definitions_from_modules(modules, variables_root) consumers = _ConsumerInterpreter( modules, module_envs, @@ -2477,7 +2477,7 @@ def _index_policyengine_us_sources( definitions, ).run() return _PolicyEngineUSSourceIndex( - definitions=MappingProxyType(definitions), + definitions=definitions, consumers=consumers, ) @@ -2485,9 +2485,13 @@ def _index_policyengine_us_sources( def _index_policyengine_us_variable_sources( variables_root: Path, ) -> Mapping[str, _SourceVariableDefinition]: - """Compatibility metadata view over the one combined source index.""" - - return _index_policyengine_us_sources(variables_root).definitions + """Read exact declaration metadata without parameter-dependent consumer work.""" + if not variables_root.is_dir(): + raise RuntimeError( + "The installed PolicyEngine-US variable source tree is unavailable " + f"at {variables_root}." + ) + return _definitions_from_modules(_inventory_modules(variables_root), variables_root) __all__ = [ diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py index b22113ad2..4731c67ba 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py @@ -38,6 +38,9 @@ from microcosm.frame.adapters._policyengine_us_source_index import ( _index_policyengine_us_sources as _build_policyengine_us_source_index, ) +from microcosm.frame.adapters._policyengine_us_source_index import ( + _index_policyengine_us_variable_sources as _build_policyengine_us_variable_index, +) from microcosm.frame.bundle import Frame from microcosm.frame.materialize import ( engine_tables, @@ -195,9 +198,9 @@ def _index_policyengine_us_sources( def _index_policyengine_us_variable_sources( variables_root: Path, ) -> Mapping[str, _SourceVariableDefinition]: - """Compatibility metadata view over the combined source index.""" + """Shared declaration parser without parameter-dependent consumer work.""" - return _index_policyengine_us_sources(variables_root).definitions + return _build_policyengine_us_variable_index(variables_root) def _index_policyengine_us_generated_variable_sources( @@ -247,8 +250,7 @@ def _index_policyengine_us_generated_variable_sources( return MappingProxyType(definitions) -@lru_cache(maxsize=1) -def _installed_policyengine_us_variable_sources() -> _PolicyEngineUSSourceIndex: +def _installed_policyengine_us_source_parts(): try: package = distribution("policyengine-us") except PackageNotFoundError as exc: @@ -267,11 +269,11 @@ def _installed_policyengine_us_variable_sources() -> _PolicyEngineUSSourceIndex: package_root, version=package.version, ) - source_index = _index_policyengine_us_sources( - variables_root, - parameters_root=package_root / "parameters", - ) - definitions = dict(source_index.definitions) + return package_root, variables_root, generated + + +def _merge_installed_variable_definitions(ordinary, generated): + definitions = dict(ordinary) duplicates = sorted(set(definitions) & set(generated)) if duplicates: raise RuntimeError( @@ -279,8 +281,29 @@ def _installed_policyengine_us_variable_sources() -> _PolicyEngineUSSourceIndex: f"classes: {duplicates}." ) definitions.update(generated) + return MappingProxyType(definitions) + + +@lru_cache(maxsize=1) +def _installed_policyengine_us_variable_definitions(): + """Installed ordinary/generated ownership metadata, with the same audits.""" + _, variables_root, generated = _installed_policyengine_us_source_parts() + return _merge_installed_variable_definitions( + _index_policyengine_us_variable_sources(variables_root), generated + ) + + +@lru_cache(maxsize=1) +def _installed_policyengine_us_variable_sources() -> _PolicyEngineUSSourceIndex: + package_root, variables_root, generated = _installed_policyengine_us_source_parts() + source_index = _index_policyengine_us_sources( + variables_root, + parameters_root=package_root / "parameters", + ) return _PolicyEngineUSSourceIndex( - definitions=MappingProxyType(definitions), + definitions=_merge_installed_variable_definitions( + source_index.definitions, generated + ), consumers=source_index.consumers, ) @@ -297,12 +320,28 @@ class PolicyEngineUSVariableMetadataIndex: an unreviewed wheel version or source change fails closed. """ - def __init__(self) -> None: - source_index = _installed_policyengine_us_variable_sources() - self._definitions = source_index.definitions - self._consumers = source_index.consumers + def __init__(self, *, include_consumers: bool = True) -> None: + if type(include_consumers) is not bool: + raise TypeError("include_consumers must be an explicit boolean.") + if include_consumers: + source_index = _installed_policyengine_us_variable_sources() + self._definitions = source_index.definitions + self._consumers = source_index.consumers + else: + self._definitions = _installed_policyengine_us_variable_definitions() + self._consumers = None self._engine_version = distribution("policyengine-us").version + def _consumer_index(self): + if self._consumers is None: + source_index = _installed_policyengine_us_variable_sources() + if source_index.definitions != self._definitions: + raise RuntimeError( + "PolicyEngine-US definitions changed before consumer indexing." + ) + self._consumers = source_index.consumers + return self._consumers + def variable_metadata(self, name: str) -> VariableMetadata: definition = self._definitions.get(name) if definition is None: @@ -322,7 +361,7 @@ def consumer_receipts(self, name: str) -> tuple[ConsumerReceipt, ...]: if name not in self._definitions: raise ValueError(f"Unknown PolicyEngine-US source variable {name!r}.") - return self._consumers.get(name, ()) + return self._consumer_index().get(name, ()) def variable_dependency_closure(self, name: str) -> VariableDependencyClosure: """Return the statically authenticated transitive graph for ``name``. @@ -338,7 +377,7 @@ def variable_dependency_closure(self, name: str) -> VariableDependencyClosure: raise ValueError(f"Unknown PolicyEngine-US source variable {name!r}.") dependencies: dict[str, set[str]] = {} - for target, receipts in self._consumers.items(): + for target, receipts in self._consumer_index().items(): for receipt in receipts: if receipt.consumer in self._definitions: dependencies.setdefault(receipt.consumer, set()).add(target) diff --git a/packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py b/packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py new file mode 100644 index 000000000..04515bdca --- /dev/null +++ b/packages/microcosm-frame/tests/test_policyengine_us_ownership_index.py @@ -0,0 +1,100 @@ +"""Ownership uses the real declaration parser without consumer parameter I/O.""" + +from pathlib import Path + +import pytest + +from microcosm.frame.adapters import _policyengine_us_source_index as source +from microcosm.frame.adapters import policyengine_us as adapter + + +def _variables(tmp_path): + root = tmp_path / "variables" + root.mkdir() + (root / "example.py").write_text(""" +from policyengine_us.model_api import * + +class earnings(Variable): + value_type = float + entity = Person + definition_period = YEAR + +class deduction(Variable): + value_type = float + entity = Person + definition_period = YEAR + +class computed(Variable): + value_type = float + entity = Person + definition_period = YEAR + adds = "gov.example.components" + +class dated(Variable): + value_type = float + entity = Person + definition_period = YEAR + def formula_2024(person, period, parameters): + return person("earnings", period) +""") + return root + + +def test_same_metadata_parser_without_parameter_reads_then_full_consumer_parity( + tmp_path, monkeypatch +): + variables = _variables(tmp_path) + parameters = tmp_path / "parameters" + path = parameters / "gov/example/components.yaml" + path.parent.mkdir(parents=True) + path.write_text("values:\n 2024-01-01:\n value: [earnings, deduction]\n") + reads = [] + original = Path.read_text + + def observed(self, *args, **kwargs): + reads.append(self) + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", observed) + definitions = source._index_policyengine_us_variable_sources(variables) + assert not any(p.suffix == ".yaml" for p in reads) + assert definitions["computed"].formula_owned + assert definitions["dated"].computed_at(2024) + assert not definitions["dated"].computed_at(2023) + assert not definitions["earnings"].formula_owned + complete = source._index_policyengine_us_sources( + variables, parameters_root=parameters + ) + assert complete.definitions == definitions + assert path in reads + assert any(r.consumer == "computed" for r in complete.consumers["earnings"]) + assert adapter._index_policyengine_us_variable_sources(variables) == definitions + + +def test_unavailable_consumer_parameters_do_not_invalidate_formula_ownership(tmp_path): + variables = _variables(tmp_path) + definitions = source._index_policyengine_us_variable_sources(variables) + assert definitions["computed"].formula_owned + with pytest.raises(RuntimeError, match="Unresolved dynamic"): + source._index_policyengine_us_sources( + variables, parameters_root=tmp_path / "missing" + ) + + +def test_ownership_parser_retains_missing_metadata_and_duplicate_class_failures( + tmp_path, +): + variables = _variables(tmp_path) + (variables / "duplicate.py").write_text(""" +class earnings(Variable): + value_type = float + entity = Person + definition_period = YEAR +""") + with pytest.raises(RuntimeError, match="Duplicate PolicyEngine variable"): + source._index_policyengine_us_variable_sources(variables) + (variables / "duplicate.py").write_text( + "class invalid(Variable):\n entity = Person\n" + ) + with pytest.raises(RuntimeError, match="no static assignment"): + source._index_policyengine_us_variable_sources(variables) diff --git a/packages/microcosm-graph/src/microcosm/graph/artifact_edges.py b/packages/microcosm-graph/src/microcosm/graph/artifact_edges.py new file mode 100644 index 000000000..0b84849e5 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/artifact_edges.py @@ -0,0 +1,164 @@ +"""Typed byte edges: declarations, numeric refusal, and portable descriptors.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from .decl import ArtifactType, CompiledGraph, Node +from .errors import NodeRejectedError +from .kernel import ( + ArtifactValue, + Capabilities, + KernelRegistry, + Numeric, + NumericScope, + Tolerance, +) +from .keys import opaque_artifact_key, platform_fingerprint + + +def numeric_scope(capabilities: Capabilities) -> NumericScope: + return NumericScope( + numeric=capabilities.numeric, + tolerance=capabilities.tolerance, + platform=platform_fingerprint() + if capabilities.numeric is Numeric.PLATFORM_BITWISE + else None, + ) + + +def scope_payload(scope: NumericScope) -> dict[str, object]: + return { + "numeric": scope.numeric.value, + "tolerance": None + if scope.tolerance is None + else { + "rtol": scope.tolerance.rtol, + "atol": scope.tolerance.atol, + "ulps": scope.tolerance.ulps, + }, + "platform": scope.platform, + } + + +def scope_from_payload(raw: object) -> NumericScope: + if not isinstance(raw, Mapping) or set(raw) != {"numeric", "tolerance", "platform"}: + raise ValueError("Malformed artifact numeric scope.") + tolerance = raw["tolerance"] + if tolerance is not None: + if not isinstance(tolerance, Mapping) or set(tolerance) != { + "rtol", + "atol", + "ulps", + }: + raise ValueError("Malformed artifact tolerance.") + tolerance = Tolerance(**tolerance) + return NumericScope( + Numeric(raw["numeric"]), tolerance=tolerance, platform=raw["platform"] + ) + + +def require_compatible_scope(scope: NumericScope, consumer: Capabilities) -> None: + """Refuse scope laundering; output tolerance is the consumer's own contract.""" + if scope.platform is not None and scope.numeric is Numeric.TOLERANCE_BOUND: + raise NodeRejectedError( + "Typed artifacts combining platform and tolerance scopes are unsupported." + ) + if ( + scope.numeric is Numeric.PLATFORM_BITWISE + and consumer.numeric is not Numeric.PLATFORM_BITWISE + ): + raise NodeRejectedError( + "A platform_bitwise artifact requires a platform_bitwise consumer." + ) + if ( + scope.numeric is Numeric.TOLERANCE_BOUND + and consumer.numeric is not Numeric.TOLERANCE_BOUND + ): + raise NodeRejectedError( + "A tolerance_bound artifact requires a tolerance_bound consumer." + ) + + +def descriptor( + *, + producer: str, + artifact: str, + type_: ArtifactType, + producer_key: str, + capabilities: Capabilities, +) -> dict[str, object]: + return { + "producer": producer, + "artifact": artifact, + "producer_key": producer_key, + "key": opaque_artifact_key(producer_key, artifact), + "type": {"name": type_.name, "schema_version": type_.schema_version}, + "numerics": scope_payload(numeric_scope(capabilities)), + } + + +def typed_contracts( + compiled: CompiledGraph, + node: Node, + keys: Mapping[str, str], + kernels: KernelRegistry, +) -> dict[str, object]: + if not node.artifact_inputs and not node.artifact_outputs: + return {} + consumer = kernels.get(node.kernel).capabilities + inputs = {} + for binding in node.artifact_inputs: + producer = compiled.graph.node(binding.producer) + capabilities = kernels.get(producer.kernel).capabilities + require_compatible_scope(numeric_scope(capabilities), consumer) + inputs[binding.name] = descriptor( + producer=producer.id, + artifact=binding.artifact, + type_=binding.type, + producer_key=keys[producer.id], + capabilities=capabilities, + ) + return { + "inputs": inputs, + "outputs": { + output.name: descriptor( + producer=node.id, + artifact=output.name, + type_=output.type, + producer_key=keys[node.id], + capabilities=consumer, + ) + for output in node.artifact_outputs + }, + } + + +def value_from_descriptor(payload: bytes, raw: object) -> ArtifactValue: + if not isinstance(raw, Mapping) or set(raw) != { + "producer", + "artifact", + "producer_key", + "key", + "type", + "numerics", + }: + raise ValueError("Malformed typed artifact descriptor.") + if any( + not isinstance(raw[name], str) or not raw[name] + for name in ("producer", "artifact") + ): + raise ValueError("Typed artifact producer/name must be nonempty strings.") + type_raw = raw["type"] + if not isinstance(type_raw, Mapping) or set(type_raw) != {"name", "schema_version"}: + raise ValueError("Malformed typed artifact type.") + value = ArtifactValue( + payload, + ArtifactType(type_raw["name"], type_raw["schema_version"]), + raw["key"], + raw["producer_key"], + scope_from_payload(raw["numerics"]), + ) + if value.key != opaque_artifact_key(value.producer_key, raw["artifact"]): + raise ValueError("Typed artifact identity does not match its producer.") + return value diff --git a/packages/microcosm-graph/src/microcosm/graph/attachments.py b/packages/microcosm-graph/src/microcosm/graph/attachments.py new file mode 100644 index 000000000..885e23607 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/attachments.py @@ -0,0 +1,239 @@ +"""Private, non-portable population attachments and execution lifetimes. + +These store references never enter node records or portable manifest identity. +""" + +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import RLock +from types import MappingProxyType +from weakref import WeakValueDictionary + +from microcosm.frame import Frame + +from .canonical import canonical_json, sha256_domain +from .decl import CompiledGraph, StructuralDelta +from .manifest import NodeReceipt, PopulationView +from .population import MassRecord, Population +from .store import ( + _FRAME_FORMAT, + ContentStore, + StoreCorrupt, + _payload_table, + _write_frame, +) + + +@dataclass(frozen=True) +class _StoredPopulation: + key: str + node_key: str + metadata: Mapping[str, object] + payload_hash: str + + +class _LazyPopulations(Mapping[str, PopulationView]): + """Immutable attachment index, with caller-lifetime weak view memoization.""" + + def __init__( + self, store: ContentStore, references: Mapping[str, _StoredPopulation] + ): + if any( + not isinstance(k, str) or type(v) is not _StoredPopulation + for k, v in references.items() + ): + raise TypeError("Lazy populations require validated stored references.") + self._store = store + self._references = MappingProxyType(dict(references)) + self._cache: WeakValueDictionary[str, PopulationView] = WeakValueDictionary() + self._lock = RLock() + + def __iter__(self) -> Iterator[str]: + return iter(self._references) + + def __contains__(self, version: object) -> bool: + return version in self._references + + def __len__(self) -> int: + return len(self._references) + + def __getitem__(self, version: str) -> PopulationView: + with self._lock: + view = self._cache.get(version) + if view is None: + reference = self._references[version] + metadata = self._store.metadata(reference.key, kind="frame") + if _payload_hash(metadata["payloads"]) != reference.payload_hash: + raise StoreCorrupt( + "Stored population attachment changed after execution." + ) + frame = self._store.load_frame( + reference.key, node_key=reference.node_key + ) + # A valid same-key replacement can occur after either the + # check above or load_frame's own metadata verification. Bind + # the actual decoded content before exposing or memoizing it. + if _decoded_payload_hash(self._store, frame) != reference.payload_hash: + raise StoreCorrupt( + "The decoded population attachment changed after execution." + ) + view = PopulationView(frame) + # The existing Frame store codec omits metadata. Retain the + # original Frame's already-frozen scalar/container descriptor, + # which cannot own tables/arrays, for this live manifest only. + object.__setattr__(view, "_metadata", reference.metadata) + self._cache[version] = view + return view + + +class _PopulationRetention: + """Retire realized versions after their final scheduled population use.""" + + def __init__( + self, compiled: CompiledGraph, store: ContentStore, *, verify_existing: bool + ): + self._compiled = compiled + self._store = store + self._verify_existing = verify_existing + last_use: dict[str, str] = {} + self._versions: list[str] = [] + for node_id in compiled.order: + node = compiled.graph.node(node_id) + if node.structural is not StructuralDelta.NONE: + self._versions.append(node_id) + last_use[node_id] = node_id + if node.structural is StructuralDelta.NONE: + last_use[compiled.versions[node_id]] = node_id + elif node.base is not None: + last_use[node.base] = node_id + self._retire_at: dict[str, list[str]] = {} + for version, node_id in last_use.items(): + self._retire_at.setdefault(node_id, []).append(version) + self._references: dict[str, _StoredPopulation] = {} + self._ledgers: dict[str, tuple[MassRecord, ...]] = {} + self._patches: dict[str, list[NodeReceipt]] = {} + self._force_snapshot: set[str] = set() + + def finish( + self, + node_id: str, + populations: dict[str, Population], + receipts: Mapping[str, NodeReceipt], + *, + structural_collision: bool = False, + ) -> None: + node = self._compiled.graph.node(node_id) + receipt = receipts[node_id] + if structural_collision: + self._force_snapshot.add(node_id) + if node.structural is StructuralDelta.NONE and ( + receipt.artifacts or receipt.weight_key + ): + version = self._compiled.versions[node_id] + self._patches.setdefault(version, []).append(receipt) + for version in self._retire_at.get(node_id, ()): + population = populations.pop(version, None) + if population is None: # Unreached structural descendants have no Frame. + continue + structural = receipts[version] + key = structural.frame_key + assert key is not None + patches = self._patches.pop(version, ()) + # Derive the binding from the actual live Frame even when its + # structural artifact seems reusable. Otherwise a concurrent + # replacement could become the expected signature at retirement. + key, payload_hash = _snapshot( + self._store, + population, + structural, + patches, + verify_existing=self._verify_existing, + reuse_structural=not patches and version not in self._force_snapshot, + ) + self._force_snapshot.discard(version) + self._references[version] = _StoredPopulation( + key, structural.key, population.frame.metadata, payload_hash + ) + self._ledgers[version] = population.mass_ledger + + def views(self) -> _LazyPopulations: + return _LazyPopulations( + self._store, + {v: self._references[v] for v in self._versions if v in self._references}, + ) + + def ledgers(self) -> Mapping[str, tuple[MassRecord, ...]]: + return {v: self._ledgers[v] for v in self._versions if v in self._ledgers} + + +def _payload_hash(payloads: object) -> str: + return sha256_domain("population-attachment-payloads/1", canonical_json(payloads)) + + +def _decoded_payload_hash(store: ContentStore, frame: Frame) -> str: + """Hash the actual decoded Frame, using its canonical codec representation. + + Temporary writes are bounded by one Frame and use the same per-column + codec as snapshots. Reading metadata again would leave the same race. + """ + + with TemporaryDirectory( + prefix="population-read-check-", dir=store.tmp + ) as directory: + staging = Path(directory) + _write_frame(staging, frame) + return _payload_hash(_payload_table(staging)) + + +def _snapshot( + store: ContentStore, + population: Population, + structural: NodeReceipt, + patches: Sequence[NodeReceipt], + *, + verify_existing: bool, + reuse_structural: bool = False, +) -> tuple[str, str]: + """Publish one codec snapshot bound to both provenance and actual bytes. + + A tolerance-bound recomputation can emit different bytes under the same + node key. Receipt-only attachment identities would silently reuse the + previous Frame. Stage once, hash bounded payload files, then rename those + files into the store's normal atomic publication path without copying. + """ + + population.frame.revalidate() + with TemporaryDirectory( + prefix="population-attachment-", dir=store.tmp + ) as directory: + staging = Path(directory) + _write_frame(staging, population.frame) + payload_hash = _payload_hash(_payload_table(staging)) + if reuse_structural: + assert structural.frame_key is not None + metadata = store.metadata(structural.frame_key, kind="frame") + if ( + metadata.get("node_key") == structural.key + and _payload_hash(metadata["payloads"]) == payload_hash + ): + return structural.frame_key, payload_hash + key = sha256_domain( + "population-attachment/1", + canonical_json( + { + "structural": structural._content_payload(), + "patches": [patch._content_payload() for patch in patches], + "payload_hash": payload_hash, + } + ), + ) + + def publish(destination: Path) -> Mapping[str, object]: + for child in staging.iterdir(): + child.rename(destination / child.name) + return {"frame_format": _FRAME_FORMAT, "node_key": structural.key} + + store._put(key, "frame", publish, verify_existing=verify_existing) + return key, payload_hash diff --git a/packages/microcosm-graph/src/microcosm/graph/availability.py b/packages/microcosm-graph/src/microcosm/graph/availability.py new file mode 100644 index 000000000..e89b1d90f --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/availability.py @@ -0,0 +1,92 @@ +"""Executor-owned exceptional outcomes, with no fabricated data products.""" + +from collections.abc import Mapping + +from .decl import StructuralDelta +from .kernel import Capabilities, KernelRole + +EXECUTION_SCHEMA = "microcosm.graph.execution.v1" + + +def has_execution(receipt: Mapping[str, object]) -> bool: + """Leave pre-existing free-form execution diagnostics uninterpreted.""" + execution = receipt.get("execution") + return ( + isinstance(execution, Mapping) and execution.get("schema") == EXECUTION_SCHEMA + ) + + +def execution_state(receipt: Mapping[str, object]) -> str | None: + execution = receipt.get("execution") + state = execution.get("state") if has_execution(receipt) else None + return state if isinstance(state, str) else None + + +def unavailable_artifacts( + receipt: Mapping[str, object], outputs: Mapping[str, object] +) -> frozenset[str]: + return ( + frozenset(outputs) + if execution_state(receipt) in {"gate_exception", "unreached"} + else frozenset() + ) + + +def validate_execution( + receipt: Mapping[str, object], + capabilities: Capabilities, + outputs: Mapping[str, object], + opaque: Mapping[str, object], + *, + has_products: bool, +) -> None: + """Validate exceptional receipt shape at cache and manifest boundaries.""" + if not has_execution(receipt): + return + execution = receipt["execution"] + if not isinstance(execution, Mapping): + raise ValueError("Executor execution metadata must be a mapping.") + state = execution.get("state") + if state == "gate_exception": + evidence = receipt.get("evidence") + missing = execution.get("unavailable_artifacts") + if ( + set(execution) != {"schema", "state", "unavailable_artifacts"} + or capabilities.role is not KernelRole.GATE + or capabilities.structural is not StructuralDelta.NONE + or receipt.get("outcome") != "fail" + or not outputs + or not isinstance(missing, list | tuple) + or tuple(missing) != tuple(sorted(outputs)) + or opaque + or not isinstance(evidence, Mapping) + or not isinstance(evidence.get("exception_type"), str) + or not isinstance(evidence.get("message"), str) + ): + raise ValueError("Invalid gate exception or unavailable artifact evidence.") + elif state == "unreached": + blockers = execution.get("blocked_by") + if ( + set(execution) != {"schema", "state", "blocked_by"} + or receipt.get("outcome") != "unreached" + or not isinstance(blockers, Mapping) + or not blockers + or any( + not isinstance(name, str) + or not name + or not isinstance(key, str) + or len(key) != 64 + or any(char not in "0123456789abcdef" for char in key) + for name, key in blockers.items() + ) + or has_products + or opaque + ): + raise ValueError("Invalid unreached blocker evidence or invented products.") + if ( + capabilities.role is KernelRole.RELEASE + and receipt.get("tier") != "evidence" + ): + raise ValueError("An unreached release must remain evidence-tier.") + else: + raise ValueError(f"Unknown executor execution state {state!r}.") diff --git a/packages/microcosm-graph/src/microcosm/graph/randomness.py b/packages/microcosm-graph/src/microcosm/graph/randomness.py new file mode 100644 index 000000000..5a5510bc2 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/randomness.py @@ -0,0 +1,69 @@ +"""Stable random coordinates, independent of graph packing and cache identity. + +A stream is ("sha256-u53-v1", experiment_id, replicate, base_seed). Each +nonempty coordinate tuple identifies a draw, conventionally (person_id, +process, period, draw_index). The top 53 bits of the SHA-256 digest, interpreted +big-endian, divided by 2**53 define a uniform in [0, 1). Stream parameters must +be normative node params; kernels must hash this module as implementing source +and declare SeedSource.KEYED. Repeated coordinates intentionally repeat draws. +""" + +from __future__ import annotations + +import hashlib +import math +from collections.abc import Sequence + +import numpy as np + +from .canonical import canonical_json + +__all__ = ["keyed_uniform"] + + +def _coordinate(value: object) -> list[object]: + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, bool): + return ["bool", value] + if isinstance(value, int): + return ["int", value] + if isinstance(value, str): + return ["str", value] + if isinstance(value, float) and math.isfinite(value): + return ["float", value] + raise TypeError("Random coordinates must be non-null finite scalar identities.") + + +def keyed_uniform(*, stream: tuple, keys: Sequence[tuple]) -> np.ndarray: + """Return bytes-backed, read-only float64 draws keyed by stable coordinates. + + Row order, chunk boundaries, and unrelated inserted identities cannot affect + a draw. Integers, strings, booleans and floats have distinct canonical tags. + The experiment name is nonempty; replicate and base seed are non-negative + Python integers (booleans refused). No numpy RNG state is read or mutated. + """ + if not isinstance(stream, tuple) or len(stream) != 4: + raise TypeError( + "stream must be (algorithm, experiment_id, replicate, base_seed)." + ) + algorithm, experiment, replicate, base_seed = stream + if algorithm != "sha256-u53-v1": + raise ValueError(f"Unsupported random stream algorithm {algorithm!r}.") + if not isinstance(experiment, str) or not experiment: + raise ValueError("Random stream experiment_id must be non-empty.") + if any(type(value) is not int or value < 0 for value in (replicate, base_seed)): + raise ValueError( + "Random stream replicate/base_seed must be non-negative integers." + ) + prefix = b"microcosm-graph/keyed-uniform/1\0" + canonical_json(stream) + b"\0" + values = [] + for key in keys: + if not isinstance(key, tuple) or not key: + raise TypeError("Each random key must be a nonempty coordinate tuple.") + encoded = canonical_json([_coordinate(value) for value in key]) + digest = hashlib.sha256(prefix + encoded).digest() + values.append((int.from_bytes(digest[:8], "big") >> 11) / 2**53) + return np.frombuffer( + np.asarray(values, dtype=np.float64).tobytes(), dtype=np.float64 + ) diff --git a/packages/microcosm-graph/src/microcosm/graph/schema.py b/packages/microcosm-graph/src/microcosm/graph/schema.py new file mode 100644 index 000000000..56a802eb5 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/schema.py @@ -0,0 +1,316 @@ +"""Bounded, value-only inspection of a compiled graph and its visible fields. + +This is declaration metadata, not execution evidence or source authority. A +population's schema includes its ordinary derived fields as well as its carried +fields; it is not a snapshot of a Frame halfway through execution. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import fields +from types import MappingProxyType + +from .decl import ( + ROWS_ALL, + ArtifactInput, + ArtifactOutput, + ArtifactType, + CompiledGraph, + Graph, + GraphError, + Node, + Owned, + Ownership, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, +) +from .serialize import graph_from_json, graph_to_json + +__all__ = ["graph_schema"] + +PROTOCOL = "microcosm.graph.schema.v1" +# Inspector development limits; none describe microdata size or release scope. +MAX_NODES = 256 +MAX_SOURCES = 256 +MAX_DEPTH = 64 +MAX_DECLARED_FIELDS = 20_000 +MAX_ROWS = 100_000 +MAX_DECLARATION_BYTES = 8 * 1024 * 1024 +MAX_EXPORT_BYTES = 32 * 1024 * 1024 +MAX_ITEMS = 200_000 +MAX_STRING_CHARS = 16_384 +_DECLARATIONS = frozenset( + { + Graph, + Node, + SourceRef, + Slice, + Owned, + WeightTransition, + ArtifactInput, + ArtifactOutput, + ArtifactType, + } +) + + +def _require(condition: bool, detail: str) -> None: + if not condition: + raise GraphError(f"Graph schema: {detail}.") + + +def _preflight(value: object) -> None: + """Bound existing declaration objects before serializer/decoder allocation. + + The conservative byte charge allows six JSON bytes per string character. + This bounds metadata construction, not total Python process memory. + """ + remaining, items = MAX_DECLARATION_BYTES, 0 + + def walk(child: object, depth: int) -> None: + nonlocal remaining, items + items += 1 + _require(items <= MAX_ITEMS and depth <= MAX_DEPTH, "declaration complexity") + kind = type(child) + cost = 16 + if kind is str: + _require(len(child) <= MAX_STRING_CHARS, "string length") + cost = 6 * len(child) + 2 + elif child is None or kind is bool: + pass + elif kind is int: + _require(child.bit_length() <= 1024, "integer size") + cost = child.bit_length() + 2 + elif kind is float: + _require(math.isfinite(child), "finite number required") + cost = 32 + elif kind in {Ownership, StructuralDelta}: + walk(child.value, depth + 1) + elif kind in _DECLARATIONS: + for field in fields(child): + walk(field.name, depth + 1) + walk(getattr(child, field.name), depth + 1) + elif kind in {tuple, list}: + _require(len(child) <= MAX_ITEMS - items, "container size") + for part in child: + walk(part, depth + 1) + elif kind in {dict, MappingProxyType}: + _require(len(child) <= (MAX_ITEMS - items) // 2, "mapping size") + for key, part in child.items(): + walk(key, depth + 1) + walk(part, depth + 1) + else: + raise GraphError(f"Graph schema: unsupported declaration {kind.__name__}.") + remaining -= cost + _require(remaining >= 0, "prospective declaration size") + + walk(value, 0) + + +def _structure(compiled: CompiledGraph) -> dict[str, object]: + return { + "order": list(compiled.order), + "versions": dict(sorted(compiled.versions.items())), + "predecessors": { + key: list(value) for key, value in sorted(compiled.predecessors.items()) + }, + "owners": [ + list(key) + [owner] for key, owner in sorted(compiled.owners.items()) + ], + } + + +def graph_schema(compiled: CompiledGraph) -> dict[str, object]: + """Export the actual declaration, checked compilation and visible fields. + + ``producer`` is the local declared owner, otherwise the structural carrier + of the requested population. ``declared_in`` traces the nearest actual + Owned declaration; its rows/ownership/rewrite describe that declaration, + not a claim about the current row membership or scientific availability. + + ``input_bindings`` covers explicit slices, named slice/output masks and + implicit rewrite incumbents. It does not enumerate runtime-injected IDs, + memberships, resolved weights/strata or parameter-driven EXPAND cells. + Typed artifact dependencies remain in graph and compiled predecessors. + """ + _require(type(compiled) is CompiledGraph, "actual CompiledGraph required") + _require(type(compiled.graph) is Graph, "actual Graph required") + graph = compiled.graph + _require(len(graph.nodes) <= MAX_NODES, "node limit") + _require(len(graph.sources) <= MAX_SOURCES, "source limit") + _require( + sum(len(node.outputs) for node in graph.nodes) <= MAX_DECLARED_FIELDS, + "declared field limit", + ) + _preflight(graph) + _preflight( + (compiled.order, compiled.versions, compiled.predecessors, compiled.owners) + ) + raw = graph_to_json(graph).encode("utf-8") + _require(len(raw) <= MAX_DECLARATION_BYTES, "declaration size") + # Reuse the real declaration parser/compiler. A caller-constructed or stale + # CompiledGraph must not supply fictitious owners or edges to the inspector. + detached = graph_from_json(raw.decode("utf-8")) + try: + checked = compile_graph(detached) + except RecursionError: + raise GraphError("Graph schema: compilation depth.") from None + structure = _structure(checked) + _require(_structure(compiled) == structure, "compiled structure mismatch") + by_id = {node.id: node for node in detached.nodes} + declarations = { + (node.id, output.entity, output.column): output + for node in detached.nodes + for output in node.outputs + } + depth: dict[str, int] = {} + for node_id in checked.order: + depth[node_id] = 1 + max( + (depth[parent] for parent in checked.predecessors[node_id]), default=0 + ) + _require(depth[node_id] <= MAX_DEPTH, "graph depth") + + # Each map holds the nearest actual declaration, including derived fields + # anywhere in that version. Structural descendants carry the completed map. + visible: dict[str, dict[tuple[str, str], tuple[str, Owned]]] = {} + rows: list[dict[str, object]] = [] + bindings: list[dict[str, object]] = [] + used = len(raw) + encoder = json.JSONEncoder( + ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + + def append(target: list[dict[str, object]], row: dict[str, object]) -> None: + nonlocal used + _require(len(rows) + len(bindings) < MAX_ROWS, "expanded row limit") + size = sum(len(part.encode("utf-8")) for part in encoder.iterencode(row)) + _require(size <= MAX_EXPORT_BYTES - used, "prospective export size") + used += size + target.append(row) + + for version in checked.order: + holder = by_id[version] + if holder.structural is StructuralDelta.NONE: + continue + current = {} if holder.base is None else dict(visible[holder.base]) + local = { + (entity, column): owner + for (population, entity, column), owner in checked.owners.items() + if population == version + } + # Count before copying an expanded version into exported row objects. + _require( + len(rows) + len(current.keys() | local.keys()) <= MAX_ROWS, + "expanded schema limit", + ) + for coordinate, owner in local.items(): + declaration = declarations[owner, *coordinate] + current[coordinate] = (owner, declaration) + visible[version] = current + for (entity, column), (declared_in, declaration) in sorted(current.items()): + append( + rows, + { + "population": version, + "entity": entity, + "column": column, + "dtype": declaration.dtype, + "producer": local.get((entity, column), version), + "declared_in": declared_in, + "rows": declaration.rows, + "ownership": declaration.ownership.value, + "rewrite": declaration.rewrite, + }, + ) + + def binding( + node: Node, version: str, entity: str, column: str, row_mask: str, kind: str + ) -> None: + coordinate = (entity, column) + owner = checked.owners.get((version, entity, column)) + declaration_version = version + if owner == node.id: + # Rewrites receive the current structural carrier's incumbent, + # potentially materialized by EXPAND, not the output being written. + declaration_version = by_id[version].base + _require(declaration_version is not None, "rewrite base required") + owner = None + _require(coordinate in visible[declaration_version], "input field missing") + declared_in, _declaration = visible[declaration_version][coordinate] + append( + bindings, + { + "node": node.id, + "population": version, + "entity": entity, + "column": column, + "rows": row_mask, + "producer": version if owner is None else owner, + "declared_in": declared_in, + "kind": kind, + }, + ) + + for node_id in checked.order: + node = by_id[node_id] + if node.structural is StructuralDelta.CREATE: + continue + version = ( + checked.versions[node_id] + if node.structural is StructuralDelta.NONE + else node.base + ) + for slice_ in node.inputs: + for column in slice_.columns: + binding(node, version, slice_.entity, column, slice_.rows, "slice") + if slice_.rows != ROWS_ALL: + binding( + node, version, slice_.entity, slice_.rows, ROWS_ALL, "slice_mask" + ) + for output in node.outputs: + if output.rewrite: + binding( + node, + version, + output.entity, + output.column, + output.rows, + "rewrite_incumbent", + ) + if output.rows != ROWS_ALL: + binding( + node, version, output.entity, output.rows, ROWS_ALL, "output_mask" + ) + bindings.sort( + key=lambda row: tuple( + str(row[key]) for key in ("node", "kind", "entity", "column", "rows") + ) + ) + result = { + "protocol": PROTOCOL, + "country": detached.country, + "graph_sha256": hashlib.sha256(raw).hexdigest(), + "graph": json.loads(raw), + "compiled": structure, + "schema": rows, + "input_bindings": bindings, + } + # Stream the final size accounting; do not build another unbounded JSON + # string. All variable-sized graph/row construction was bounded above. + total = 0 + for part in encoder.iterencode(result): + total += len(part.encode("utf-8")) + _require(total <= MAX_EXPORT_BYTES, "export size") + _preflight(compiled.graph) + _require( + graph_to_json(compiled.graph).encode("utf-8") == raw + and _structure(compiled) == structure, + "input changed during export", + ) + return result From 69b94b4a5fddbead983a8316ba3c269381b89063 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 004/255] Stage integration layer 02: GRAPH-RESTORE.patch --- PROGRESS.md | 2 + .../src/microcosm/graph/__init__.py | 14 + .../src/microcosm/graph/codecs.py | 254 ++++++++++- .../src/microcosm/graph/decl.py | 93 ++++ .../src/microcosm/graph/executor.py | 419 ++++++++++++++++-- .../src/microcosm/graph/explain.py | 5 + .../src/microcosm/graph/kernel.py | 53 ++- .../src/microcosm/graph/keys.py | 69 ++- .../src/microcosm/graph/manifest.py | 210 ++++++++- .../src/microcosm/graph/serialize.py | 74 ++++ .../src/microcosm/graph/view.py | 16 + 11 files changed, 1141 insertions(+), 68 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index aeb510209..d9ac95e35 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -739,3 +739,5 @@ Verified requested clean branch, base HEAD, main ancestry and preservation pins. Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index aaa781484..bbcbc5ea7 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -16,6 +16,9 @@ PARTITION_DTYPES, ROWS_ALL, WEIGHT_KINDS, + ArtifactInput, + ArtifactOutput, + ArtifactType, CompiledGraph, Graph, GraphError, @@ -37,6 +40,7 @@ StoreUnavailableError, ) from .kernel import ( + ArtifactValue, Capabilities, Determinism, Kernel, @@ -52,8 +56,14 @@ source_hash, ) from .keys import platform_fingerprint +from .randomness import keyed_uniform __all__ = [ + "ArtifactInput", + "ArtifactOutput", + "ArtifactType", + "ArtifactValue", + "keyed_uniform", "platform_fingerprint", "DESCRIPTIVE_FIELDS", "DTYPES", @@ -95,6 +105,7 @@ "NumericScope", "Tolerance", "Slice", + "SourceBytesCodec", "SourceCodec", "SourceCodecRegistry", "SourceRef", @@ -113,6 +124,7 @@ "graph_from_json", "graph_to_json", "load_source", + "load_source_bytes", "run_graph", "source_hash", ] @@ -136,9 +148,11 @@ def _check_frame_version() -> None: from .codecs import ( # noqa: E402 - check dependency series before runtime import SOURCE_CODECS, + SourceBytesCodec, SourceCodec, SourceCodecRegistry, load_source, + load_source_bytes, ) from .executor import NodeRejected, run_graph # noqa: E402 from .explain import explain_html # noqa: E402 diff --git a/packages/microcosm-graph/src/microcosm/graph/codecs.py b/packages/microcosm-graph/src/microcosm/graph/codecs.py index 4cf4bd3e4..3ebc9d8e9 100644 --- a/packages/microcosm-graph/src/microcosm/graph/codecs.py +++ b/packages/microcosm-graph/src/microcosm/graph/codecs.py @@ -1,12 +1,32 @@ -"""Source codecs: the sole boundary from source bytes to :class:`Frame`. - -Two codecs ship with the graph runtime: - -``frame-store`` +"""Source codecs: the boundary from source bytes to a decoded input. + +A codec is registered in exactly one of two explicit modes, and a name +belongs to at most one mode: + +*Frame mode* (:data:`SourceCodec`, :meth:`SourceCodecRegistry.register`, +:meth:`SourceCodecRegistry.load`) decodes a source into a population +:class:`Frame`. A ``CREATE`` node's kernel calls it to build a population, +and so does any other node whose source really is one — a held-out reference +frame read for scoring, say. + +*Raw-byte mode* (:data:`SourceBytesCodec`, +:meth:`SourceCodecRegistry.register_bytes`, +:meth:`SourceCodecRegistry.load_bytes`) decodes a source into immutable +``bytes``. A lookup table (an NPZ of ratios, a CSV crosswalk) is not a +population, and an import kernel that turns one into a typed +:class:`~microcosm.graph.decl.ArtifactOutput` must be able to read its real +bytes without a Frame codec registered as a pretence. Neither mode can be +loaded through the other: the mismatch is a :class:`TypeError` naming the +mode the codec actually has, so no caller receives bytes where it declared a +Frame. + +Three codecs ship with the graph runtime: + +``frame-store`` (Frame) Loads a content-verified Frame from the path of a ``ContentStore`` frame object directory. -``csv-tables`` +``csv-tables`` (Frame) Loads one CSV per entity using ``schema.json``. The schema may give a ``tables`` mapping (otherwise ``.csv`` is used), global or per-entity dtype mappings, a ``strata_column``, and either (a) a weight @@ -14,11 +34,19 @@ A JSON weight entry is ``{"kind": "design", "values": [...]}`` or ``{"kind": "design", "column": "household_weight"}``; entries are keyed by entity, or may carry their own ``entity`` field. + +``raw-bytes-v1`` (raw bytes) + Reads one regular file, bounded at :data:`RAW_BYTES_MAX_BYTES`. It + interprets nothing: the consuming kernel owns the payload's format and + validates it. Identity, the pre-run content key, and the post-run + mutation check stay where they already are, in the executor. """ from __future__ import annotations import json +import os +import stat from collections.abc import Callable, Mapping from pathlib import Path from types import MappingProxyType @@ -32,39 +60,97 @@ from .store import ContentStore, StoreUnavailable __all__ = [ + "RAW_BYTES_MAX_BYTES", "SOURCE_CODECS", + "SourceBytesCodec", "SourceCodec", "SourceCodecRegistry", "load_csv_tables", "load_frame_store", + "load_raw_bytes", "load_source", + "load_source_bytes", ] type SourceCodec = Callable[..., Frame] +type SourceBytesCodec = Callable[..., bytes] + +RAW_BYTES_MAX_BYTES = 64 * 1024 * 1024 +"""The most ``raw-bytes-v1`` will read from one source file (64 MiB).""" class SourceCodecRegistry: - """Named source-to-Frame loaders.""" + """Named source loaders in two explicit modes: Frame and raw bytes. + + A name is registered in one mode only. :meth:`get` answers the + availability question the executor asks before any kernel runs, in + either mode; :meth:`load` and :meth:`load_bytes` are what hold a codec + to the mode it was registered in. + """ def __init__(self) -> None: self._loaders: dict[str, SourceCodec] = {} + self._byte_loaders: dict[str, SourceBytesCodec] = {} - def register(self, name: str, loader: SourceCodec) -> SourceCodec: - """Register and return ``loader`` under a non-empty codec name.""" - + @staticmethod + def _check_declaration(name: str, loader: object) -> None: if not isinstance(name, str) or not name: raise ValueError("Source codec names must be non-empty strings.") if not callable(loader): raise TypeError("Source codec loaders must be callable.") + + def register(self, name: str, loader: SourceCodec) -> SourceCodec: + """Register and return a Frame ``loader`` under a non-empty codec name.""" + + self._check_declaration(name, loader) + if name in self._byte_loaders: + raise ValueError( + f"Source codec {name!r} is already registered as a raw-bytes codec." + ) incumbent = self._loaders.get(name) if incumbent is not None and incumbent is not loader: raise ValueError(f"Source codec {name!r} is already registered.") self._loaders[name] = loader return loader - def get(self, name: str) -> SourceCodec: - """Resolve ``name`` or raise fatal :class:`StoreUnavailable`.""" + def register_bytes(self, name: str, loader: SourceBytesCodec) -> SourceBytesCodec: + """Register and return a raw-byte ``loader`` under a non-empty codec name. + + A raw-byte codec returns the source's bytes and claims nothing about + their meaning; it never stands in for a population. + """ + + self._check_declaration(name, loader) + if name in self._loaders: + raise ValueError( + f"Source codec {name!r} is already registered as a Frame codec." + ) + incumbent = self._byte_loaders.get(name) + if incumbent is not None and incumbent is not loader: + raise ValueError(f"Source codec {name!r} is already registered.") + self._byte_loaders[name] = loader + return loader + + def get(self, name: str) -> SourceCodec | SourceBytesCodec: + """Resolve ``name`` in either mode, or raise fatal :class:`StoreUnavailable`. + + This is availability, not decoding: a registered raw-byte codec is + installed, and resolving it here never turns it into a Frame codec. + """ + + for loaders in (self._loaders, self._byte_loaders): + try: + return loaders[name] + except KeyError: + continue + raise StoreUnavailable(f"Source codec {name!r} is not installed.") + def _frame_loader(self, name: str) -> SourceCodec: + if name in self._byte_loaders: + raise TypeError( + f"Source codec {name!r} is a raw-bytes codec; read it with " + "load_bytes. Bytes are never presented as a Frame." + ) try: return self._loaders[name] except KeyError as error: @@ -72,6 +158,19 @@ def get(self, name: str) -> SourceCodec: f"Source codec {name!r} is not installed." ) from error + def _bytes_loader(self, name: str) -> SourceBytesCodec: + if name in self._loaders: + raise TypeError( + f"Source codec {name!r} is a Frame codec; read it with load. " + "A Frame is never presented as raw bytes." + ) + try: + return self._byte_loaders[name] + except KeyError as error: + raise StoreUnavailable( + f"Source codec {name!r} is not installed." + ) from error + def load( self, name: str, @@ -79,9 +178,9 @@ def load( *, store: ContentStore | None = None, ) -> Frame: - """Decode ``path`` with ``name`` and require a Frame result.""" + """Decode ``path`` with the Frame codec ``name`` and require a Frame.""" - loader = self.get(name) + loader = self._frame_loader(name) try: frame = loader(Path(path), store=store) except StoreUnavailable: @@ -96,16 +195,66 @@ def load( ) return frame + def load_bytes( + self, + name: str, + path: Path, + *, + store: ContentStore | None = None, + ) -> bytes: + """Read ``path`` with the raw-byte codec ``name`` and require bytes. + + This decodes an external source path, unlike + :meth:`ContentStore.load_bytes`, which reads a stored object back by + its content key. + + Unavailability keeps the classification :meth:`load` gives it: an + uninstalled codec or a missing dependency is fatal + :class:`StoreUnavailable`, never a decoded value. + """ + + loader = self._bytes_loader(name) + try: + payload = loader(Path(path), store=store) + except StoreUnavailable: + raise + except ImportError as error: + raise StoreUnavailable( + f"Source codec {name!r} needs an unavailable dependency." + ) from error + if not isinstance(payload, bytes): + raise TypeError( + f"Source codec {name!r} returned {type(payload).__name__}, not bytes." + ) + return payload + def names(self) -> tuple[str, ...]: - """Registered names in canonical order.""" + """The registered Frame codec names, in canonical order. + + Each mode is enumerated by its own pair — ``names``/``as_mapping`` + here, :meth:`bytes_names`/:meth:`as_bytes_mapping` there — so a + snapshot taken through either pair stays resolvable through it. The + name space is still shared: a name in neither tuple is not therefore + free, because registering it in one mode reserves it in both. + """ return tuple(sorted(self._loaders)) + def bytes_names(self) -> tuple[str, ...]: + """The registered raw-byte codec names, in canonical order.""" + + return tuple(sorted(self._byte_loaders)) + def as_mapping(self) -> Mapping[str, SourceCodec]: - """A read-only snapshot of registered loaders.""" + """A read-only snapshot of the registered Frame loaders.""" return MappingProxyType(dict(self._loaders)) + def as_bytes_mapping(self) -> Mapping[str, SourceBytesCodec]: + """A read-only snapshot of the registered raw-byte loaders.""" + + return MappingProxyType(dict(self._byte_loaders)) + def load_frame_store(path: Path, *, store: ContentStore | None = None) -> Frame: """Load a verified frame object from its content-store directory.""" @@ -372,9 +521,68 @@ def load_csv_tables(path: Path, *, store: ContentStore | None = None) -> Frame: return Frame(tables, schema, weights, strata) +def _nonblocking_opener(path: str, flags: int) -> int: + """Open without blocking, so a FIFO in a source's place cannot hang a run.""" + + return os.open(path, flags | getattr(os, "O_NONBLOCK", 0)) + + +def load_raw_bytes(path: Path, *, store: ContentStore | None = None) -> bytes: + """Read one regular file's bytes, bounded at :data:`RAW_BYTES_MAX_BYTES`. + + The codec claims nothing about the payload: a lookup NPZ, a crosswalk + CSV, and a corrupt file are all just bytes here, and the kernel that + imports them validates their format. What this function does own is the + refusal to read something that is not one bounded regular file. + + Symlinks are followed, as the executor's ``resolve(strict=True)`` and + ``source_content_key`` already do. A directory, a FIFO, a device, or a + socket is refused: the mode is read from the descriptor this function + itself opened, and the open is non-blocking, so the codec cannot be made + to wait on a pipe or stream a device. The read is bounded rather than + trusted to ``st_size``, because a file may grow after it is measured. + Detecting that a source moved is the executor's post-run content check; + this bound only keeps the codec from reading an unbounded amount first. + """ + + del store # raw bytes are self-describing; no content store is consulted + source = Path(path) + try: + handle = open(source, "rb", opener=_nonblocking_opener) + except IsADirectoryError as error: + raise ValueError( + f"Raw source at {source} is a directory; raw-bytes-v1 reads one " + "regular file." + ) from error + except OSError as error: + raise ValueError(f"Raw source at {source} is not readable: {error}") from error + with handle: + # A directory never reaches here: opening one raises IsADirectoryError + # above. Everything else that is not a regular file is refused from the + # descriptor's own mode, not from a second look at the path. + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + raise ValueError( + f"Raw source at {source} is not a regular file; raw-bytes-v1 " + "reads one regular file." + ) + try: + payload = handle.read(RAW_BYTES_MAX_BYTES + 1) + except OSError as error: + raise ValueError( + f"Raw source at {source} is not readable: {error}" + ) from error + if len(payload) > RAW_BYTES_MAX_BYTES: + raise ValueError( + f"Raw source at {source} is larger than the " + f"{RAW_BYTES_MAX_BYTES}-byte raw-bytes-v1 limit." + ) + return payload + + SOURCE_CODECS = SourceCodecRegistry() SOURCE_CODECS.register("frame-store", load_frame_store) SOURCE_CODECS.register("csv-tables", load_csv_tables) +SOURCE_CODECS.register_bytes("raw-bytes-v1", load_raw_bytes) def load_source( @@ -384,6 +592,18 @@ def load_source( store: ContentStore | None = None, registry: SourceCodecRegistry = SOURCE_CODECS, ) -> Frame: - """Decode one source through the selected registry.""" + """Decode one source into a Frame through the selected registry.""" return registry.load(codec, path, store=store) + + +def load_source_bytes( + codec: str, + path: Path, + *, + store: ContentStore | None = None, + registry: SourceCodecRegistry = SOURCE_CODECS, +) -> bytes: + """Read one source's bytes through the selected registry.""" + + return registry.load_bytes(codec, path, store=store) diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index b25e4f621..b6134c210 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -48,6 +48,9 @@ from types import MappingProxyType __all__ = [ + "ArtifactType", + "ArtifactInput", + "ArtifactOutput", "DESCRIPTIVE_FIELDS", "DTYPES", "GATE_OUTCOMES", @@ -175,6 +178,48 @@ def __post_init__(self) -> None: _nonempty("SourceRef.codec", self.codec) +@dataclass(frozen=True) +class ArtifactType: + """Nominal, versioned byte-payload contract; consumers validate the payload.""" + + name: str + schema_version: int + + def __post_init__(self) -> None: + _nonempty("ArtifactType.name", self.name) + if type(self.schema_version) is not int or self.schema_version < 1: + raise GraphError("ArtifactType.schema_version must be a positive integer.") + + +@dataclass(frozen=True) +class ArtifactOutput: + """A named, typed subset of a kernel's existing opaque byte outputs.""" + + name: str + type: ArtifactType + + def __post_init__(self) -> None: + _nonempty("ArtifactOutput.name", self.name) + if not isinstance(self.type, ArtifactType): + raise GraphError("ArtifactOutput.type must be an ArtifactType.") + + +@dataclass(frozen=True) +class ArtifactInput: + """A declared artifact edge, with a consumer-local alias and exact type.""" + + name: str + producer: str + artifact: str + type: ArtifactType + + def __post_init__(self) -> None: + for field_name in ("name", "producer", "artifact"): + _nonempty(f"ArtifactInput.{field_name}", getattr(self, field_name)) + if not isinstance(self.type, ArtifactType): + raise GraphError("ArtifactInput.type must be an ArtifactType.") + + @dataclass(frozen=True) class Slice: """What a node reads: columns of one entity, optionally under a row mask. @@ -279,6 +324,11 @@ class Node: Attributes: id: Unique within the graph. kernel: Kernel reference, e.g. ``"fit.qrf@1"``. + artifact_inputs: Typed byte dependencies, including other populations. + Each local alias names a declared producer output of exactly the + expected nominal type/version. + artifact_outputs: Required typed outputs within KernelResult.artifacts; + other untyped diagnostic bytes remain legal. inputs: Slices the kernel receives. Their owners are this node's predecessors. outputs: Cells this node owns. A ``CREATE`` node declares every @@ -318,10 +368,25 @@ class Node: description: str = "" citation: str = "" entrants: bool = False + artifact_inputs: tuple[ArtifactInput, ...] = () + artifact_outputs: tuple[ArtifactOutput, ...] = () def __post_init__(self) -> None: _nonempty("Node.id", self.id) _nonempty("Node.kernel", self.kernel) + for name, kind in ( + ("artifact_inputs", ArtifactInput), + ("artifact_outputs", ArtifactOutput), + ): + declarations = getattr(self, name) + if not isinstance(declarations, tuple) or any( + not isinstance(item, kind) for item in declarations + ): + raise GraphError( + f"Node {self.id!r}: {name} must be a tuple of {kind.__name__}." + ) + if len({item.name for item in declarations}) != len(declarations): + raise GraphError(f"Node {self.id!r}: duplicate names in {name}.") if not isinstance(self.structural, StructuralDelta): raise GraphError(f"Node {self.id!r}: structural must be a StructuralDelta.") if self.mass not in MASS_POLICIES: @@ -417,6 +482,10 @@ def normative(self) -> dict[str, object]: f.name: getattr(self, f.name) for f in fields(self) if f.name not in DESCRIPTIVE_FIELDS + and not ( + f.name in {"artifact_inputs", "artifact_outputs"} + and not getattr(self, f.name) + ) } @@ -678,6 +747,30 @@ def check_mask(node_id: str, version: str, entity: str, mask: str) -> None: f"but the incumbent is declared {base_dtype!r}." ) + # Artifact edges cross population versions, without changing cell ownership. + for node in graph.nodes: + for binding in node.artifact_inputs: + if binding.producer == node.id: + raise GraphError( + f"Node {node.id!r} depends on itself through an artifact." + ) + producer = by_id.get(binding.producer) + if producer is None: + raise GraphError( + f"Node {node.id!r}: unknown artifact producer {binding.producer!r}." + ) + outputs = {output.name: output for output in producer.artifact_outputs} + output = outputs.get(binding.artifact) + if output is None: + raise GraphError( + f"Node {node.id!r}: producer {producer.id!r} has no declared artifact {binding.artifact!r}." + ) + if output.type != binding.type: + raise GraphError( + f"Node {node.id!r}: artifact {binding.artifact!r} type does not match its producer." + ) + predecessors[node.id].add(producer.id) + depth: dict[str, int] = {} def depth_of(node_id: str, trail: tuple[str, ...]) -> int: diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 689c1cf9a..acb0a77a9 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -6,7 +6,7 @@ import json import socket import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from datetime import UTC, datetime from pathlib import Path from types import MappingProxyType @@ -17,6 +17,15 @@ from microcosm.frame import Frame, WeightKind, Weights from . import keys as graph_keys +from .artifact_edges import scope_payload, typed_contracts, value_from_descriptor +from .attachments import _PopulationRetention +from .availability import ( + EXECUTION_SCHEMA, + execution_state, + has_execution, + unavailable_artifacts, + validate_execution, +) from .canonical import canonical_json, sha256_domain from .codecs import SOURCE_CODECS, SourceCodecRegistry from .decl import ( @@ -30,6 +39,7 @@ ) from .errors import NodeRejectedError from .kernel import ( + ArtifactValue, Capabilities, KernelContext, KernelRegistry, @@ -134,6 +144,19 @@ def _failed_gate_result( columns=MappingProxyType(columns), receipt={ "outcome": "fail", + **( + { + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "gate_exception", + "unavailable_artifacts": sorted( + output.name for output in node.artifact_outputs + ), + } + } + if node.artifact_outputs + else {} + ), "evidence": { "exception_type": type(error).__name__, "message": str(error), @@ -386,6 +409,21 @@ def _context_digest(context: KernelContext) -> bytes: digest.update(weights.kind.value.encode("ascii") + b"\0") _update_array(digest, weights.values) _update_series(digest, context.strata) + for name, value in sorted(context.artifacts.items()): + digest.update( + canonical_json( + ( + name, + value.key, + value.producer_key, + value.type.name, + value.type.schema_version, + scope_payload(value.numerics), + ) + ) + ) + digest.update(len(value.payload).to_bytes(8, "little")) + digest.update(value.payload) return digest.digest() @@ -461,6 +499,7 @@ def _project_context( sources: Mapping[str, Path], tolerances: Mapping[tuple[str, str], Tolerance | None], numerics: Mapping[tuple[str, str], NumericScope], + artifacts: Mapping[str, ArtifactValue] | None = None, ) -> KernelContext: if population is None: return KernelContext( @@ -473,6 +512,7 @@ def _project_context( sources=MappingProxyType({name: sources[name] for name in node.sources}), tolerances=tolerances, numerics=numerics, + artifacts={} if artifacts is None else artifacts, ) frame = population.frame @@ -564,6 +604,7 @@ def _project_context( sources=MappingProxyType({name: sources[name] for name in node.sources}), tolerances=tolerances, numerics=numerics, + artifacts={} if artifacts is None else artifacts, ) @@ -1126,7 +1167,30 @@ def _validate_result( if not isinstance(payload, bytes): raise NodeRejected(f"Node {node.id!r} artifact {name!r} is not bytes.") artifacts[name] = payload + for output in node.artifact_outputs: + if ( + output.name not in artifacts + and execution_state(result.receipt) != "gate_exception" + ): + error = StoreMiss if cache_hit else NodeRejected + raise error( + f"Node {node.id!r} is missing declared artifact {output.name!r}." + ) receipt = _normal_json_mapping(result.receipt, f"Node {node.id!r} receipt") + try: + validate_execution( + receipt, + kernel_capabilities, + {output.name: output for output in node.artifact_outputs}, + artifacts, + has_products=bool( + result.columns or result.frame is not None or result.weights is not None + ), + ) + except ValueError as error: + raise NodeRejected( + f"Node {node.id!r} execution evidence rejected: {error}" + ) from error if node.structural is StructuralDelta.EXPAND: if cache_hit: if not isinstance(receipt.get("expand"), dict): @@ -1378,21 +1442,27 @@ def _write_node( receipt: Mapping[str, object], opaque_artifacts: Mapping[str, bytes], verify_existing: bool, + typed_artifacts: Mapping[str, object] | None = None, ) -> tuple[dict[tuple[str, str], str], dict[str, object]]: - columns: dict[tuple[str, str], tuple[pd.Series, str]] = {} if node.structural is StructuralDelta.NONE: declared = {(owned.entity, owned.column): owned for owned in node.outputs} - for coordinate, series in result.columns.items(): - columns[coordinate] = (series, declared[coordinate].dtype) + coordinates = sorted(result.columns) else: - for entity in population.frame.entities: - for column in population.frame.table(entity).columns: - series = _series_for_column(population.frame, entity, column) - columns[(entity, column)] = (series, _dtype_token(series)) + coordinates = sorted( + (entity, column) + for entity in population.frame.entities + for column in population.frame.table(entity).columns + ) column_entries: list[dict[str, str]] = [] manifest_artifacts: dict[tuple[str, str], str] = {} - for (entity, column), (series, token) in sorted(columns.items()): + for entity, column in coordinates: + if node.structural is StructuralDelta.NONE: + series = result.columns[(entity, column)] + token = declared[(entity, column)].dtype + else: + series = _series_for_column(population.frame, entity, column) + token = _dtype_token(series) output_key = artifact_key(key, entity, column) store.put_column( output_key, @@ -1404,6 +1474,7 @@ def _write_node( ) column_entries.append({"entity": entity, "column": column, "key": output_key}) manifest_artifacts[(entity, column)] = output_key + del series stored_frame_key: str | None = None if node.structural is not StructuralDelta.NONE: @@ -1462,7 +1533,12 @@ def _write_node( opaque_entries.append({"name": name, "key": output_key}) record: dict[str, object] = { - "schema_version": 1, + "schema_version": 3 + if execution_state(receipt) + else 2 + if typed_artifacts + else 1, + **({"typed_artifacts": dict(typed_artifacts)} if typed_artifacts else {}), "node_id": node.id, "node_key": key, "kernel_ref": node.kernel, @@ -1490,6 +1566,7 @@ def _require_record_shape( key: str, kernel_impl_hash: str, capabilities: Capabilities, + typed_artifacts: Mapping[str, object] | None = None, ) -> dict[str, object]: if not isinstance(raw, dict): raise StoreCorrupt(f"Cached receipt for node {node.id!r} is not an object.") @@ -1506,16 +1583,62 @@ def _require_record_shape( "weight", "opaque", } + if typed_artifacts: + required.add("typed_artifacts") if set(raw) != required: raise StoreCorrupt( f"Cached receipt for node {node.id!r} has fields {sorted(raw)}, " f"not {sorted(required)}." ) - if raw["schema_version"] != 1: + raw_receipt = raw["receipt"] + if not isinstance(raw_receipt, Mapping): + raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + exceptional = execution_state(raw_receipt) + if raw["schema_version"] != (3 if exceptional else 2 if typed_artifacts else 1): raise StoreUnavailable( f"Cached receipt for node {node.id!r} uses unsupported schema " f"{raw['schema_version']!r}." ) + if typed_artifacts and raw.get("typed_artifacts") != dict(typed_artifacts): + raise StoreCorrupt( + f"Cached node {node.id!r} typed artifact contracts disagree with the graph." + ) + try: + validate_execution( + raw_receipt, + capabilities, + (typed_artifacts or {}).get("outputs", {}), + { + entry.get("name"): entry.get("key") + for entry in _record_entries(raw, "opaque") + }, + has_products=bool( + raw["columns"] + or raw["frame_key"] is not None + or raw["weight"] is not None + ), + ) + except ValueError as error: + raise StoreCorrupt( + f"Cached node {node.id!r} execution evidence rejected: {error}" + ) from error + if typed_artifacts: + opaque = _record_entries(raw, "opaque") + names = [entry.get("name") for entry in opaque] + if len(set(names)) != len(names): + raise StoreCorrupt(f"Cached node {node.id!r} repeats an opaque artifact.") + actual_outputs = {entry.get("name"): entry.get("key") for entry in opaque} + for output in node.artifact_outputs: + if exceptional: + continue + if output.name not in actual_outputs: + raise StoreMiss( + f"Cached node {node.id!r} is missing declared artifact {output.name!r}." + ) + if actual_outputs[output.name] != _opaque_artifact_key(key, output.name): + raise StoreCorrupt( + f"Cached node {node.id!r} artifact identity mismatch." + ) expected = (node.id, key, node.kernel, kernel_impl_hash) actual = ( raw["node_id"], @@ -1543,7 +1666,7 @@ def _require_record_shape( f"Cached receipt capabilities for node {node.id!r} disagree with " "the registered kernel contract." ) - if node.structural is StructuralDelta.EXPAND: + if node.structural is StructuralDelta.EXPAND and exceptional != "unreached": raw_receipt = raw["receipt"] if not isinstance(raw_receipt, Mapping): raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") @@ -1581,6 +1704,7 @@ def _load_record( key: str, kernel_impl_hash: str, capabilities: Capabilities, + typed_artifacts: Mapping[str, object] | None = None, ) -> dict[str, object]: raw = store.load_json(_cache_record_key(key)) return _require_record_shape( @@ -1589,6 +1713,7 @@ def _load_record( key=key, kernel_impl_hash=kernel_impl_hash, capabilities=capabilities, + typed_artifacts=typed_artifacts, ) @@ -1850,6 +1975,115 @@ def _all_node_keys( return keys, implementations +def _blocked_by( + compiled: CompiledGraph, + node: Node, + keys: Mapping[str, str], + receipts: Mapping[str, Mapping[str, object]], +) -> dict[str, str]: + """Unavailable byte inputs block consumers; unreached causal parents propagate.""" + blocked = { + parent: keys[parent] + for parent in compiled.predecessors[node.id] + if execution_state(receipts.get(parent, {})) == "unreached" + } + for binding in node.artifact_inputs: + producer = compiled.graph.node(binding.producer) + if binding.artifact in unavailable_artifacts( + receipts.get(binding.producer, {}), + {output.name: output for output in producer.artifact_outputs}, + ): + blocked[binding.producer] = keys[binding.producer] + return dict(sorted(blocked.items())) + + +def _unreached_node( + compiled: CompiledGraph, + node: Node, + *, + blockers: Mapping[str, str], + receipts: Mapping[str, NodeReceipt], + store: ContentStore, + key: str, + implementation: str, + capabilities: Capabilities, + typed: Mapping[str, object], + resume: ResumePolicy, +) -> NodeReceipt: + """Cache a proven lack of inputs, without running or fabricating products.""" + receipt: dict[str, object] = { + "outcome": "unreached", + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": dict(blockers), + }, + "evidence": {"reason": "Required graph inputs are unavailable."}, + "capabilities": _capabilities_projection(capabilities), + } + if capabilities.role is KernelRole.RELEASE: + tier, gate_ids = _release_tier(compiled, node.id, receipts) + if tier != "evidence": + raise NodeRejected("An unavailable release has no failing gate ancestor.") + receipt.update( + tier=tier, + gate_ancestry=list(gate_ids), + requires_decisions=list(_required_decision_names(node)), + ) + hit = False + replace_stale_record = False + if resume != "forbid": + try: + record = _load_record( + store, + node, + key=key, + kernel_impl_hash=implementation, + capabilities=capabilities, + typed_artifacts=typed, + ) + if record["receipt"] != receipt: + raise StoreCorrupt( + f"Cached node {node.id!r} blocked provenance disagrees with its inputs." + ) + hit = True + except StoreMiss: + replace_stale_record = store.has(_cache_record_key(key)) + if resume == "require": + raise + if not hit: + record = { + "schema_version": 3, + **({"typed_artifacts": dict(typed)} if typed else {}), + "node_id": node.id, + "node_key": key, + "kernel_ref": node.kernel, + "kernel_impl_hash": implementation, + "capabilities": _capabilities_projection(capabilities), + "receipt": receipt, + "columns": [], + "frame_key": None, + "weight": None, + "opaque": [], + } + store.put_json( + _cache_record_key(key), + record, + node_key=key, + verify_existing=resume != "forbid" and not replace_stale_record, + ) + return NodeReceipt( + key=key, + hit=hit, + seed=seed(key), + kernel_ref=node.kernel, + kernel_impl_hash=implementation, + capabilities=capabilities, + receipt=receipt, + typed_artifacts=typed, + ) + + def _preflight_require( compiled: CompiledGraph, store: ContentStore, @@ -1858,6 +2092,7 @@ def _preflight_require( kernels: KernelRegistry, ) -> None: missing: list[str] = [] + receipts: dict[str, Mapping[str, object]] = {} for node_id in compiled.order: node = compiled.graph.node(node_id) try: @@ -1867,14 +2102,35 @@ def _preflight_require( key=keys[node_id], kernel_impl_hash=implementations[node_id], capabilities=kernels.get(node.kernel).capabilities, + typed_artifacts=typed_contracts(compiled, node, keys, kernels), ) - _require_tolerance_writer_receipt( - node, - record, - _input_writers(compiled, node_id), - exact=False, - ) + if any(parent not in receipts for parent in compiled.predecessors[node_id]): + missing.append(node_id) + continue + blockers = _blocked_by(compiled, node, keys, receipts) + expected_execution = { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": blockers, + } + if blockers: + if record["receipt"].get("execution") != expected_execution: + raise StoreCorrupt( + f"Cached node {node_id!r} blocked provenance disagrees with its inputs." + ) + elif execution_state(record["receipt"]) == "unreached": + raise StoreCorrupt( + f"Cached node {node_id!r} has no unavailable input blocker." + ) + else: + _require_tolerance_writer_receipt( + node, + record, + _input_writers(compiled, node_id), + exact=False, + ) _preflight_record(store, record) + receipts[node_id] = record["receipt"] except StoreMiss: missing.append(node_id) if missing: @@ -1907,11 +2163,28 @@ def run_graph( kernels: KernelRegistry, resume: ResumePolicy = "auto", decisions: tuple[Decision, ...] = (), + population_retention: str = "eager", + _population_observer: Callable[[str, Population], None] | None = None, ) -> RunManifest: - """Execute a compiled graph with content-addressed reuse and receipts.""" + """Execute a compiled graph with content-addressed reuse and receipts. + + ``population_retention="lazy"`` retires each population after its final + scheduled use and loads historical manifest views from verified storage. + It requires a writable store, including for missing auxiliary attachments + under ``resume="require"``. The default ``"eager"`` retains all Frames. + Node identities and portable manifest provenance are unchanged. + + The private population observer exposes the actual admitted population, + including its original design anchors, to integrating verifiers. It runs + for both cold execution and restored cache hits, before persistence and + retention. It must not mutate the population; raising refuses the run. + It is never a kernel capability or part of a cached kernel's execution. + """ if resume not in ("auto", "require", "forbid"): raise ValueError("resume must be 'auto', 'require', or 'forbid'.") + if population_retention not in ("eager", "lazy"): + raise ValueError("population_retention must be 'eager' or 'lazy'.") normalized_decisions: list[Decision] = [] for decision in decisions: if isinstance(decision, Decision): @@ -1926,11 +2199,21 @@ def run_graph( started_at = _now() source_paths, source_keys = _source_paths_and_keys(compiled, sources, store) keys, implementations = _all_node_keys(compiled, kernels, source_keys) + contracts = { + node_id: typed_contracts(compiled, compiled.graph.node(node_id), keys, kernels) + for node_id in compiled.order + } if resume == "require": _preflight_require(compiled, store, keys, implementations, kernels) + retention = ( + _PopulationRetention(compiled, store, verify_existing=resume != "forbid") + if population_retention == "lazy" + else None + ) populations: dict[str, Population] = {} receipts: dict[str, NodeReceipt] = {} + receipt_payloads: dict[str, Mapping[str, object]] = {} for node_id in compiled.order: node_started = time.perf_counter() node = compiled.graph.node(node_id) @@ -1943,6 +2226,30 @@ def run_graph( "capabilities." ) + blockers = _blocked_by( + compiled, + node, + keys, + receipt_payloads, + ) + if blockers: + receipts[node_id] = _unreached_node( + compiled, + node, + blockers=blockers, + receipts=receipts, + store=store, + key=key, + implementation=implementation, + capabilities=kernel.capabilities, + typed=contracts[node_id], + resume=resume, + ) + receipt_payloads[node_id] = receipts[node_id].receipt + if retention is not None: + retention.finish(node_id, populations, receipts) + continue + if node.structural is StructuralDelta.CREATE: incumbent: Population | None = None elif node.structural is StructuralDelta.NONE: @@ -1965,6 +2272,18 @@ def run_graph( ) tolerance_writers = _tolerance_writer_payload(input_writers) + typed = contracts[node_id] + artifact_values = {} + for binding in node.artifact_inputs: + entry = typed["inputs"][binding.name] + producer_receipt = receipts[binding.producer] + if producer_receipt.opaque_artifacts.get(binding.artifact) != entry["key"]: + raise StoreCorrupt( + f"Node {node.id!r} artifact producer receipt disagrees with its declaration." + ) + artifact_values[binding.name] = value_from_descriptor( + store.load_bytes(entry["key"]), entry + ) hit = False replace_stale_record = False result: KernelResult | None = None @@ -1978,7 +2297,12 @@ def run_graph( key=key, kernel_impl_hash=implementation, capabilities=kernel.capabilities, + typed_artifacts=typed, ) + if execution_state(record["receipt"]) == "unreached": + raise StoreCorrupt( + f"Cached node {node_id!r} has no unavailable input blocker." + ) try: _require_tolerance_writer_receipt( node, record, input_writers, exact=True @@ -2005,6 +2329,7 @@ def run_graph( sources=source_paths, tolerances=input_tolerances, numerics=input_numerics, + artifacts=artifact_values, ) before = _context_digest(context) try: @@ -2018,6 +2343,17 @@ def run_graph( raise NodeRejected( f"Node {node.id!r} kernel {node.kernel!r} failed: {error}" ) from error + else: + # Invalid returned metadata is a contract rejection, not an + # exception raised while a gate was computing its evidence. + if ( + isinstance(result, KernelResult) + and isinstance(result.receipt, Mapping) + and has_execution(result.receipt) + ): + raise NodeRejected( + "Kernel receipts may not author executor execution metadata." + ) after = _context_digest(context) if before != after: raise NodeRejected(f"Node {node.id!r} mutated its input context.") @@ -2137,6 +2473,16 @@ def run_graph( else: populations[node.id] = updated + if _population_observer is not None: + _population_observer(node_id, updated) + + structural_collision = ( + retention is not None + and not hit + and resume != "forbid" + and node.structural is not StructuralDelta.NONE + and store.has(frame_key(key)) + ) if not hit: manifest_artifacts, record = _write_node( store, @@ -2149,6 +2495,7 @@ def run_graph( receipt=cache_receipt, opaque_artifacts=opaque, verify_existing=(resume != "forbid" and not replace_stale_record), + typed_artifacts=typed, ) assert record is not None @@ -2172,6 +2519,7 @@ def run_graph( receipt_opaque[name] = artifact_identity receipts[node_id] = NodeReceipt( + typed_artifacts=typed, key=key, hit=hit, seed=seed(key), @@ -2185,6 +2533,14 @@ def run_graph( weight_key=receipt_weight_key, opaque_artifacts=MappingProxyType(receipt_opaque), ) + receipt_payloads[node_id] = receipts[node_id].receipt + if retention is not None: + retention.finish( + node_id, + populations, + receipts, + structural_collision=structural_collision, + ) return RunManifest( country=compiled.graph.country, @@ -2193,13 +2549,24 @@ def run_graph( started_at=started_at, finished_at=_now(), host=socket.gethostname(), - populations=MappingProxyType( - {version: population.frame for version, population in populations.items()} + populations=( + retention.views() + if retention is not None + else MappingProxyType( + { + version: population.frame + for version, population in populations.items() + } + ) ), - mass_ledgers=MappingProxyType( - { - version: population.mass_ledger - for version, population in populations.items() - } + mass_ledgers=( + retention.ledgers() + if retention is not None + else MappingProxyType( + { + version: population.mass_ledger + for version, population in populations.items() + } + ) ), ) diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index 9d10d8be6..26abfdbd4 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -506,6 +506,11 @@ def _receipt_payload(receipt: NodeReceipt) -> dict[str, object]: "frame_key": receipt.frame_key, "weight_key": receipt.weight_key, "opaque_artifacts": receipt.opaque_artifacts, + **( + {"typed_artifacts": receipt.typed_artifacts} + if receipt.typed_artifacts + else {} + ), "wall_time": receipt.wall_time, } if receipt.legacy_capabilities: diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index 567a57ea5..0846a3eaa 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -59,9 +59,10 @@ from microcosm.frame import Frame, Weights -from .decl import Node, Param, StructuralDelta +from .decl import ArtifactType, Node, Param, StructuralDelta __all__ = [ + "ArtifactValue", "Capabilities", "Determinism", "Kernel", @@ -151,6 +152,7 @@ class SeedSource(StrEnum): EXECUTOR = "executor" # ``KernelContext.rng``, derived from the node key PARAM = "param" # a literal ``seed`` parameter (legacy parity kernels) + KEYED = "keyed" # normative stream params and stable draw coordinates NONE = "none" @@ -270,6 +272,33 @@ def __post_init__(self) -> None: raise ValueError("A bitwise scope holds on every platform.") +@dataclass(frozen=True) +class ArtifactValue: + """Verified immutable bytes and the executor's typed producer provenance.""" + + payload: bytes + type: ArtifactType + key: str + producer_key: str + numerics: NumericScope + + def __post_init__(self) -> None: + if not isinstance(self.payload, bytes): + raise TypeError("ArtifactValue.payload must be immutable bytes.") + if not isinstance(self.type, ArtifactType) or not isinstance( + self.numerics, NumericScope + ): + raise TypeError("ArtifactValue requires an ArtifactType and NumericScope.") + for name in ("key", "producer_key"): + value = getattr(self, name) + if ( + not isinstance(value, str) + or len(value) != 64 + or any(c not in "0123456789abcdef" for c in value) + ): + raise ValueError(f"ArtifactValue.{name} must be a SHA-256 identity.") + + @dataclass(frozen=True) class KernelContext: """Everything a kernel may read. Nothing here is writable. @@ -286,8 +315,12 @@ class KernelContext: in the node's inputs or outputs. strata: Read-only per-person strata of the population version. params: The node's parameters. - rng: A generator seeded from the node key. The only randomness a - kernel may use. + rng: The default generator seeded from the node key. KEYED kernels + instead use normative stream params and stable coordinates through + keyed_uniform; PARAM kernels use their declared literal seed. + artifacts: Immutable typed bytes for declared artifact aliases only. + Consumers validate versioned payloads before using them; nominal + types do not themselves verify arbitrary serialized data. sources: Source name to a content-verified path, for declared sources only. tolerances: ``(entity, column)`` of each declared input column to @@ -309,6 +342,20 @@ class KernelContext: sources: Mapping[str, Path] = field(default_factory=dict) tolerances: Mapping[tuple[str, str], Tolerance | None] = field(default_factory=dict) numerics: Mapping[tuple[str, str], NumericScope] = field(default_factory=dict) + artifacts: Mapping[str, ArtifactValue] = field(default_factory=dict) + + def __post_init__(self) -> None: + values = dict(self.artifacts) + if any( + not isinstance(name, str) + or not name + or not isinstance(value, ArtifactValue) + for name, value in values.items() + ): + raise TypeError( + "KernelContext.artifacts must map non-empty aliases to ArtifactValue." + ) + object.__setattr__(self, "artifacts", MappingProxyType(values)) @dataclass(frozen=True) diff --git a/packages/microcosm-graph/src/microcosm/graph/keys.py b/packages/microcosm-graph/src/microcosm/graph/keys.py index 43939dbae..3844ec56e 100644 --- a/packages/microcosm-graph/src/microcosm/graph/keys.py +++ b/packages/microcosm-graph/src/microcosm/graph/keys.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import os import platform as _platform import sys from collections.abc import Mapping @@ -13,6 +14,7 @@ from .kernel import Capabilities, Numeric __all__ = [ + "opaque_artifact_key", "platform_fingerprint", "artifact_key", "frame_key", @@ -27,6 +29,35 @@ def _hash_parts(domain: str, *parts: object) -> str: return sha256_domain(domain, canonical_json(parts)) +def _stream_file(path: Path, digest: object, *, prefix_length: bool = False) -> int: + """Append bounded byte chunks and refuse a detected concurrent change.""" + + def identity(stat: os.stat_result) -> tuple[int, ...]: + return ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + + size = 0 + with path.open("rb") as stream: + before = os.fstat(stream.fileno()) + if prefix_length: + digest.update(before.st_size.to_bytes(8, "little")) + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + if ( + size != before.st_size + or identity(before) != identity(os.fstat(stream.fileno())) + or identity(before) != identity(path.stat()) + ): + raise OSError(f"Source file changed while hashing: {path}") + return size + + def _directory_identity(path: Path) -> tuple[str, int]: """Hash a directory as a deterministic sequence of relative file bytes. @@ -41,12 +72,9 @@ def _directory_identity(path: Path) -> tuple[str, int]: files = sorted(candidate for candidate in path.rglob("*") if candidate.is_file()) for candidate in files: relative = candidate.relative_to(path).as_posix().encode("utf-8") - content = candidate.read_bytes() digest.update(len(relative).to_bytes(8, "little")) digest.update(relative) - digest.update(len(content).to_bytes(8, "little")) - digest.update(content) - size += len(content) + size += _stream_file(candidate, digest, prefix_length=True) return digest.hexdigest(), size @@ -60,9 +88,9 @@ def source_content_key(name: str, path: str | Path) -> str: source_path = Path(path) if source_path.is_file(): - content = source_path.read_bytes() - content_hash = hashlib.sha256(content).hexdigest() - size = len(content) + digest = hashlib.sha256() + size = _stream_file(source_path, digest) + content_hash = digest.hexdigest() elif source_path.is_dir(): content_hash, size = _directory_identity(source_path) else: @@ -88,6 +116,11 @@ def artifact_key(node_key: str, entity: str, column: str) -> str: return _hash_parts("artifact", node_key, entity, column) +def opaque_artifact_key(node_key: str, name: str) -> str: + """Identity of an opaque or typed byte output (legacy domain preserved).""" + return _hash_parts("node-artifact", node_key, name) + + def frame_key(node_key: str) -> str: """Derive the structural frame artifact identity from its node.""" @@ -240,6 +273,27 @@ def node_key( if kernel_capabilities.numeric is Numeric.PLATFORM_BITWISE else () ) + typed_inputs = ( + ( + { + "typed_artifacts": tuple( + ( + item.name, + opaque_artifact_key( + _required_key(input_keys, item.producer, node_id), + item.artifact, + ), + normative(item.type), + ) + for item in sorted( + node.artifact_inputs, key=lambda value: value.name + ) + ) + }, + ) + if node.artifact_inputs + else () + ) return _hash_parts( "node", normative(node), @@ -250,6 +304,7 @@ def node_key( graph_facts, capabilities, *platform_scope, + *typed_inputs, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/manifest.py b/packages/microcosm-graph/src/microcosm/graph/manifest.py index 90b4fa3bc..f7b1dd066 100644 --- a/packages/microcosm-graph/src/microcosm/graph/manifest.py +++ b/packages/microcosm-graph/src/microcosm/graph/manifest.py @@ -13,6 +13,13 @@ from microcosm.frame import Frame +from .artifact_edges import require_compatible_scope, value_from_descriptor +from .availability import ( + execution_state, + has_execution, + unavailable_artifacts, + validate_execution, +) from .canonical import canonical_json, sha256_domain from .decl import GATE_OUTCOMES, StructuralDelta from .errors import NodeRejectedError, StoreCorruptError @@ -34,6 +41,8 @@ __all__ = ["Decision", "NodeReceipt", "PopulationView", "RunManifest"] _SCHEMA_VERSION = 2 +_TYPED_SCHEMA_VERSION = 3 +_EXCEPTION_SCHEMA_VERSION = 4 _LEGACY_SCHEMA_VERSION = 1 _CERTIFYING_GATE_OUTCOMES = frozenset({"pass", "not_applicable"}) @@ -51,7 +60,7 @@ class PopulationView(Frame): accessor for every entity name. The source Frame keeps its original type. """ - __slots__ = () + __slots__ = ("__weakref__",) def __init__(self, frame: Frame) -> None: if not isinstance(frame, Frame): @@ -220,6 +229,7 @@ class NodeReceipt: weight_key: str | None = None opaque_artifacts: Mapping[str, str] = field(default_factory=dict) legacy_capabilities: bool = field(default=False, kw_only=True) + typed_artifacts: Mapping[str, object] = field(default_factory=dict, kw_only=True) def __post_init__(self) -> None: if not isinstance(self.key, str): @@ -244,6 +254,26 @@ def __post_init__(self) -> None: object.__setattr__(self, "hit", False) elif not isinstance(self.capabilities, Capabilities): raise TypeError("NodeReceipt.capabilities must be Capabilities") + typed = _freeze_json(self.typed_artifacts) + if not isinstance(typed, Mapping): + raise TypeError("NodeReceipt.typed_artifacts must be a mapping.") + if typed: + if set(typed) != {"inputs", "outputs"} or any( + not isinstance(typed[name], Mapping) for name in typed + ): + raise ValueError( + "Typed artifact provenance requires input/output mappings." + ) + for bindings in typed.values(): + for alias, descriptor in bindings.items(): + if not isinstance(alias, str) or not alias: + raise ValueError( + "Typed artifact aliases must be nonempty strings." + ) + value_from_descriptor(b"", descriptor) + if self.legacy_capabilities: + raise ValueError("Legacy capabilities cannot describe typed artifacts.") + object.__setattr__(self, "typed_artifacts", typed) frozen_receipt = _freeze_json(self.receipt) if not isinstance(frozen_receipt, Mapping): raise TypeError("NodeReceipt.receipt must be a mapping") @@ -278,6 +308,21 @@ def __post_init__(self) -> None: opaque_artifacts[name] = key object.__setattr__(self, "opaque_artifacts", MappingProxyType(opaque_artifacts)) + if has_execution(self.receipt): + if self.legacy_capabilities: + raise ValueError("Legacy receipts cannot carry executor outcomes.") + validate_execution( + self.receipt, + self.capabilities, + self.typed_artifacts.get("outputs", {}), + self.opaque_artifacts, + has_products=bool( + self.artifacts + or self.frame_key is not None + or self.weight_key is not None + ), + ) + wall_time = float(self.wall_time) if not math.isfinite(wall_time) or wall_time < 0: raise ValueError("NodeReceipt.wall_time must be finite and non-negative") @@ -347,6 +392,11 @@ def _payload(self) -> dict[str, object]: "frame_key": self.frame_key, "weight_key": self.weight_key, "opaque_artifacts": self.opaque_artifacts, + **( + {"typed_artifacts": self.typed_artifacts} + if self.typed_artifacts + else {} + ), "wall_time": self.wall_time, } @@ -419,6 +469,7 @@ def __post_init__(self) -> None: raise TypeError("RunManifest.nodes values must be NodeReceipt") nodes[node_id] = receipt object.__setattr__(self, "nodes", MappingProxyType(nodes)) + _validate_typed_ancestry(nodes) decisions = tuple( decision @@ -433,18 +484,23 @@ def __post_init__(self) -> None: for name in ("started_at", "finished_at", "host"): if not isinstance(getattr(self, name), str): raise TypeError(f"RunManifest.{name} must be a string") - populations: dict[str, PopulationView] = {} - for version_id, frame in self.populations.items(): - if not isinstance(version_id, str): - raise TypeError("RunManifest.populations keys must be strings") - if not isinstance(frame, Frame): - raise TypeError("RunManifest.populations values must be Frame") - populations[version_id] = PopulationView(frame) - object.__setattr__( - self, - "populations", - MappingProxyType(populations), - ) + # Only our private validated mapping can bypass eager value validation. + # The local import avoids a manifest/attachment module import cycle. + from .attachments import _LazyPopulations + + if type(self.populations) is not _LazyPopulations: + populations: dict[str, PopulationView] = {} + for version_id, frame in self.populations.items(): + if not isinstance(version_id, str): + raise TypeError("RunManifest.populations keys must be strings") + if not isinstance(frame, Frame): + raise TypeError("RunManifest.populations values must be Frame") + populations[version_id] = PopulationView(frame) + object.__setattr__( + self, + "populations", + MappingProxyType(populations), + ) mass_ledgers: dict[str, tuple[MassRecord, ...]] = {} for version_id, records in self.mass_ledgers.items(): if not isinstance(version_id, str): @@ -592,7 +648,11 @@ def to_json(self) -> str: """Serialize the complete portable provenance as canonical JSON.""" payload = { - "schema_version": _SCHEMA_VERSION, + "schema_version": _EXCEPTION_SCHEMA_VERSION + if any(execution_state(node.receipt) for node in self.nodes.values()) + else _TYPED_SCHEMA_VERSION + if any(node.typed_artifacts for node in self.nodes.values()) + else _SCHEMA_VERSION, "key": self.key, "tier": self.tier, "known_failures": self.known_failures, @@ -633,6 +693,8 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: if type(schema_version) is not int or schema_version not in { _LEGACY_SCHEMA_VERSION, _SCHEMA_VERSION, + _TYPED_SCHEMA_VERSION, + _EXCEPTION_SCHEMA_VERSION, }: raise ValueError(f"unsupported manifest schema version {schema_version!r}") @@ -657,6 +719,14 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: finished_at=_string_field(raw, "finished_at"), host=_string_field(raw, "host"), ) + if schema_version == _TYPED_SCHEMA_VERSION and not any( + node.typed_artifacts for node in nodes.values() + ): + raise ValueError("Schema-v3 manifest must carry typed artifact provenance.") + if schema_version == _EXCEPTION_SCHEMA_VERSION and not any( + execution_state(node.receipt) for node in nodes.values() + ): + raise ValueError("Schema-v4 manifest must carry executor outcomes.") body = raw.get("content_addressed") if not isinstance(body, Mapping): raise ValueError("manifest content-addressed body must be an object") @@ -678,7 +748,7 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: raise ValueError( "manifest content key mismatch: serialized provenance was altered" ) - if schema_version == _SCHEMA_VERSION and serialized_key != manifest.key: + if schema_version != _LEGACY_SCHEMA_VERSION and serialized_key != manifest.key: raise ValueError( "manifest content key mismatch: serialized key differs from " "reconstructed portable provenance" @@ -1090,6 +1160,18 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec frame_key = value.get("frame_key") weight_key = value.get("weight_key") opaque_artifacts = value.get("opaque_artifacts", {}) + typed_artifacts = value.get("typed_artifacts", {}) + if ( + isinstance(value.get("receipt"), Mapping) + and has_execution(value["receipt"]) + and schema_version != _EXCEPTION_SCHEMA_VERSION + ): + raise ValueError("Executor exceptional outcomes require manifest schema 4.") + if "typed_artifacts" in value and schema_version not in { + _TYPED_SCHEMA_VERSION, + _EXCEPTION_SCHEMA_VERSION, + }: + raise ValueError("Typed artifact provenance requires manifest schema 3 or 4.") capabilities_payload = value.get("capabilities") if schema_version == _LEGACY_SCHEMA_VERSION: # Every schema-v1 receipt is legacy: v1 never recorded a tolerance, so @@ -1163,4 +1245,102 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec weight_key=weight_key, opaque_artifacts=opaque_artifacts, legacy_capabilities=legacy_capabilities, + typed_artifacts=typed_artifacts, ) + + +def _validate_typed_ancestry(nodes: Mapping[str, NodeReceipt]) -> None: + edges: dict[str, set[str]] = {node_id: set() for node_id in nodes} + for node_id, node in nodes.items(): + if not node.typed_artifacts: + continue + for name, entry in node.typed_artifacts["outputs"].items(): + value = value_from_descriptor(b"", entry) + if ( + entry["producer"] != node_id + or entry["artifact"] != name + or value.producer_key != node.key + or ( + name + not in unavailable_artifacts( + node.receipt, node.typed_artifacts["outputs"] + ) + and node.opaque_artifacts.get(name) != value.key + ) + ): + raise ValueError( + f"Node {node_id!r} typed artifact output provenance mismatch." + ) + if ( + value.numerics.numeric is not node.capabilities.numeric + or value.numerics.tolerance != node.capabilities.tolerance + ): + raise ValueError( + f"Node {node_id!r} typed artifact numeric contract mismatch." + ) + for entry in node.typed_artifacts["inputs"].values(): + value = value_from_descriptor(b"", entry) + producer_id = entry["producer"] + producer = nodes.get(producer_id) + if producer is None or producer.key != value.producer_key: + raise ValueError( + f"Node {node_id!r} typed artifact producer is missing or inconsistent." + ) + if ( + producer.typed_artifacts.get("outputs", {}).get(entry["artifact"]) + != entry + ): + raise ValueError( + f"Node {node_id!r} typed artifact does not match its producer output." + ) + require_compatible_scope(value.numerics, node.capabilities) + if entry["artifact"] in unavailable_artifacts( + producer.receipt, producer.typed_artifacts.get("outputs", {}) + ) and ( + execution_state(node.receipt) != "unreached" + or producer_id not in node.receipt["execution"]["blocked_by"] + ): + raise ValueError("An unavailable artifact consumer must be unreached.") + edges[node_id].add(producer_id) + + for node_id, node in nodes.items(): + if execution_state(node.receipt) != "unreached": + continue + for parent_id, parent_key in node.receipt["execution"]["blocked_by"].items(): + parent = nodes.get(parent_id) + if parent is None or parent.key != parent_key: + raise ValueError("Unreached blocker is missing or has a different key.") + if execution_state(parent.receipt) not in {"gate_exception", "unreached"}: + raise ValueError("Unreached blocker has no unavailable outputs.") + if execution_state(parent.receipt) == "gate_exception" and not any( + entry["producer"] == parent_id + for entry in node.typed_artifacts.get("inputs", {}).values() + ): + raise ValueError( + "A gate exception blocker must be a declared typed input." + ) + edges[node_id].add(parent_id) + + memo: dict[str, frozenset[str]] = {} + + def visit(node_id: str, trail: frozenset[str]) -> frozenset[str]: + if node_id in trail: + raise ValueError("Typed artifact ancestry contains a cycle.") + if node_id in memo: + return memo[node_id] + ancestors = set(edges[node_id]) + for parent in edges[node_id]: + ancestors.update(visit(parent, trail | {node_id})) + memo[node_id] = frozenset(ancestors) + return memo[node_id] + + for node_id, node in nodes.items(): + ancestors = visit(node_id, frozenset()) + if _capability_role(node) is KernelRole.RELEASE: + artifact_gates = { + parent + for parent in ancestors + if _capability_role(nodes[parent]) is KernelRole.GATE + } + if not artifact_gates.issubset(set(node.receipt.get("gate_ancestry", ()))): + raise ValueError("Release omitted a typed artifact gate ancestor.") diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index 33b5437f3..85c71b4de 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -7,6 +7,9 @@ from .canonical import canonical_json from .decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, Graph, Node, Owned, @@ -90,6 +93,40 @@ def _partition_from_payload(value: object, label: str) -> tuple[str, str] | None def _node_payload(node: Node) -> dict[str, object]: return { + **( + { + "artifact_inputs": [ + { + "name": item.name, + "producer": item.producer, + "artifact": item.artifact, + "type": { + "name": item.type.name, + "schema_version": item.type.schema_version, + }, + } + for item in node.artifact_inputs + ] + } + if node.artifact_inputs + else {} + ), + **( + { + "artifact_outputs": [ + { + "name": item.name, + "type": { + "name": item.type.name, + "schema_version": item.type.schema_version, + }, + } + for item in node.artifact_outputs + ] + } + if node.artifact_outputs + else {} + ), "id": node.id, "kernel": node.kernel, "inputs": [ @@ -161,6 +198,9 @@ def _node_from_payload(value: object, index: int) -> Node: "description", "citation", } + fields.update( + name for name in ("artifact_inputs", "artifact_outputs") if name in payload + ) if "entrants" in payload: fields.add("entrants") _exact_fields(payload, fields, label) @@ -174,6 +214,18 @@ def _node_from_payload(value: object, index: int) -> Node: population = _optional_string(payload["population"], f"{label}.population") base = _optional_string(payload["base"], f"{label}.base") return Node( + artifact_inputs=tuple( + _artifact_from_payload(value, input_=True) + for value in _array( + payload.get("artifact_inputs", []), f"{label}.artifact_inputs" + ) + ), + artifact_outputs=tuple( + _artifact_from_payload(value, input_=False) + for value in _array( + payload.get("artifact_outputs", []), f"{label}.artifact_outputs" + ) + ), id=_string(payload["id"], f"{label}.id"), kernel=_string(payload["kernel"], f"{label}.kernel"), inputs=tuple( @@ -302,3 +354,25 @@ def _exact_fields( def _reject_json_constant(value: str) -> object: raise ValueError(f"graph JSON contains non-finite constant {value}") + + +def _artifact_from_payload( + value: object, *, input_: bool +) -> ArtifactInput | ArtifactOutput: + raw = _mapping(value, "artifact declaration") + fields = {"name", "type"} | ({"producer", "artifact"} if input_ else set()) + _exact_fields(raw, fields, "artifact declaration") + type_raw = _mapping(raw["type"], "artifact type") + _exact_fields(type_raw, {"name", "schema_version"}, "artifact type") + type_ = ArtifactType( + _string(type_raw["name"], "artifact type.name"), type_raw["schema_version"] + ) + name = _string(raw["name"], "artifact name") + if input_: + return ArtifactInput( + name, + _string(raw["producer"], "artifact producer"), + _string(raw["artifact"], "artifact output"), + type_, + ) + return ArtifactOutput(name, type_) diff --git a/packages/microcosm-graph/src/microcosm/graph/view.py b/packages/microcosm-graph/src/microcosm/graph/view.py index 63820048a..446a78838 100644 --- a/packages/microcosm-graph/src/microcosm/graph/view.py +++ b/packages/microcosm-graph/src/microcosm/graph/view.py @@ -87,6 +87,22 @@ def describe( f"Implementation hash: {implementation}", ] ) + if node.artifact_inputs: + lines.append( + "Artifact inputs: " + + "; ".join( + f"{item.name} <- {item.producer}.{item.artifact} ({item.type.name}@{item.type.schema_version})" + for item in node.artifact_inputs + ) + ) + if node.artifact_outputs: + lines.append( + "Artifact outputs: " + + "; ".join( + f"{item.name} ({item.type.name}@{item.type.schema_version})" + for item in node.artifact_outputs + ) + ) if run_receipt is None: lines.append( 'Seed: int.from_bytes(sha256(b"seed\\0" + node_key)[:8], "little")' From 6e3907f86a487e2302e0a048855a7ea68af16e98 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 005/255] Stage integration layer 03: ACCEPTED-SHARED-RESTORE.patch --- PROGRESS.md | 2 + .../src/microcosm/build/frame_checkpoint.py | 167 ++- .../microcosm/build/us_runtime/acs_inputs.py | 8 + .../microcosm/build/us_runtime/acs_pums.py | 202 ++- .../build/us_runtime/acs_transfer.py | 21 +- .../build/us_runtime/asec_checkpoint.py | 179 ++- .../congressional_district_vintage.py | 16 + .../microcosm/build/us_runtime/cps_carried.py | 128 +- .../build/us_runtime/eligibility_inputs.py | 290 +++- .../build/us_runtime/hours_worked.py | 98 +- .../build/us_runtime/housing_inputs.py | 305 +++- .../build/us_runtime/multispine_pool.py | 1310 ++++++++++++++--- .../build/us_runtime/operator_boundary.py | 29 +- .../build/us_runtime/prior_year_income.py | 106 +- .../us_runtime/puf_capital_gains_tail.py | 44 +- .../microcosm/build/us_runtime/qbi_inputs.py | 50 +- .../build/us_runtime/relationship_inputs.py | 256 +++- .../build/us_runtime/spine_assembly.py | 504 +++++-- .../build/us_runtime/stacked_spine.py | 651 ++++---- .../microcosm-build/tests/test_us_acs_pums.py | 177 ++- .../tests/test_us_asec_checkpoint.py | 304 ++++ .../src/microcosm/calibrate/__init__.py | 2 + .../microcosm-fit/src/microcosm/fit/qrf.py | 84 ++ 23 files changed, 4066 insertions(+), 867 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index d9ac95e35..4852c7f73 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -741,3 +741,5 @@ Apply thirteen explicit source layers, commit each, then separately review isola Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py index ac26729c6..0e4cf984a 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py @@ -41,10 +41,15 @@ "write_frame_checkpoint", ] -FRAME_CHECKPOINT_SCHEMA_VERSION = 3 +FRAME_CHECKPOINT_SCHEMA_VERSION = 4 +_NULLABLE_BOOLEAN_SCHEMA_VERSION = 3 _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION = 2 _SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS = frozenset( - {_LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, FRAME_CHECKPOINT_SCHEMA_VERSION} + { + _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, + _NULLABLE_BOOLEAN_SCHEMA_VERSION, + FRAME_CHECKPOINT_SCHEMA_VERSION, + } ) _ARTIFACT_KIND = "populace_frame_checkpoint" @@ -56,6 +61,20 @@ _ENCODING_TIMEDELTA = "timedelta64" _ENCODING_OBJECT = "object_scalars_v1" _ENCODING_NULLABLE_BOOLEAN = "nullable_boolean_v1" +_ENCODING_NULLABLE_INTEGER = "nullable_integer_v1" +_NULLABLE_INTEGER_DTYPES = { + str(dtype): dtype + for dtype in ( + pd.Int8Dtype(), + pd.Int16Dtype(), + pd.Int32Dtype(), + pd.Int64Dtype(), + pd.UInt8Dtype(), + pd.UInt16Dtype(), + pd.UInt32Dtype(), + pd.UInt64Dtype(), + ) +} _TAG_NONE = 0 _TAG_PD_NA = 1 @@ -346,25 +365,21 @@ def _checkpoint_metadata( ) strata_spec = _series_spec(frame.strata, label="strata") - uses_nullable_boolean = any( - column.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - for table in table_specs - for column in table["columns"] - ) or any( - table["index"].get("encoding") == _ENCODING_NULLABLE_BOOLEAN + encodings = { + spec.get("encoding") for table in table_specs - ) - uses_nullable_boolean = uses_nullable_boolean or ( - strata_spec.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - ) + for spec in [*table["columns"], table["index"]] + } | {strata_spec.get("encoding")} + if _ENCODING_NULLABLE_INTEGER in encodings: + version = FRAME_CHECKPOINT_SCHEMA_VERSION + elif _ENCODING_NULLABLE_BOOLEAN in encodings: + version = _NULLABLE_BOOLEAN_SCHEMA_VERSION + else: + version = _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION schema = frame.schema return { "artifact_kind": _ARTIFACT_KIND, - "schema_version": ( - FRAME_CHECKPOINT_SCHEMA_VERSION - if uses_nullable_boolean - else _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION - ), + "schema_version": version, "schema": { "person_entity": schema.person_entity, "group_entities": list(schema.group_entities), @@ -420,6 +435,15 @@ def _series_spec(series: pd.Series, *, label: str) -> dict[str, object]: "encoding": _ENCODING_NULLABLE_BOOLEAN, "has_null_mask": bool(series.isna().any()), } + if ( + str(dtype) in _NULLABLE_INTEGER_DTYPES + and dtype == (_NULLABLE_INTEGER_DTYPES[str(dtype)]) + ): + return { + "dtype": str(dtype), + "encoding": _ENCODING_NULLABLE_INTEGER, + "has_null_mask": bool(series.isna().any()), + } if isinstance(dtype, pd.api.extensions.ExtensionDtype) and not isinstance( dtype, pd.StringDtype ): @@ -561,6 +585,19 @@ def _write_series(group: Any, series: pd.Series, spec: Mapping[str, object]) -> null_mask.astype(np.uint8, copy=False), ) return + if encoding == _ENCODING_NULLABLE_INTEGER: + dtype = _NULLABLE_INTEGER_DTYPES[str(spec["dtype"])] + null_mask = series.isna().to_numpy(dtype=np.bool_, copy=False) + # Keep exact integer width and observed values, including UInt64 IDs. + # Hidden extension-array storage is not part of the logical payload. + values = series.to_numpy(dtype=dtype.numpy_dtype, na_value=0, copy=True) + values[null_mask] = 0 + _write_numpy_dataset(group, "values", values) + if spec.get("has_null_mask") is True: + _write_numpy_dataset( + group, "null_mask", null_mask.astype(np.uint8, copy=False) + ) + return raise RuntimeError(f"Unknown checkpoint series encoding {encoding!r}.") @@ -641,6 +678,55 @@ def _read_series( pd.arrays.BooleanArray(values, mask, copy=False), copy=False, ) + elif encoding == _ENCODING_NULLABLE_INTEGER: + integer_dtype = _NULLABLE_INTEGER_DTYPES.get(dtype) + if integer_dtype is None: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer encoding " + f"has unsupported declared dtype {dtype!r}." + ) + has_null_mask = spec.get("has_null_mask") + if type(has_null_mask) is not bool: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer " + "has_null_mask must be a boolean." + ) + values = _read_numpy_dataset(group, "values", path) + if values.ndim != 1 or values.dtype != integer_dtype.numpy_dtype: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer values " + f"must be a one-dimensional {integer_dtype.numpy_dtype} array." + ) + if has_null_mask: + if "null_mask" not in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} is missing its null mask." + ) + null_mask = _read_numpy_dataset(group, "null_mask", path) + if ( + null_mask.ndim != 1 + or null_mask.dtype != np.dtype(np.uint8) + or len(null_mask) != len(values) + or ((null_mask != 0) & (null_mask != 1)).any() + or not null_mask.any() + ): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask must be a " + "one-dimensional uint8 0/1 array aligned to its values." + ) + mask = null_mask.astype(np.bool_, copy=False) + if (values[mask] != 0).any(): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask covers " + "noncanonical nonzero integer storage." + ) + else: + if "null_mask" in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} has an unexpected null mask." + ) + mask = np.zeros(len(values), dtype=np.bool_) + series = pd.Series(pd.arrays.IntegerArray(values, mask, copy=False), copy=False) else: raise ValueError( f"Frame checkpoint {path} has unknown encoding {encoding!r} for {label!r}." @@ -838,39 +924,62 @@ def _read_metadata(root: Any, path: Path) -> dict[str, Any]: f"Frame checkpoint {path} schema version is {version!r}; expected " f"one of {sorted(_SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS)}." ) - nullable_spec_count = 0 + boolean_spec_count = 0 + integer_spec_count = 0 for label, spec in _checkpoint_series_specs(metadata): dtype = spec.get("dtype") encoding = spec.get("encoding") - declares_nullable = dtype == "boolean" - uses_nullable_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_boolean = dtype == "boolean" + uses_boolean_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_integer = isinstance(dtype, str) and dtype in _NULLABLE_INTEGER_DTYPES + uses_integer_encoding = encoding == _ENCODING_NULLABLE_INTEGER + if version < FRAME_CHECKPOINT_SCHEMA_VERSION and ( + declares_integer or uses_integer_encoding + ): + raise ValueError( + f"Frame checkpoint {path} schema version {version} cannot carry " + f"nullable integer spec {label!r}." + ) if version == _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable or uses_nullable_encoding + declares_boolean or uses_boolean_encoding ): raise ValueError( f"Frame checkpoint {path} schema version 2 cannot carry nullable " f"boolean spec {label!r}." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable != uses_nullable_encoding + if version >= _NULLABLE_BOOLEAN_SCHEMA_VERSION and ( + declares_boolean != uses_boolean_encoding ): raise ValueError( - f"Frame checkpoint {path} schema version 3 nullable boolean spec " - f"{label!r} must pair declared dtype 'boolean' with encoding " + f"Frame checkpoint {path} schema version {version} nullable boolean " + f"spec {label!r} must pair declared dtype 'boolean' with encoding " f"{_ENCODING_NULLABLE_BOOLEAN!r}." ) - if uses_nullable_encoding: - nullable_spec_count += 1 + if declares_integer != uses_integer_encoding: + raise ValueError( + f"Frame checkpoint {path} schema version {version} nullable integer " + f"spec {label!r} must pair a supported pandas integer dtype " + f"with encoding {_ENCODING_NULLABLE_INTEGER!r}." + ) + if uses_boolean_encoding or uses_integer_encoding: + family = "boolean" if uses_boolean_encoding else "integer" if type(spec.get("has_null_mask")) is not bool: raise ValueError( - f"Frame checkpoint {path} {label!r} nullable boolean " + f"Frame checkpoint {path} {label!r} nullable {family} " "has_null_mask must be a boolean." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and nullable_spec_count == 0: + boolean_spec_count += uses_boolean_encoding + integer_spec_count += uses_integer_encoding + if version == _NULLABLE_BOOLEAN_SCHEMA_VERSION and boolean_spec_count == 0: raise ValueError( f"Frame checkpoint {path} schema version 3 requires at least one " "nullable boolean spec." ) + if version == FRAME_CHECKPOINT_SCHEMA_VERSION and integer_spec_count == 0: + raise ValueError( + f"Frame checkpoint {path} schema version 4 requires at least one " + "nullable integer spec." + ) return metadata diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py index 88b29412f..2d8e8e99e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py @@ -247,6 +247,14 @@ def _map_tenure( transformation="ACS TEN enum recode through SPM membership", register=register, ) + # These enum outputs are strings even when the selected source households + # all have unreported tenure. Declare that type without inventing a value; + # generic serialization cannot infer a type from an all-missing object axis. + string_dtype = pd.StringDtype(storage="python", na_value=np.nan) + household["tenure_type"] = household["tenure_type"].astype(string_dtype) + spm_unit["spm_unit_tenure_type"] = spm_unit["spm_unit_tenure_type"].astype( + string_dtype + ) def _map_housing_amounts( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index dd0456fca..0038c91d7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -46,6 +46,8 @@ ACS_2024_1YR_SPINE = "acs_2024_1yr" ACS_2024_1YR_VINTAGE = 2024 DEFAULT_CHUNKSIZE = 100_000 +MAX_EXACT_HOUSEHOLDS = 1_000_000 +MAX_EXACT_PERSON_ROWS = 1_000_000 _HOUSEHOLD_REQUIRED = ( "SERIALNO", @@ -69,6 +71,14 @@ "TAXAMT", "TYPEHUGQ", ) +# 2024 ACS PUMS dictionary p.3: TYPEHUGQ 1 is a housing-unit record, 2 an +# institutional and 3 a noninstitutional group-quarters person record. Only +# occupied code-1 rows survive the vacancy drop in _occupied_households. +_HOUSEHOLD_AXIS_KINDS = { + 1: "occupied_housing_unit", + 2: "institutional_gq_person", + 3: "noninstitutional_gq_person", +} _PERSON_REQUIRED = ( "SERIALNO", "SPORDER", @@ -168,11 +178,36 @@ def __post_init__(self) -> None: if self.max_households is not None and self.max_households <= 0: raise ValueError("max_households must be positive when provided.") + @staticmethod + def snapshot_serialnos(serialnos): + """Freeze exact raw native keys; no sampling probability is implied.""" + if serialnos is None: + return None + if ( + type(serialnos) is not tuple + or not 0 < len(serialnos) <= MAX_EXACT_HOUSEHOLDS + or any( + type(key) is not str + or len(key) != 13 + or key[:6] not in {"2024HU", "2024GQ"} + or not key[6:].isascii() + or not key[6:].isdigit() + or int(key[6:]) == 0 + for key in serialnos + ) + or len(set(serialnos)) != len(serialnos) + ): + raise ValueError( + "ACS exact selection requires bounded unique raw native keys." + ) + return serialnos + def load_acs_pums_tables( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[dict[str, pd.DataFrame], dict[str, Any]]: """Read and key-align the household and person PUMS tables. @@ -180,6 +215,9 @@ def load_acs_pums_tables( missing; this stage never converts an out-of-universe blank to zero. """ + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + if serialnos is not None and source.max_households is not None: + raise ValueError("ACS exact serialnos and max_households are ambiguous together.") if chunksize <= 0: raise ValueError("chunksize must be positive.") household, household_members = _read_archive( @@ -207,7 +245,24 @@ def load_acs_pums_tables( raise ValueError(f"ACS duplicate household SERIALNO value(s): {examples}.") household = household.sort_values("SERIALNO", kind="stable").reset_index(drop=True) all_household_serials = frozenset(household["SERIALNO"].tolist()) + full_household = household household, vacant_count = _occupied_households(household) + if serialnos is not None: + if not set(serialnos) <= all_household_serials: + raise ValueError("ACS exact selection contains absent household keys.") + weights = pd.to_numeric(household.WGTP, errors="coerce").to_numpy( + dtype=float + ) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError("ACS WGTP must be finite and nonnegative.") + _validate_amount_columns( + (household.RNTP, household.GRNTP, household.TAXAMT), household.ADJHSG + ) + household = household.loc[household.SERIALNO.isin(serialnos)].reset_index( + drop=True + ) + if household.NP.sum() > MAX_EXACT_PERSON_ROWS: + raise ValueError("ACS selected complete roster exceeds native person budget.") if source.max_households is not None and len(household) > source.max_households: household = _smoke_household_selection(household, source.max_households) selected_serials = frozenset(household["SERIALNO"].tolist()) @@ -220,6 +275,7 @@ def load_acs_pums_tables( chunksize=chunksize, valid_serials=all_household_serials, retained_serials=selected_serials, + validate_households=full_household if serialnos is not None else None, ) duplicate_people = person.duplicated(["SERIALNO", "SPORDER"], keep=False) if duplicate_people.any(): @@ -247,6 +303,14 @@ def load_acs_pums_tables( "vacant_household_rows_dropped": vacant_count, "max_households": source.max_households, } + if serialnos is not None: + metadata["exact_selection"] = { + "requested_serialnos": serialnos, + "populated_serialnos": tuple(household.SERIALNO), + "vacant_serialnos": tuple(sorted(set(serialnos) - selected_serials)), + "selection_kind": "engineering_exact_keys", + "full_source_inclusion_probability": None, + } return {"household": household, "person": person}, metadata @@ -254,10 +318,13 @@ def build_acs_pums_unit_frame( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[Frame, dict[str, Any]]: """Construct the ACS 2024 1-year US entity frame.""" - tables, metadata = load_acs_pums_tables(source, chunksize=chunksize) + tables, metadata = load_acs_pums_tables( + source, chunksize=chunksize, serialnos=serialnos + ) household = tables["household"].copy() person = tables["person"].copy() @@ -299,6 +366,11 @@ def build_acs_pums_unit_frame( metadata.update( { "weighted_household_population": frame.weights_for("household").total, + # That total mixes housing-unit and GQ-person design mass. + "household_axis_composition": _household_axis_composition( + household, + household_weights, + ), "relationship_pointer_policy": ( "RELSHIPP reference/spouse pairing; own/adopted/stepchildren " "point to reference person and present spouse; all other " @@ -330,10 +402,12 @@ def _read_archive( chunksize: int, valid_serials: frozenset[str] | None = None, retained_serials: frozenset[str] | None = None, + validate_households: pd.DataFrame | None = None, ) -> tuple[pd.DataFrame, list[str]]: if not path.is_file(): raise FileNotFoundError(f"ACS PUMS archive not found: {path}") pieces: list[pd.DataFrame] = [] + source_roster = {} with ZipFile(path) as archive: members = sorted( name @@ -357,7 +431,7 @@ def _read_archive( usecols = [column for column in (*required, *optional) if column in columns] string_columns = { column: "string" - for column in ("SERIALNO", "ST", "STATE", "PUMA") + for column in ("SERIALNO", "ST", "STATE", "PUMA", "AGEP") if column in usecols } with archive.open(member_name) as member: @@ -369,6 +443,11 @@ def _read_archive( low_memory=False, ) for chunk in reader: + if "AGEP" in chunk: + # Validate original literals before numeric inference or + # selection can hide an unresolved source age. The native + # coverage contract accepts only one/two ASCII digits. + chunk["AGEP"] = _original_source_ages(chunk["AGEP"]) if valid_serials is not None: orphan = ~chunk["SERIALNO"].isin(valid_serials) if orphan.any(): @@ -382,15 +461,88 @@ def _read_archive( "ACS person SERIALNO value(s) missing from the " f"household archive: {examples}." ) + if validate_households is not None: + _validate_source_chunk(chunk, source_roster) if retained_serials is not None: chunk = chunk.loc[chunk["SERIALNO"].isin(retained_serials)] if not chunk.empty: pieces.append(chunk) + if validate_households is not None: + _validate_source_roster(validate_households, source_roster) if not pieces: return pd.DataFrame(columns=[*required, *optional]), members return pd.concat(pieces, ignore_index=True), members +def _original_source_ages(ages: pd.Series) -> np.ndarray: + """Preserve the native coverage owner's literal 0–99 age identity contract.""" + if not ages.str.fullmatch(r"[0-9]{1,2}", na=False).all(): + raise ValueError("ACS original AGEP must contain one or two ASCII digits.") + return ages.to_numpy(dtype=np.int64) + + +def _validate_amount_columns(amounts, factor): + """Keep the native mapper's finite observed-amount/factor gates before filtering.""" + adjustment = pd.to_numeric(factor, errors="coerce").to_numpy(dtype=float) + for values in amounts: + amount = pd.to_numeric(values, errors="coerce").to_numpy(dtype=float) + observed = ~np.isnan(amount) + if (observed & ~np.isfinite(amount)).any(): + raise ValueError("ACS source amount must be finite.") + if (observed & (~np.isfinite(adjustment) | (adjustment <= 0))).any(): + raise ValueError("ACS source adjustment must be finite and positive.") + + +def _validate_source_chunk(person, roster): + # Only global key sets and small household structural summaries survive a + # chunk. Unselected observation rows never enter the accumulated person table. + lines = _required_integer(person, "SPORDER") + marital = _required_integer(person, "MAR") + relations = _required_integer(person, "RELSHIPP") + sexes = _required_integer(person, "SEX") + if not set(marital) <= set(_ACS_TO_CPS_MARITAL_STATUS): + raise ValueError("ACS source MAR contains unsupported codes.") + if not set(relations) <= {20, *_ACS_SPOUSE_CODES, *_ACS_TO_CPS_RELATIONSHIP}: + raise ValueError("ACS source RELSHIPP contains unsupported codes.") + if not set(sexes) <= {1, 2}: + raise ValueError("ACS source SEX contains unsupported codes.") + _validate_amount_columns( + (person.WAGP, person.SEMP, person.INTP, person.RETP, person.SSP, person.SSIP), + person.ADJINC, + ) + weights = pd.to_numeric(person.PWGTP, errors="coerce").to_numpy(dtype=float) + for serial, line, relation, mar, weight in zip( + person.SERIALNO, lines, relations, marital, weights, strict=True + ): + state = roster.setdefault(serial, [set(), 0, 0, True, True, True]) + if line in state[0]: + raise ValueError("ACS duplicate person key in complete source.") + state[0].add(int(line)) + state[1] += int(relation == 20) + state[2] += int(relation in _ACS_SPOUSE_CODES) + state[3] &= relation not in {20, *_ACS_SPOUSE_CODES} or mar == 1 + state[4] &= relation in {37, 38} + state[5] &= bool(np.isfinite(weight) and weight > 0) + + +def _validate_source_roster(household, roster): + for serial, count, kind in household[ + ["SERIALNO", "NP", "TYPEHUGQ"] + ].itertuples(index=False, name=None): + state = dict.get(roster, serial) + if (len(state[0]) if state else 0) != int(count): + raise ValueError("ACS NP/person row-count mismatch in complete source.") + if not state: + continue + if int(kind) != 1 and not state[5]: + raise ValueError("ACS GQ PWGTP must be finite and positive.") + if not state[4]: + if state[1] != 1 or state[2] > 1: + raise ValueError("ACS source reference/spouse roster is invalid.") + if state[2] and not state[3]: + raise ValueError("ACS source spouse pair must have MAR=1.") + + def _occupied_households(household: pd.DataFrame) -> tuple[pd.DataFrame, int]: people = pd.to_numeric(household["NP"], errors="coerce") if people.isna().any() or (people < 0).any() or (people % 1 != 0).any(): @@ -616,6 +768,52 @@ def _household_weights(household: pd.DataFrame, person: pd.DataFrame) -> Weights return Weights(raw, WeightKind.DESIGN) +def _household_axis_composition( + household: pd.DataFrame, + household_weights: Weights, +) -> dict[str, int | float]: + """Split the loaded household-axis rows and DESIGN mass by TYPEHUGQ. + + The axis mixes occupied physical housing units with one-person group- + quarters person placeholders. Each part retains its own statistical unit; + their mixed sum is not a housing-unit count or a person count. These are + source design weights, with no common-survey or calibrated-population + claim. The rows described are the ones this build + loaded, after the vacancy drop and any ``max_households`` selection. + + Admissibility of the codes and their weights is decided upstream by + ``_occupied_households``; this only partitions what that check accepted, + and refuses if the accepted rows do not partition. + """ + + if "TYPEHUGQ" not in household.columns: + raise ValueError("ACS household-axis composition requires the TYPEHUGQ column.") + design = np.asarray(household_weights.values, dtype=np.float64) + if design.shape[0] != len(household): + raise ValueError( + "ACS household-axis composition needs one DESIGN weight per loaded " + f"household row; got {design.shape[0]} for {len(household)} row(s)." + ) + kind = pd.to_numeric(household["TYPEHUGQ"], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + composition: dict[str, int | float] = {} + partitioned = 0 + for code, label in _HOUSEHOLD_AXIS_KINDS.items(): + selected = kind == code + rows = int(selected.sum()) + composition[f"{label}_rows"] = rows + composition[f"{label}_design_weight_total"] = float(design[selected].sum()) + partitioned += rows + if partitioned != len(household): + raise ValueError( + "ACS household-axis composition does not partition the loaded rows: " + f"{partitioned} of {len(household)} carry a described TYPEHUGQ code." + ) + return composition + + def _attach_household_source_columns( frame: Frame, source_household: pd.DataFrame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index a3fb00a6b..e6cf40c8c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -48,6 +48,10 @@ from microcosm.fit.qrf import detect_regime from microcosm.frame import EntitySchema, Frame, Weights +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS as ACS_DERIVED_TRANSFER_INPUTS, +) + QRF: Any | None = None __all__ = [ @@ -182,6 +186,7 @@ def _pregnancy_structural_policy_identity(*, enabled: bool) -> dict[str, object] ).hexdigest() return payload + _IMMIGRATION_STATUS_TARGETS = ( "ssn_card_type", "immigration_status_str", @@ -635,14 +640,6 @@ def required_acs_transfer_inputs() -> frozenset[str]: ) -#: Person columns the default transfer DERIVES deterministically after the -#: QRF fits (never fitted themselves). Coverage checks require them on the -#: recipient exactly like declared plan targets. -ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( - "schedule_d_capital_gain_distributions", -) - - def acs_derived_transfer_expectations( target_families: TargetFamilies, ) -> dict[str, str]: @@ -1138,8 +1135,8 @@ def _prepare_pregnancy_structural_plan( "through 44." ) - source_codes, group_count, key_column, representatives = ( - _pregnancy_source_groups(table) + source_codes, group_count, key_column, representatives = _pregnancy_source_groups( + table ) eligible_min = np.ones(group_count, dtype=np.int8) eligible_max = np.zeros(group_count, dtype=np.int8) @@ -1449,9 +1446,7 @@ def transfer_acs_inputs( person, pregnancy_plan, ) - pregnancy_entity, pregnancy_family, pregnancy_targets = ( - pregnancy_request - ) + pregnancy_entity, pregnancy_family, pregnancy_targets = pregnancy_request if pregnancy_targets != (_PREGNANCY_TARGET,): # pragma: no cover raise AssertionError( "Pregnancy structural target was not isolated before receipt." diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa..6299a4762 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -12,6 +12,7 @@ import re from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path import numpy as np @@ -24,9 +25,15 @@ frame_identity, ) from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, +) from microcosm.build.us_runtime.operator_boundary import ( assert_operator_free_source_frame, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights __all__ = [ @@ -34,9 +41,11 @@ "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", ] _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" @@ -61,6 +70,9 @@ # artifacts lack the gate column and must fail loudly rather than let # PAW_VAL-only conflation back in (microcosm#591). ASEC_RAW_STAGE_SCHEMA_VERSION = 3 +# V4 is a distinct measured-coverage contract. Never reinterpret a v3 file +# under this schema merely because its columns happen to look compatible. +ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION = 4 ASEC_RAW_STAGE_STAGE = "raw_source_mapping" _RAW_STAGE_BINDING_KEYS = frozenset( { @@ -100,6 +112,23 @@ ) +@dataclass(frozen=True) +class _RawStagePolicy: + version: int + coverage_columns: frozenset[str] = frozenset() + + @property + def mapping_columns(self) -> frozenset[str]: + return _RAW_SOURCE_MAPPING_COLUMNS | self.coverage_columns + + +_RAW_STAGE_V3 = _RawStagePolicy(ASEC_RAW_STAGE_SCHEMA_VERSION) +_RAW_STAGE_V4 = _RawStagePolicy( + ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION, + frozenset(ASEC_REPORTED_COVERAGE_RAW_COLUMNS), +) + + def load_asec_pre_clone_checkpoint( path: str | Path, ) -> tuple[Frame, dict[str, object]]: @@ -152,11 +181,33 @@ def load_asec_raw_stage_checkpoint( contract even if it happens to carry a structurally valid US ``Frame``. """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V3) + + +def load_asec_raw_stage_checkpoint_v4( + path: str | Path, +) -> tuple[Frame, dict[str, object]]: + """Load only v4 measured-coverage sources with registered per-year pins. + + This validates a declared source artifact, not model accuracy or release + eligibility. The caller must independently authenticate the whole file's + digest; shape and registered source pins alone do not authenticate values. + """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V4) + + +def _load_asec_raw_stage_checkpoint( + path: str | Path, + *, + policy: _RawStagePolicy, +) -> tuple[Frame, dict[str, object]]: + checkpoint_path = Path(path) loaded = load_frame_checkpoint(checkpoint_path) metadata = _validate_raw_stage_binding( loaded.metadata, path=checkpoint_path, + policy=policy, ) stored_identity = FrameIdentity.from_payload( metadata["identity"], @@ -176,7 +227,11 @@ def load_asec_raw_stage_checkpoint( loaded.frame, label=f"ASEC raw-stage checkpoint {checkpoint_path}", ) - _validate_raw_stage_source_columns(loaded.frame, path=checkpoint_path) + _validate_raw_stage_source_columns( + loaded.frame, path=checkpoint_path, policy=policy + ) + if policy.coverage_columns: + _validate_coverage_v4_binding(loaded.frame, metadata, path=checkpoint_path) source_construction_identity = FrameIdentity.from_payload( metadata["source_construction_identity"], label="ASEC raw-stage source-construction identity", @@ -250,6 +305,7 @@ def _validate_raw_stage_binding( metadata: dict[str, object], *, path: Path, + policy: _RawStagePolicy = _RAW_STAGE_V3, ) -> dict[str, object]: actual_keys = frozenset(metadata) if actual_keys != _RAW_STAGE_BINDING_KEYS: @@ -265,8 +321,10 @@ def _validate_raw_stage_binding( "ASEC artifact." ) schema_version = metadata["schema_version"] - if schema_version != ASEC_RAW_STAGE_SCHEMA_VERSION or isinstance( - schema_version, bool + if ( + schema_version != policy.version + or isinstance(schema_version, bool) + or (policy.coverage_columns and type(schema_version) is not int) ): raise ValueError( f"ASEC raw-stage checkpoint {path} has an unsupported raw-stage " @@ -292,7 +350,9 @@ def _validate_raw_stage_binding( "lowercase SHA-256 digest." ) _validate_source_receipt(metadata["source_receipt"], path=path) - _validate_raw_source_mappings(metadata["raw_source_mappings"], path=path) + _validate_raw_source_mappings( + metadata["raw_source_mappings"], path=path, policy=policy + ) return dict(metadata) @@ -338,17 +398,19 @@ def _validate_source_receipt(receipt: object, *, path: Path) -> None: years.add(year) -def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: +def _validate_raw_source_mappings( + mappings: object, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: if not isinstance(mappings, Mapping): raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must be an object." ) - if frozenset(mappings) != _RAW_SOURCE_MAPPING_COLUMNS: + if frozenset(mappings) != policy.mapping_columns: raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must bind " - f"exactly {sorted(_RAW_SOURCE_MAPPING_COLUMNS)}." + f"exactly {sorted(policy.mapping_columns)}." ) - for column in sorted(_RAW_SOURCE_MAPPING_COLUMNS): + for column in sorted(policy.mapping_columns): mapping = mappings[column] if not isinstance(mapping, Mapping) or frozenset(mapping) != ( _RAW_SOURCE_MAPPING_KEYS @@ -400,9 +462,13 @@ def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: ) -def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: +def _validate_raw_stage_source_columns( + frame: Frame, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: person = frame.table("person") - missing = sorted(_RAW_STAGE_REQUIRED_PERSON_COLUMNS - set(person)) + missing = sorted( + (_RAW_STAGE_REQUIRED_PERSON_COLUMNS | policy.coverage_columns) - set(person) + ) if missing: raise ValueError( f"ASEC raw-stage checkpoint {path} is not input-complete; missing " @@ -458,6 +524,99 @@ def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: "in {0, 1, 2, 3}." ) + for column in sorted(policy.coverage_columns): + values = pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64 + ) + boolean = ( + person[column] + .map(lambda value: isinstance(value, (bool, np.bool_))) + .to_numpy() + ) + if not (np.isfinite(values) & np.isin(values, (1.0, 2.0)) & ~boolean).all(): + raise ValueError( + f"ASEC v4 raw-stage checkpoint {path} {column} must be complete " + "integer recodes in {1, 2}, not booleans." + ) + + +def _validate_coverage_v4_binding( + frame: Frame, metadata: Mapping, *, path: Path +) -> None: + """Cross-check source vintages and actual registered coverage source pins.""" + person = frame.table("person") + normalized_years = pd.to_numeric(person["source_year"]).astype(np.int64) + frame_years = set(normalized_years) + source_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if frame_years != source_years: + raise ValueError( + f"ASEC v4 checkpoint {path} frame/source receipt year coverage differs." + ) + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError( + f"ASEC v4 checkpoint {path} requires household design weights." + ) + peridnum = person["PERIDNUM"].map( + lambda value: value.decode() if isinstance(value, bytes) else value + ) + if not peridnum.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(): + raise ValueError( + f"ASEC v4 checkpoint {path} PERIDNUM must be exact 22-digit strings." + ) + keys = pd.DataFrame({"source_year": normalized_years, "PERIDNUM": peridnum}) + if keys.duplicated().any(): + raise ValueError( + f"ASEC v4 checkpoint {path} repeats a source-year/PERIDNUM identity." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + mapping = metadata["raw_source_mappings"][column] + pins = mapping["source_pins"] + years = [pin["income_year"] for pin in pins] + if len(years) != len(set(years)) or set(years) != source_years: + raise ValueError( + f"ASEC v4 {column} source pin year coverage differs or repeats." + ) + for pin in pins: + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(pin["income_year"]) + if registered is None or pin != { + "income_year": registered.income_year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + }: + raise ValueError( + f"ASEC v4 {column} source pin differs from the registered archive." + ) + audit = mapping["audit"] + if set(audit) != {str(year) for year in source_years}: + raise ValueError(f"ASEC v4 {column} audit year coverage differs.") + for year in source_years: + row = audit[str(year)] + if not isinstance(row, Mapping) or set(row) != { + "rows", + "yes_rows", + "no_rows", + "weighted_yes_share", + }: + raise ValueError(f"ASEC v4 {column} audit is malformed.") + counts = [row[key] for key in ("rows", "yes_rows", "no_rows")] + share = row["weighted_yes_share"] + if ( + any(type(count) is not int or count < 0 for count in counts) + or counts[0] != ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].rows + or counts[1] + counts[2] != counts[0] + or isinstance(share, bool) + or not isinstance(share, (int, float)) + or not np.isfinite(share) + or not 0 <= share <= 1 + ): + raise ValueError(f"ASEC v4 {column} audit counts/share are invalid.") + def _validate_asec_frame( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index f9f00468e..bedc3e745 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -7,6 +7,7 @@ import json from collections.abc import Iterable, Mapping from importlib.resources import files +from io import BytesIO from pathlib import Path from typing import Any @@ -94,6 +95,21 @@ def default_congressional_district_vintage_crosswalk_path() -> Path: ) +def decode_congressional_district_vintage_crosswalk(payload: bytes) -> pd.DataFrame: + """Decode immutable CSV artifact bytes under the existing crosswalk contract. + + The caller separately declares whether national roster validation is + required. Generic crosswalks retain the same behavior as the file loader. + """ + if not isinstance(payload, bytes): + raise TypeError("CD vintage crosswalk payload must be immutable bytes.") + return _prepare_crosswalk( + pd.read_csv(BytesIO(payload)), + source_prefix=SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, + target_prefix=CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + ) + + def load_default_congressional_district_vintage_crosswalk() -> pd.DataFrame: """Load the packaged Census-built default 117th->119th CD crosswalk. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py index 82c172e56..12ff2315d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass import numpy as np import pandas as pd @@ -21,6 +22,9 @@ PAW_TYPE_TANF_CODES, fill_asec_public_assistance_type_source, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + fill_asec_reported_coverage_source, +) from microcosm.frame import US_SCHEMA, Frame __all__ = [ @@ -35,6 +39,8 @@ "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", "WIC_CARRIER_ADJUDICATION_URL", "derive_us_cps_carried_inputs", + "derive_us_cps_carried_tables", + "CpsCarriedTables", "reported_snap_receipt_by_spm_unit", "reported_tanf_enrollment_by_spm_unit", "reported_wic_receipt_carrier", @@ -132,17 +138,75 @@ } ) +# Actual person reads for the no-sidecar measured mapping. Optional restoration +# remains a separate explicit argument on the legacy/table entrypoints. +CPS_CARRIED_RAW_PERSON_COLUMNS = frozenset( + { + "A_AGE", + "A_SEX", + "WICYN", + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + "SS_VAL", + "RESNSS1", + "RESNSS2", + "PNSN_VAL", + "ANN_VAL", + "OI_VAL", + "OI_OFF", + "RNT_VAL", + "FRSE_VAL", + "UC_VAL", + "PHIP_VAL", + "PMED_VAL", + "POTC_VAL", + "NOW_MRK", + "NOW_NONM", + "NOW_MCAID", + "NOW_GRP", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "PAW_VAL", + "PAW_TYP", + "SPM_SNAPSUB", + "SPM_CHILDCAREXPNS", + *(f"DST_SC{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + *(f"DST_VAL{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + } +) +CPS_CARRIED_BOOLEAN_INPUTS = frozenset( + { + "is_female", + "receives_wic", + "receives_tanf", + "receives_snap", + *US_REPORTED_COVERAGE_PERSON_INPUTS, + } +) + def derive_us_cps_carried_inputs( frame: Frame, *, public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, ) -> Frame: """Carry raw CPS ASEC values onto PE input leaves. - Existing leaf input columns are preserved, making the transform idempotent. - The transform refuses to run on a non-US frame and never creates - formula-owned aggregate variables. + Existing leaves keep the original presence rules; the raw other-income + pair intentionally recomputes its three split leaves. The transform + refuses a non-US frame and never creates formula-owned aggregates. + + Optional ``reported_coverage_source`` restores seven measured at-interview + recodes before deriving coverage leaves. It requires an exact source join; + conflicting observations fail. Native ``NOW_GRP`` and ``NOW_MRK`` remain + unchanged. These observed inputs do not certify a fitted model or release. ``PAW_VAL``, ``SPM_SNAPSUB``, and ``WICYN`` are annual reported facts, while the engine's ``receives_tanf``, ``receives_snap``, and @@ -164,7 +228,49 @@ def derive_us_cps_carried_inputs( if frame.schema != US_SCHEMA: raise ValueError("CPS-carried derivations require the US schema.") tables = {entity: frame.table(entity).copy() for entity in frame.entities} - person = tables["person"] + produced = derive_us_cps_carried_tables( + tables["person"], + tables["spm_unit"], + public_assistance_type_source=public_assistance_type_source, + reported_coverage_source=reported_coverage_source, + ) + tables["person"] = produced.person + tables["spm_unit"] = produced.spm_unit + + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +@dataclass(frozen=True) +class CpsCarriedTables: + """The two actual entity tables touched by the CPS-carried operator.""" + + person: pd.DataFrame + spm_unit: pd.DataFrame + + +def derive_us_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, +) -> CpsCarriedTables: + """Run the original measured mapping on isolated person and SPM views. + + Group outputs align by actual SPM IDs. Monetary amounts retain their + existing nominal source-year basis; this operation does no price alignment. + """ + person = person.copy(deep=True) + spm_unit = spm_unit.copy(deep=True) + if reported_coverage_source is not None: + person = fill_asec_reported_coverage_source(person, reported_coverage_source) _fill_missing(person, "age", _source(person, "A_AGE")) _fill_bool_missing(person, "is_female", _integer_source(person, "A_SEX") == 2) @@ -222,7 +328,6 @@ def derive_us_cps_carried_inputs( _fill_missing(person, "taxable_ira_distributions", _ira_distributions(person)) person = derive_us_alimony_from_asec(person) - tables["person"] = person direct_sources: Mapping[str, str] = { "rental_income": "RNT_VAL", @@ -241,10 +346,10 @@ def derive_us_cps_carried_inputs( _fill_health_coverage_inputs(person) _fill_spm_unit_reported_enrollment_inputs( person, - tables["spm_unit"], + spm_unit, public_assistance_type_source=public_assistance_type_source, ) - _fill_spm_unit_childcare_inputs(person, tables["spm_unit"]) + _fill_spm_unit_childcare_inputs(person, spm_unit) formula_owned = sorted(CPS_CARRIED_FORMULA_OWNED_COLUMNS.intersection(person)) if formula_owned: @@ -253,14 +358,7 @@ def derive_us_cps_carried_inputs( f"columns: {formula_owned}." ) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + return CpsCarriedTables(person, spm_unit) def reported_tanf_enrollment_by_spm_unit( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py index dc7e69826..a1e157806 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py @@ -52,6 +52,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -69,6 +70,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -78,6 +82,11 @@ "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_ELIGIBILITY_INPUTS_STAGE_NAME", "derive_us_eligibility_inputs_from_manifest", + "prepare_us_eligibility_person", + "us_eligibility_inputs_person_carries_signal", + "us_eligibility_inputs_gate_from_summary", + "us_eligibility_inputs_person_gate", + "us_eligibility_inputs_person_summary", "us_eligibility_inputs_signal_gate", "us_eligibility_inputs_stage_spec", "us_eligibility_inputs_summary", @@ -273,6 +282,108 @@ def _disabled_carries_signal(person: pd.DataFrame) -> bool: return values.nunique() > 1 +def us_eligibility_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the complete legacy pass-through decision on the real person table. + + All five outputs must exist, and observed disability must vary. Preserve + the current partial-null rule; other output variation, raw source columns + and weight resolution belong to separate operations and are not inferred. + """ + have_all = all( + column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + ) + return have_all and _disabled_carries_signal(person) + + +def _validated_eligibility_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US eligibility-inputs weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US eligibility-inputs weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US eligibility-inputs weights must be nonnegative.") + return array + + +def prepare_us_eligibility_person( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + time_period: int, +) -> pd.DataFrame: + """Derive measured ASEC eligibility inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_eligibility_inputs`: + it always runs the ``eligibility_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that + idempotence/pass-through decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns this stage reads (see + ``US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS``), in its own + index and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with all five eligibility columns attached + (``is_disabled``, ``is_blind``, and + ``is_full_time_college_student`` as ``bool``; + ``own_children_in_household`` and ``veterans_benefits`` as + ``float64``), in ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_eligibility_weights(person, weights) + stage_person = person.copy(deep=True) + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values + output = run_source_stage( + us_eligibility_inputs_stage_spec(), + tables={"person": stage_person}, + operation_handlers={ + "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, + }, + config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + ) + aligned = output.set_index("person_id").reindex(person["person_id"]) + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if aligned[column].isna().any(): + raise ValueError( + f"US eligibility-inputs stage output does not cover every " + f"person for {column!r}." + ) + + result = person.copy(deep=True) + bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if column in bool_columns: + result[column] = aligned[column].to_numpy(dtype=bool) + else: + result[column] = aligned[column].to_numpy(dtype=np.float64) + return result + + def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: """Run the ``eligibility_inputs`` manifest stage over a US frame. @@ -302,37 +413,17 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> if frame.schema != US_SCHEMA: raise ValueError("US eligibility inputs require the US schema.") person = frame.table("person") - have_all = all( - column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - ) - if have_all and _disabled_carries_signal(person): + if us_eligibility_inputs_person_carries_signal(person): return frame - stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values - output = run_source_stage( - us_eligibility_inputs_stage_spec(), - tables={"person": stage_person}, - operation_handlers={ - "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, - }, - config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + new_person = prepare_us_eligibility_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, ) - aligned = output.set_index("person_id").reindex(person["person_id"]) - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if aligned[column].isna().any(): - raise ValueError( - f"US eligibility-inputs stage output does not cover every " - f"person for {column!r}." - ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if column in bool_columns: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) - else: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) + tables["person"] = new_person return Frame( tables, frame.schema, @@ -343,15 +434,31 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> ) -def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: - """Weighted eligibility-share summary for gates and release manifests.""" +def us_eligibility_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_eligibility_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying the five + eligibility columns. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_eligibility_inputs_summary`. + """ + + weight_values = _validated_eligibility_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(mask: np.ndarray) -> float: - return float(weights[mask].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[mask].sum()) / total_weight if total_weight > 0 else 0.0 + ) disabled = person["is_disabled"].astype(bool).to_numpy() student = person["is_full_time_college_student"].astype(bool).to_numpy() @@ -377,31 +484,51 @@ def _share(mask: np.ndarray) -> float: } -def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: - """Require the eligibility surface to carry plausible distributions. +def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests.""" - Fails when a column is missing or constant, or when the weighted - disabled, full-time-student, parent, or veterans'-payment shares leave - their plausibility bands — each of which reproduces (or inverts) the - everyone-defaults failure of microcosm #244. + return us_eligibility_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) + + +def us_eligibility_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check eligibility-input plausibility bands and signal from a summary. + + The pure decision core of :func:`us_eligibility_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_eligibility_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the five output columns are present; missing columns + are a separate failure mode (see + :func:`us_eligibility_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. """ - person = frame.table("person") + validate_person_signal_summary( + summary, + family="eligibility", + outputs=US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "disabled_share": ("disabled_share_band", _DISABLED_SHARE_BAND), + "full_time_college_student_share": ( + "full_time_student_share_band", + _FULL_TIME_STUDENT_SHARE_BAND, + ), + "parent_share": ("parent_share_band", _PARENT_SHARE_BAND), + "veterans_benefits_share": ( + "veterans_benefits_share_band", + _VETERANS_BENEFITS_SHARE_BAND, + ), + }, + ) failures: list[str] = [] - missing = [ - column - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - if column not in person.columns - ] - if missing: - return GateResult( - name="eligibility_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) - - summary = us_eligibility_inputs_summary(frame) for column, count in summary["unique_counts"].items(): if count < 2: failures.append( @@ -434,3 +561,58 @@ def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_eligibility_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + The real-person-table counterpart of + :func:`us_eligibility_inputs_signal_gate`, composing + :func:`us_eligibility_inputs_person_summary` and + :func:`us_eligibility_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_eligibility_inputs_person_summary(person, weights) + return us_eligibility_inputs_gate_from_summary(summary) + + +def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + Fails when a column is missing or constant, or when the weighted + disabled, full-time-student, parent, or veterans'-payment shares leave + their plausibility bands — each of which reproduces (or inverts) the + everyone-defaults failure of microcosm #244. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + + summary = us_eligibility_inputs_summary(frame) + return us_eligibility_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py index df08aa857..528957104 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py @@ -59,7 +59,7 @@ SourceRuntimeError, run_source_stage, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -70,10 +70,13 @@ "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", "US_HOURS_WORKED_STAGE_NAME", "derive_us_hours_worked_from_manifest", + "us_hours_worked_gate_from_summary", + "us_hours_worked_person_summary", "us_hours_worked_signal_gate", "us_hours_worked_summary", "us_hours_worked_stage_spec", "with_us_hours_worked_inputs", + "with_us_hours_worked_person", ] US_HOURS_WORKED_STAGE_NAME = "hours_worked" @@ -233,8 +236,36 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> if have_all and _weekly_hours_carry_signal(person): return frame + produced = with_us_hours_worked_person( + person, + weights=frame.resolve_weights("person"), + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = produced + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def with_us_hours_worked_person( + person: pd.DataFrame, *, weights: Weights, seed: int, time_period: int +) -> pd.DataFrame: + """Run the same manifest handler on a declared person view and typed weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + if all(column in person for column in US_HOURS_WORKED_OUTPUT_COLUMNS) and ( + _weekly_hours_carry_signal(person) + ): + return person stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values output = run_source_stage( us_hours_worked_stage_spec(), tables={"person": stage_person}, @@ -251,24 +282,27 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> f"{column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + produced = person.copy() for column in US_HOURS_WORKED_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + produced[column] = aligned[column].to_numpy(dtype=np.float64) + return produced def us_hours_worked_summary(frame: Frame) -> dict[str, object]: """Weighted hours-distribution summary for gates and release manifests.""" - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + return us_hours_worked_person_summary( + frame.table("person"), weights=frame.resolve_weights("person") + ) + + +def us_hours_worked_person_summary( + person: pd.DataFrame, *, weights: Weights +) -> dict[str, object]: + """Observe the actual person view with canonically resolved, aligned weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + weights = np.asarray(weights.values, dtype=np.float64) weekly = pd.to_numeric( person["weekly_hours_worked_before_lsr"], errors="coerce" ).fillna(0.0) @@ -306,7 +340,6 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: """ person = frame.table("person") - failures: list[str] = [] missing = [ column for column in US_HOURS_WORKED_OUTPUT_COLUMNS @@ -321,7 +354,40 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: ) summary = us_hours_worked_summary(frame) - for column, count in summary["unique_counts"].items(): + return us_hours_worked_gate_from_summary(summary) + + +def us_hours_worked_gate_from_summary(summary: dict[str, object]) -> GateResult: + """Apply the original signal checks to actual typed producer observations.""" + if ( + not isinstance(summary, dict) + or set(summary) + != { + "worked_share", + "mean_weekly_hours_workers", + "worked_share_band", + "mean_weekly_hours_band", + "unique_counts", + } + or summary["worked_share_band"] != list(_WORKED_SHARE_BAND) + or summary["mean_weekly_hours_band"] != list(_MEAN_WEEKLY_HOURS_BAND) + or not isinstance(summary["unique_counts"], dict) + or set(summary["unique_counts"]) != set(US_HOURS_WORKED_OUTPUT_COLUMNS) + or any( + type(value) is not int or value < 0 + for value in summary["unique_counts"].values() + ) + or any( + type(summary[name]) not in (int, float) or not np.isfinite(summary[name]) + for name in ("worked_share", "mean_weekly_hours_workers") + ) + ): + raise ValueError("US hours-worked gate requires the complete original summary.") + failures: list[str] = [] + # Canonical artifact JSON sorts mappings; failure order remains the + # original declared output order, including after a cache roundtrip. + for column in US_HOURS_WORKED_OUTPUT_COLUMNS: + count = summary["unique_counts"][column] if count < 2: failures.append( f"{column}: constant column (one observed value) — the hours " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py index 1f83bd82c..f6f1dbbba 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py @@ -24,8 +24,11 @@ import hashlib import warnings +from collections.abc import Mapping +from dataclasses import dataclass from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Any import numpy as np @@ -63,6 +66,11 @@ "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", "derive_us_housing_inputs", + "AcsRentDonorPreparation", + "AcsRentRecipientPreparation", + "prepare_acs_rent_donor", + "prepare_acs_rent_recipient", + "finalize_acs_rent", "impute_us_pre_subsidy_rent", "impute_us_housing_assistance_to_puf_support", "load_acs_2022_rent_donor", @@ -465,14 +473,17 @@ def _constant_source_by_unit( return values.to_numpy(dtype=np.float64) -def derive_us_housing_inputs(frame: Frame) -> Frame: - """Carry the three exact ASEC housing/tenure inputs onto their entities.""" +def derive_us_housing_tables( + person: pd.DataFrame, household: pd.DataFrame, spm_unit: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Return copied household/SPM tables with the exact measured housing leaves. + + Inputs are the actual linked tables (or declared views) at the existing + housing derivation boundary. SPM raw values align through person_spm_unit_id; + returned group row order, indices, other columns and input tables are kept. + This helper neither reconstructs other entities nor changes raw code meaning. + """ - if frame.schema != US_SCHEMA: - raise ValueError("US housing inputs require the US schema.") - person = frame.table("person") - household = frame.table("household") - spm_unit = frame.table("spm_unit") _required_columns( person, ("SPM_CAPHOUSESUB", "SPM_TENMORTSTATUS", "person_spm_unit_id"), @@ -521,15 +532,30 @@ def derive_us_housing_inputs(frame: Frame) -> Frame: f"{unknown_spm_codes}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - tables["household"]["tenure_type"] = np.asarray( + household_result = household.copy() + spm_result = spm_unit.copy() + household_result["tenure_type"] = np.asarray( [_HOUSEHOLD_TENURE_MAP[code] for code in raw_household_codes], dtype=object ) - tables["spm_unit"]["receives_housing_assistance"] = subsidy > 0.0 - tables["spm_unit"]["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 - tables["spm_unit"]["spm_unit_tenure_type"] = np.asarray( + spm_result["receives_housing_assistance"] = subsidy > 0.0 + spm_result["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 + spm_result["spm_unit_tenure_type"] = np.asarray( [_SPM_TENURE_MAP[code] for code in raw_spm_codes], dtype=object ) + return household_result, spm_result + + +def derive_us_housing_inputs(frame: Frame) -> Frame: + """Carry the three exact ASEC housing/tenure inputs onto their entities.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US housing inputs require the US schema.") + household, spm_unit = derive_us_housing_tables( + frame.table("person"), frame.table("household"), frame.table("spm_unit") + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"] = household + tables["spm_unit"] = spm_unit return Frame( tables, frame.schema, @@ -703,38 +729,203 @@ def _encode_acs_predictors( ) -> tuple[pd.DataFrame, pd.DataFrame, tuple[str, ...]]: """Dummy-encode the retired QRF's string predictors without ordinality.""" + context = _rent_encoding_context(training) + return ( + _encode_rent_predictors(training, context=context), + _encode_rent_predictors(prediction, context=context), + context["predictors"], + ) + + +def _rent_encoding_context(training: pd.DataFrame) -> Mapping[str, object]: numeric_predictors = tuple( predictor for predictor in ACS_RENT_PREDICTORS if predictor not in _ACS_CATEGORICAL_PREDICTORS ) - encoded_training = training.loc[:, numeric_predictors].copy() - encoded_prediction = prediction.loc[:, numeric_predictors].copy() encoded_columns = list(numeric_predictors) + category_levels = {} for column in _ACS_CATEGORICAL_PREDICTORS: training_values = training[column].astype(str) - prediction_values = prediction[column].astype(str) levels = tuple(sorted(training_values.unique())) - unknown = sorted(set(prediction_values.unique()) - set(levels)) - if unknown: - raise ValueError( - f"ACS rent recipient {column!r} has donor-unsupported value(s): " - f"{unknown}." - ) if len(levels) < 2: raise ValueError( f"ACS rent donor categorical predictor {column!r} is constant." ) + category_levels[column] = levels + encoded_columns.extend(f"{column}__{level}" for level in levels[1:]) + return MappingProxyType( + { + "schema": "microcosm.us.acs_rent_encoding.v1", + "predictors": tuple(encoded_columns), + "category_levels": MappingProxyType(category_levels), + } + ) + + +def _validated_rent_encoding_context( + context: Mapping[str, object], +) -> Mapping[str, object]: + if ( + not isinstance(context, Mapping) + or set(context) != {"schema", "predictors", "category_levels"} + or context.get("schema") != "microcosm.us.acs_rent_encoding.v1" + ): + raise ValueError("Unsupported ACS rent donor encoding context.") + levels_by_column = context["category_levels"] + if not isinstance(levels_by_column, Mapping) or set(levels_by_column) != set( + _ACS_CATEGORICAL_PREDICTORS + ): + raise ValueError( + "ACS rent donor encoding must name its exact categorical columns." + ) + expected = [ + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ] + frozen_levels = {} + for column in _ACS_CATEGORICAL_PREDICTORS: + levels = levels_by_column[column] + if ( + not isinstance(levels, (tuple, list)) + or len(levels) < 2 + or any(not isinstance(value, str) for value in levels) + or list(levels) != sorted(set(levels)) + ): + raise ValueError( + "ACS rent donor encoding levels must be sorted unique strings." + ) + frozen_levels[column] = tuple(levels) + expected.extend(f"{column}__{level}" for level in levels[1:]) + if not isinstance(context["predictors"], (tuple, list)) or tuple( + context["predictors"] + ) != tuple(expected): + raise ValueError( + "ACS rent donor encoding predictor order differs from its levels." + ) + return MappingProxyType( + { + "schema": context["schema"], + "predictors": tuple(expected), + "category_levels": MappingProxyType(frozen_levels), + } + ) + + +def _encode_rent_predictors( + table: pd.DataFrame, *, context: Mapping[str, object] +) -> pd.DataFrame: + numeric = tuple( + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ) + encoded = table.loc[:, numeric].copy() + for column in _ACS_CATEGORICAL_PREDICTORS: + values = table[column].astype(str) + levels = context["category_levels"][column] + unknown = sorted(set(values.unique()) - set(levels)) + if unknown: + raise ValueError( + f"ACS rent recipient {column!r} has donor-unsupported value(s): {unknown}." + ) for level in levels[1:]: - dummy = f"{column}__{level}" - encoded_training[dummy] = training_values.eq(level).to_numpy( - dtype=np.float64 + encoded[f"{column}__{level}"] = values.eq(level).to_numpy(dtype=np.float64) + return encoded + + +@dataclass(frozen=True) +class AcsRentDonorPreparation: + """Actual sampled donor training rows and recipient-independent encoding.""" + + training: pd.DataFrame + predictors: tuple[str, ...] + context: Mapping[str, object] + + def __post_init__(self) -> None: + context = _validated_rent_encoding_context(self.context) + if ( + self.predictors != context["predictors"] + or not isinstance(self.training, pd.DataFrame) + or tuple(self.training.columns) + != (*self.predictors, "rent", _DONOR_WEIGHT_COLUMN) + ): + raise ValueError( + "ACS rent donor training columns differ from their encoding." ) - encoded_prediction[dummy] = prediction_values.eq(level).to_numpy( - dtype=np.float64 + object.__setattr__(self, "context", context) + + +@dataclass(frozen=True) +class AcsRentRecipientPreparation: + """Predictor rows and their household heads' original person positions. + + ``head_positions[i]`` belongs to household ``predictors.index[i]``; it + need not be sorted when person and household table orders differ. + """ + + predictors: pd.DataFrame + head_positions: np.ndarray + person_index: pd.Index + + def __post_init__(self) -> None: + positions = np.asarray(self.head_positions) + if ( + not isinstance(self.predictors, pd.DataFrame) + or not isinstance(self.person_index, pd.Index) + or positions.ndim != 1 + or not np.issubdtype(positions.dtype, np.integer) + or len(positions) != len(self.predictors) + or (positions < 0).any() + or (positions >= len(self.person_index)).any() + or len(np.unique(positions)) != len(positions) + ): + raise ValueError( + "ACS rent head positions must align to the original person rows." ) - encoded_columns.append(dummy) - return encoded_training, encoded_prediction, tuple(encoded_columns) + positions = positions.astype(np.int64, copy=True) + positions.flags.writeable = False + object.__setattr__(self, "head_positions", positions) + object.__setattr__(self, "person_index", self.person_index.copy(deep=True)) + + +def prepare_acs_rent_recipient( + frame: Frame, *, donor_context: Mapping[str, object] +) -> AcsRentRecipientPreparation: + """Encode actual head predictors against donor-owned categorical levels.""" + context = _validated_rent_encoding_context(donor_context) + features, head_mask = _recipient_head_features(frame) + head_positions = np.flatnonzero(head_mask) + head_households = pd.Index( + frame.person["person_household_id"].to_numpy()[head_positions] + ) + household_order = head_households.get_indexer(features.index) + if (household_order < 0).any(): + raise ValueError("ACS rent predictor household has no selected person head.") + return AcsRentRecipientPreparation( + _encode_rent_predictors(features, context=context), + head_positions[household_order], + frame.person.index, + ) + + +def finalize_acs_rent( + prepared: AcsRentRecipientPreparation, raw_rent: pd.Series | pd.DataFrame +) -> np.ndarray: + """Clip raw draws and scatter each household's value to its own head.""" + if not isinstance(prepared, AcsRentRecipientPreparation): + raise TypeError("ACS rent finalization requires recipient preparation.") + if isinstance(raw_rent, pd.DataFrame) and tuple(raw_rent.columns) == ("rent",): + raw_rent = raw_rent["rent"] + if not isinstance(raw_rent, pd.Series) or raw_rent.name != "rent": + raise ValueError("ACS rent finalization requires the named raw rent Series.") + if not raw_rent.index.equals(prepared.predictors.index): + raise ValueError( + "ACS rent raw prediction index differs from prepared household order." + ) + predicted = pd.to_numeric(raw_rent, errors="coerce").to_numpy(dtype=np.float64) + if not np.isfinite(predicted).all(): + raise ValueError("ACS rent QRF produced nonfinite predictions.") + person_rent = np.zeros(len(prepared.person_index), dtype=np.float64) + person_rent[prepared.head_positions] = np.maximum(predicted, 0.0) + return person_rent def _stable_string_hash(value: str) -> np.uint64: @@ -822,14 +1013,12 @@ def _archived_joint_training_sample( return sampled, sampled_masks -def impute_us_pre_subsidy_rent( - frame: Frame, - donor: pd.DataFrame, - *, - seed: int, - n_estimators: int = _DEFAULT_N_ESTIMATORS, -) -> np.ndarray: - """Draw annual ACS rent once per CPS household and place it on the head.""" +def prepare_acs_rent_donor(donor: pd.DataFrame) -> AcsRentDonorPreparation: + """Select and encode actual archived ACS rent support without recipients. + + The joint rent/real-estate-tax sampling, allocation masks, reset-index + order and household design weights retain their existing semantics. + """ required = ( *ACS_RENT_PREDICTORS, @@ -869,35 +1058,39 @@ def impute_us_pre_subsidy_rent( if float(fit_frame[_DONOR_WEIGHT_COLUMN].sum()) <= 0.0: raise ValueError("ACS rent donor sampled weights sum to zero.") + context = _rent_encoding_context(fit_frame) + encoded_training = _encode_rent_predictors(fit_frame, context=context) + encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) + encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( + dtype=np.float64 + ) + return AcsRentDonorPreparation(encoded_training, context["predictors"], context) + + +def impute_us_pre_subsidy_rent( + frame: Frame, + donor: pd.DataFrame, + *, + seed: int, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> np.ndarray: + """Draw annual ACS rent once per CPS household and place it on the head.""" + + prepared_donor = prepare_acs_rent_donor(donor) + global QRF if QRF is None: from importlib import import_module QRF = import_module("microcosm.fit").QRF - features, head_mask = _recipient_head_features(frame) - encoded_training, encoded_features, encoded_predictors = _encode_acs_predictors( - fit_frame, - features, - ) - encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) - encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( - dtype=np.float64 - ) + recipient = prepare_acs_rent_recipient(frame, donor_context=prepared_donor.context) fitted = QRF(n_estimators=int(n_estimators), seed=int(seed)).fit( - encoded_training, - predictors=list(encoded_predictors), + prepared_donor.training, + predictors=list(prepared_donor.predictors), targets=["rent"], weights=_DONOR_WEIGHT_COLUMN, ) - predicted = pd.to_numeric( - fitted.predict(encoded_features)["rent"], errors="coerce" - ).to_numpy(dtype=np.float64) - if not np.isfinite(predicted).all(): - raise ValueError("ACS rent QRF produced nonfinite predictions.") - predicted = np.maximum(predicted, 0.0) - person_rent = np.zeros(frame.n("person"), dtype=np.float64) - person_rent[head_mask] = predicted - return person_rent + return finalize_acs_rent(recipient, fitted.predict(recipient.predictors)) def _person_puf_predictors(frame: Frame) -> pd.DataFrame: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 8f2ea06b9..62d3e1362 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -15,7 +15,7 @@ import hashlib import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from functools import lru_cache from importlib.metadata import version @@ -38,12 +38,18 @@ from microcosm.build.us_runtime.adult_care import with_us_adult_care_inputs from microcosm.build.us_runtime.child_support import with_us_child_support_inputs from microcosm.build.us_runtime.childcare import with_us_childcare_inputs -from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs +from microcosm.build.us_runtime.cps_carried import ( + CpsCarriedTables, + derive_us_cps_carried_inputs, + derive_us_cps_carried_tables, +) from microcosm.build.us_runtime.disability_benefits import ( with_us_disability_benefits, ) from microcosm.build.us_runtime.education_inputs import with_us_education_inputs from microcosm.build.us_runtime.eligibility_inputs import ( + prepare_us_eligibility_person, + us_eligibility_inputs_person_carries_signal, with_us_eligibility_inputs, ) from microcosm.build.us_runtime.energy_subsidy import ( @@ -51,8 +57,11 @@ ) from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, + us_hours_worked_gate_from_summary, + us_hours_worked_person_summary, us_hours_worked_signal_gate, with_us_hours_worked_inputs, + with_us_hours_worked_person, ) from microcosm.build.us_runtime.housing_inputs import ( US_HOUSING_ASSISTANCE_PUF_MAX_TRAIN_SAMPLES, @@ -70,6 +79,7 @@ ) from microcosm.build.us_runtime.pregnancy import with_us_pregnancy_inputs from microcosm.build.us_runtime.prior_year_income import ( + prepare_us_prior_year_person, with_us_prior_year_income_inputs, ) from microcosm.build.us_runtime.puf_qrf_chain import PRIMARY_QRF_TARGET_ORDER @@ -84,6 +94,8 @@ with_us_qbi_input_reconciliation, ) from microcosm.build.us_runtime.relationship_inputs import ( + prepare_us_relationship_person, + us_relationship_inputs_person_carries_signal, with_us_relationship_inputs, ) from microcosm.build.us_runtime.retirement_contributions import ( @@ -121,7 +133,7 @@ from microcosm.build.us_runtime.workers_compensation import ( with_us_workers_compensation, ) -from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame import US_SCHEMA, Frame, Weights from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, VariableDependencyClosure, @@ -163,6 +175,18 @@ "finalize_multispine_source_inputs", "materialize_multispine_agreement_outputs", "materialize_pool_deferred_transfer_inputs", + "multispine_hours_worked_boundary_columns", + "multispine_housing_boundary_columns", + "multispine_housing_implementation_contract", + "multispine_housing_output_family", + "multispine_housing_source_selection", + "assert_multispine_source_merge_labels_supported", + "merge_multispine_housing_source_outputs", + "validate_multispine_housing_person_boundary", + "observe_multispine_hours_person", + "observe_multispine_hours_worked_inputs", + "validate_multispine_hours_person_boundary", + "validate_multispine_hours_worked_boundary", "pool_input_surface", "pool_engine_input_projection_receipt", "pool_remaining_stage_input_manifest", @@ -1792,32 +1816,872 @@ def prepare_multispine_source_inputs_for_clone( def _with_gated_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: """Run the shared hours kernel, then keep only pool-owned input leaves.""" + observed = _observe_us_hours_worked_inputs(frame) + gate = observed.receipt["hours_worked_signal_gate"] + if not gate["passed"]: + raise ValueError( + "Pool pre-clone hours-worked signal gate failed:\n " + + "\n ".join(gate["failures"]) + ) + return observed + + +def _observe_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Compute actual values and gate evidence without certifying the surface.""" + produced = with_us_hours_worked_inputs( frame, seed=POOL_RANDOM_SEED, time_period=POOL_TIME_PERIOD, ) gate = us_hours_worked_signal_gate(produced) - if not gate.passed: - raise ValueError( - "Pool pre-clone hours-worked signal gate failed:\n " - + "\n ".join(gate.failures) - ) pool_surface, removed = _drop_source_output_columns( produced, {"person": US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS}, ) return PoolStageOutput( pool_surface, + _hours_gate_receipt(gate, removed), + ) + + +def _hours_gate_receipt( + gate: GateResult, removed: Mapping[str, list[str]] +) -> dict[str, object]: + return { + "hours_worked_signal_gate": { + "name": gate.name, + "passed": gate.passed, + "failures": list(gate.failures), + "details": dict(gate.details), + }, + "pool_excluded_outputs_removed": dict(removed), + } + + +def multispine_hours_worked_boundary_columns() -> tuple[str, ...]: + """Declare source-boundary evidence without exposing role dispatch downstream.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_cps_carried_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone CPS evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_cps_carried_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_cps_carried_implementation_contract() -> dict[str, object]: + """Bind live provider projections actually consumed by this source operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "cps_carried" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("derive_us_cps_carried_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + } + + +def observe_multispine_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[CpsCarriedTables, dict[str, object]]: + """Use actual person/SPM views for the original source projection and merge.""" + boundary = validate_multispine_cps_carried_person_boundary( + person, metadata=metadata + ) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or pool_rows["spm_unit"] != len(spm_unit) + or len(weights) != len(person) + ): + raise ValueError("CPS-carried population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("CPS-carried population counts differ from memberships.") + if spm_unit.spm_unit_id.duplicated().any() or set(spm_unit.spm_unit_id) != set( + person.person_spm_unit_id + ): + raise ValueError("CPS-carried SPM rows differ from actual memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected_person = without_support_role_metadata(person.loc[mask], entity="person") + selected_spm = without_support_role_metadata( + spm_unit.loc[ + spm_unit.spm_unit_id.isin(selected_person.person_spm_unit_id) + ].reset_index(drop=True), + entity="spm_unit", + ) + # Match selected Frame weight validation without constructing a partial + # Frame carrying whole-population metadata or unobserved entity tables. + weights.with_values(weights.values[mask], kind=weights.kind) + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["cps_carried"] + selected = { + "person": selected_person.drop( + columns=_unavailable_output_columns(selected_person, outputs["person"]) + ), + "spm_unit": selected_spm.drop( + columns=_unavailable_output_columns(selected_spm, outputs["spm_unit"]) + ), + } + produced = derive_us_cps_carried_tables(selected["person"], selected["spm_unit"]) + merged = {} + merged_rows = {} + for entity, outcome in ( + ("person", produced.person), + ("spm_unit", produced.spm_unit), + ): + identity = US_SCHEMA.entity_id_column(entity) + _assert_source_table_identity( + selected[entity], + outcome, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + target = person if entity == "person" else spm_unit + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target.copy(), + outcome, + outputs[entity], + entity=entity, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + selected_rows = { + "person": len(selected_person), + **{ + group: int(selected_person[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="derive_us_cps_carried_inputs", + family="cps_carried", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["derive_us_cps_carried_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows=merged_rows, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return CpsCarriedTables( + merged["person"], merged["spm_unit"] + ), _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("derive_us_cps_carried_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_prior_year_boundary_columns() -> tuple[str, ...]: + """Declare the existing pre-clone evidence at its registered source owner.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_prior_year_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only prior-year join.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_prior_year_implementation_contract() -> dict[str, object]: + """Bind the live output and transient projections consumed by this operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "prior_year_income" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_prior_year_income_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_prior_year_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone join on its declared source person projection. + + Canonical person weights are selected in the original person order. Actual + memberships determine selected group counts; authenticated full counts + describe the unchanged pool. No incomplete Frame is constructed. Support + role fields are removed only from the validated ephemeral source view, so + the helper's support-QRF path cannot run at this boundary. + """ + boundary = validate_multispine_prior_year_person_boundary(person, metadata=metadata) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Prior-year population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("Prior-year population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + produced = prepare_us_prior_year_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_prior_year_income_inputs", + family="prior_year_income", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_prior_year_income_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_prior_year_income_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_relationship_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for relationship preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_eligibility_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for eligibility preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_relationship_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only relationship derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def validate_multispine_eligibility_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only eligibility derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def _validate_multispine_person_preparation_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_relationship_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_relationship_inputs", "relationship_inputs" + ) + + +def multispine_eligibility_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_eligibility_inputs", "eligibility_inputs" + ) + + +def _multispine_person_preparation_contract( + operator_name: str, family: str +) -> dict[str, object]: + """Project the live output/transient rosters and the executed options. + + These values move through imported registries without an edit here, so a + graph identity that omits them would not describe what actually ran. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family].items() + }, + "transient_outputs": _transient_source_outputs( + (operator_name,), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_relationship_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone relationship derivation on its person projection. + + Canonical person weights are selected in the original person order and the + authenticated full counts describe the unchanged pool. No incomplete Frame + is constructed. Support role fields are removed only from the validated + ephemeral source view. The public pass-through predicate decides whether + the always-recomputing helper runs, so the wrapper's order — signal check + before weight validation and before any raw-column requirement — holds. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_relationship_inputs", + family="relationship_inputs", + label="Relationship-input", + carries_signal=lambda table: us_relationship_inputs_person_carries_signal( + table + ), + prepare=lambda table, values: prepare_us_relationship_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def observe_multispine_eligibility_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone eligibility derivation on its person projection. + + The same registered boundary, canonical weight selection and full-pool ID + merge as the relationship observer. The legacy partial-null pass-through + rule — all five outputs present and observed disability varying — is the + public predicate's, not this helper's, so the graph cannot strengthen or + weaken it. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_eligibility_inputs", + family="eligibility_inputs", + label="Eligibility-input", + carries_signal=lambda table: us_eligibility_inputs_person_carries_signal(table), + prepare=lambda table, values: prepare_us_eligibility_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def _observe_multispine_person_preparation( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], + operator_name: str, + family: str, + label: str, + carries_signal: Callable[[pd.DataFrame], bool], + prepare: Callable[[pd.DataFrame, np.ndarray], pd.DataFrame], +) -> tuple[pd.DataFrame, dict[str, object]]: + """The shared person-preparation source projection, run and full-pool merge.""" + boundary = _validate_multispine_person_preparation_boundary( + person, metadata=metadata + ) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError(f"{label} population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError(f"{label} population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + # The wrapper returns its frame untouched on a signal-carrying surface, + # reading neither weights nor raw source columns. Preserve that order. + produced = ( + selected + if carries_signal(selected) + else prepare(selected, selected_weights.values) + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name=operator_name, + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name=operator_name, + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name=operator_name, + family=family, + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS[operator_name], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=(operator_name,), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_hours_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by the hours projection. + + These values can change through imported registries without an edit to + this module. Keep their actual projection in graph implementation identity. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "hours_worked" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_hours_worked_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + } + + +def validate_multispine_hours_worked_boundary(frame: Frame) -> dict[str, object]: + """Validate the existing pre-clone boundary before graph hours derivation.""" + return validate_multispine_hours_person_boundary( + frame.person, metadata=frame.metadata + ) + + +def validate_multispine_hours_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + available = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(available.sum())} + + +def observe_multispine_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Run one registered source operator and retain its uncertified gate evidence. + + The graph's mandatory gate must accept this evidence before any downstream + context is released. Projection, structure checks and ID merge are the + exact same source-boundary implementation used by the direct path. + """ + return _run_source_operator_chain( + frame, + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + operators={"with_us_hours_worked_inputs": _observe_us_hours_worked_inputs}, + ) + + +def observe_multispine_hours_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the registered hours projection using only actual person columns. + + ``weights`` must be the pool's canonically resolved person vector. Full + entity counts come from the authenticated population descriptor; selected + group counts use actual membership IDs, as Frame.select prunes each group + to the referenced IDs. No partial Frame carries whole-pool metadata. + """ + boundary = validate_multispine_hours_person_boundary(person, metadata=metadata) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Measured hours population counts or weights are misaligned.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["hours_worked"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + selected_rows = {"person": len(selected)} + selected_rows.update( { - "hours_worked_signal_gate": { - "name": gate.name, - "passed": True, - "failures": [], - "details": dict(gate.details), - }, - "pool_excluded_outputs_removed": removed, + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + } + ) + if any(selected_rows[entity] > pool_rows[entity] for entity in pool_rows): + raise ValueError("Measured hours membership counts exceed the population.") + produced = with_us_hours_worked_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + gate = us_hours_worked_gate_from_summary( + us_hours_worked_person_summary(produced, weights=selected_weights) + ) + removed = sorted(US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS & set(produced.columns)) + produced = produced.drop(columns=removed) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_hours_worked_inputs", + family="hours_worked", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_hours_worked_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt=_hours_gate_receipt( + gate, {"person": removed} if removed else {} + ), + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_housing_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone housing evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_housing_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_housing_source_selection( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> tuple[pd.Series, dict[str, object]]: + """Return the registered CPS person mask and its boundary receipt. + + The mask is the same one the direct source-operator chain applies, so a + graph FILTER built from it selects exactly the historical projection rows. + """ + boundary = validate_multispine_housing_person_boundary(person, metadata=metadata) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return mask, boundary + + +def multispine_housing_output_family() -> Mapping[str, frozenset[str]]: + """Return the live registered housing output roster.""" + return PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["housing_inputs"] + + +def multispine_housing_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by this source operator.""" + contract = POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"] + return { + "outputs": { + entity: sorted(columns) + for entity, columns in multispine_housing_output_family().items() }, + "transient_outputs": _transient_source_outputs( + ("with_us_housing_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "formula_owned_outputs": { + entity: sorted( + set(columns) & set(_FORMULA_OWNED_SOURCE_OUTPUTS.get(entity, ())) + ) + for entity, columns in multispine_housing_output_family().items() + }, + "execution_scope": contract.execution_scope, + "phases": list(contract.phases), + "family": contract.family, + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def assert_multispine_source_merge_labels_supported( + target: pd.DataFrame, + columns: Iterable[str], + *, + entity: str, + operator_name: str, + source: pd.DataFrame | None = None, +) -> None: + """Refuse the inherited label-based merge seam instead of working around it. + + :func:`_merge_source_person_or_group_outputs` writes a non-boolean output + through ``target.loc[target.index[positions]]``. With repeated pandas index + labels that assignment reaches every row sharing a selected label, so the + combination of duplicated labels and that write is refused at this explicit + scope boundary. Fixing the shared merge is a separate approved scope; + unique labels retain exact original parity. + + The merge picks its write path from the **produced** values, so pass + ``source`` to decide from the same series it does. Without ``source`` this + falls back to the incumbent rather than the actual write dtype: it can + over-refuse a boolean-producing column whose incumbent is not boolean, and + it would under-refuse a non-boolean-producing column whose incumbent is. + """ + if not target.index.has_duplicates: + return + + def writes_by_label(column: str) -> bool: + decided = ( + target[column] + if source is None or column not in source + else (source[column]) + ) + return not _is_physical_boolean_series(decided) + + affected = sorted( + column for column in columns if column in target and writes_by_label(column) + ) + if affected: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot merge " + f"{entity!r} output(s) {affected} onto repeated index labels; the " + "shared label-based merge would write every row sharing a label. " + "Fixing that inherited seam is a separate reviewed scope." + ) + + +def merge_multispine_housing_source_outputs( + tables: Mapping[str, pd.DataFrame], + produced: Mapping[str, pd.DataFrame], + *, + pool_rows: Mapping[str, int], + selected_rows: Mapping[str, int], + evidence_rows: int, +) -> tuple[dict[str, pd.DataFrame], dict[str, int], dict[str, object]]: + """Merge the five declared housing leaves by entity ID and receipt the order. + + ``tables`` are isolated full-pool copies and ``produced`` the projected + source rows carrying every declared output. Column values, the ID merge and + the ordered receipt come from the existing registered helpers; this wrapper + adds no new formula and changes no selection semantics. + """ + outputs = multispine_housing_output_family() + if set(produced) != set(outputs) or set(outputs) - set(tables): + raise ValueError( + "US housing source merge requires its three declared output tables." + ) + merged: dict[str, pd.DataFrame] = { + entity: table for entity, table in tables.items() + } + merged_rows: dict[str, int] = {} + for entity in sorted(outputs): + columns = outputs[entity] + identity = US_SCHEMA.entity_id_column(entity) + assert_multispine_source_merge_labels_supported( + merged[entity], + columns, + entity=entity, + operator_name="with_us_housing_inputs", + source=produced[entity], + ) + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + merged[entity], + produced[entity], + columns, + entity=entity, + entity_id=identity, + operator_name="with_us_housing_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_housing_inputs", + family="housing_inputs", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"], + before_rows=dict(pool_rows), + available_rows=dict(selected_rows), + output_rows=dict(selected_rows), + merged_rows=merged_rows, + declared_outputs=dict(outputs), + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return ( + merged, + merged_rows, + _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_housing_inputs",), + evidence_rows=evidence_rows, + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), ) @@ -2265,43 +3129,21 @@ def _run_source_operator_chain( f"{operator_name!r}: input={before_rows}, output={after_rows}." ) receipts.append( - { - "order_index": order_index, - "operator": operator_name, - "family": family, - "phase": phase, - "execution_scope": contract.execution_scope, - "pool_input_rows": before_rows, - "operator_input_rows": available_rows, - "cps_available_rows": ( - available_rows - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else None - ), - "operator_output_rows": output_rows, - "merged_rows": merged_rows, - "operator_projection": { - "selection": ( - _CPS_SOURCE_EVIDENCE_COLUMN - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else _WHOLE_POOL_EXECUTION_SCOPE - ), - "lineage_state_persisted": ( - contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE - ), - "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, - }, - "output_columns": { - entity: sorted(columns) - for entity, columns in declared_outputs.items() - if columns - }, - "formula_owned_outputs_removed": formula_owned_removed, - "kernel_receipt": dict(kernel_receipt), - "overlap_ownership": ( - dict(overlap_ownership) if overlap_ownership is not None else None - ), - } + _source_operator_receipt( + order_index=order_index, + operator_name=operator_name, + family=family, + phase=phase, + contract=contract, + before_rows=before_rows, + available_rows=available_rows, + output_rows=output_rows, + merged_rows=merged_rows, + declared_outputs=declared_outputs, + formula_owned_removed=formula_owned_removed, + kernel_receipt=kernel_receipt, + overlap_ownership=overlap_ownership, + ) ) uses_cps_source = any( POOL_OPERATOR_CONTRACTS[name].execution_scope == _CPS_SOURCE_EXECUTION_SCOPE @@ -2309,38 +3151,126 @@ def _run_source_operator_chain( ) return PoolStageOutput( current, - { - "phase": phase, - "operator_order": list(operator_names), - "cps_source_evidence": ( - { - "column": _CPS_SOURCE_EVIDENCE_COLUMN, - "person_rows": int( - _cps_source_evidence_mask(frame, phase=phase).sum() - ), - } + _source_chain_receipt( + phase=phase, + operator_names=operator_names, + evidence_rows=( + int(_cps_source_evidence_mask(frame, phase=phase).sum()) if uses_cps_source else None ), - "transient_outputs_carried_through_clone": ( - _transient_source_outputs(operator_names, output_families) - if phase == _PRE_CLONE_PHASE - else {} + receipts=receipts, + output_families=output_families, + ), + ) + + +def _source_chain_receipt( + *, + phase: str, + operator_names: tuple[str, ...], + evidence_rows: int | None, + receipts: list[dict[str, object]], + output_families: Mapping[str, Mapping[str, frozenset[str]]], +) -> dict[str, object]: + return { + "phase": phase, + "operator_order": list(operator_names), + "cps_source_evidence": ( + { + "column": _CPS_SOURCE_EVIDENCE_COLUMN, + "person_rows": evidence_rows, + } + if evidence_rows is not None + else None + ), + "transient_outputs_carried_through_clone": ( + _transient_source_outputs(operator_names, output_families) + if phase == _PRE_CLONE_PHASE + else {} + ), + "suboperators": receipts, + } + + +def _source_operator_receipt( + *, + order_index: int, + operator_name: str, + family: str, + phase: str, + contract: SourceOperatorContract, + before_rows: Mapping[str, int], + available_rows: Mapping[str, int], + output_rows: Mapping[str, int], + merged_rows: Mapping[str, int], + declared_outputs: Mapping[str, frozenset[str]], + formula_owned_removed: Mapping[str, list[str]], + kernel_receipt: Mapping[str, object], + overlap_ownership: Mapping[str, object] | None, +) -> dict[str, object]: + return { + "order_index": order_index, + "operator": operator_name, + "family": family, + "phase": phase, + "execution_scope": contract.execution_scope, + "pool_input_rows": before_rows, + "operator_input_rows": available_rows, + "cps_available_rows": ( + available_rows + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else None + ), + "operator_output_rows": output_rows, + "merged_rows": merged_rows, + "operator_projection": { + "selection": ( + _CPS_SOURCE_EVIDENCE_COLUMN + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else _WHOLE_POOL_EXECUTION_SCOPE ), - "suboperators": receipts, + "lineage_state_persisted": ( + contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE + ), + "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, }, - ) + "output_columns": { + entity: sorted(columns) + for entity, columns in declared_outputs.items() + if columns + }, + "formula_owned_outputs_removed": formula_owned_removed, + "kernel_receipt": dict(kernel_receipt), + "overlap_ownership": ( + dict(overlap_ownership) if overlap_ownership is not None else None + ), + } def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: - manifest = frame.metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) + _assert_source_person_boundary( + frame.table(frame.schema.person_entity), + metadata=frame.metadata, + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _assert_source_person_boundary( + person: pd.DataFrame, + *, + metadata: Mapping[str, object], + person_entity: str, + phase: str, +) -> None: + manifest = metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) if not isinstance(manifest, Mapping): raise ValueError( "Multispine source operators require the immutable spine assembly " "manifest before any source derivation." ) - person = frame.table(frame.schema.person_entity) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) if clone_column not in person: raise ValueError( "Multispine source operators require post-assembly clone provenance; " @@ -2370,7 +3300,19 @@ def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: """Select CPS lineage only from a raw column unavailable on ACS.""" - person = frame.table(frame.schema.person_entity) + return _cps_person_evidence_mask( + frame.table(frame.schema.person_entity), + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _cps_person_evidence_mask( + person: pd.DataFrame, + *, + person_entity: str, + phase: str, +) -> pd.Series: if _CPS_SOURCE_EVIDENCE_COLUMN not in person: raise ValueError( "Multispine source operators require raw CPS evidence column " @@ -2385,7 +3327,7 @@ def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: "Multispine source operators found no CPS-evidenced person rows in " f"{_CPS_SOURCE_EVIDENCE_COLUMN!r}." ) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) clone_index = pd.to_numeric(person[clone_column], errors="coerce") evidenced_clones = set(clone_index.loc[available].astype(int).tolist()) invalid_evidence = ( @@ -2679,15 +3621,39 @@ def _assert_source_operator_structure( ) for entity in before.entities: entity_id = before.schema.entity_id_column(entity) - before_ids = before.table(entity)[entity_id] - after_ids = after.table(entity)[entity_id] - if after_ids.duplicated().any() or set(after_ids.tolist()) != set( - before_ids.tolist() - ): - raise ValueError( - f"Multispine source operator {operator_name!r} changed structural " - f"{entity_id!r} values." - ) + _assert_source_table_identity( + before.table(entity), + after.table(entity), + entity_id=entity_id, + operator_name=operator_name, + ) + + +def _assert_source_table_identity( + before: pd.DataFrame, + after: pd.DataFrame, + *, + entity_id: str, + operator_name: str, +) -> None: + before_ids = before[entity_id] + after_ids = after[entity_id] + if after_ids.duplicated().any() or set(after_ids.tolist()) != set( + before_ids.tolist() + ): + raise ValueError( + f"Multispine source operator {operator_name!r} changed structural " + f"{entity_id!r} values." + ) + + +def _unavailable_output_columns( + table: pd.DataFrame, + columns: frozenset[str], +) -> list[str]: + return [ + column for column in columns if column in table and table[column].isna().all() + ] def _without_unavailable_output_columns( @@ -2701,11 +3667,7 @@ def _without_unavailable_output_columns( for entity, columns in outputs.items(): if entity not in tables: continue - unavailable = [ - column - for column in columns - if column in tables[entity] and tables[entity][column].isna().all() - ] + unavailable = _unavailable_output_columns(tables[entity], columns) if unavailable: tables[entity] = tables[entity].drop(columns=unavailable) dropped = True @@ -2743,83 +3705,14 @@ def _merge_source_operator_outputs( target = tables[entity] source = operated.table(entity) entity_id = pool.schema.entity_id_column(entity) - if entity_id not in target or entity_id not in source: - raise ValueError( - f"Multispine source operator {operator_name!r} cannot align " - f"{entity!r} without {entity_id!r}." - ) - if source[entity_id].duplicated().any(): - raise ValueError( - f"Multispine source operator {operator_name!r} returned duplicate " - f"{entity_id!r} values." - ) - missing_outputs = sorted(set(columns) - set(source.columns)) - if missing_outputs: - raise ValueError( - f"Multispine source operator {operator_name!r} did not emit its " - f"declared {entity!r} output(s): {missing_outputs}." - ) - source_by_id = source.set_index(entity_id) - target_ids = target[entity_id] - eligible = target_ids.isin(source_by_id.index) - if int(eligible.sum()) != len(source): - raise ValueError( - f"Multispine source operator {operator_name!r} output IDs do not " - f"align one-to-one with the {entity!r} pool." - ) - for column in sorted(columns): - source_values = source_by_id[column] - aligned = source_values.reindex(target_ids) - source_is_boolean = _is_physical_boolean_series(source_values) - if source_is_boolean: - positions = np.flatnonzero(eligible.to_numpy()) - aligned_boolean = pd.Series( - pd.array(aligned, dtype="boolean"), - index=target.index, - name=column, - ) - if column not in target: - target[column] = aligned_boolean - continue - incumbent = target[column] - invalid_incumbent = incumbent.dropna().map( - lambda value: not isinstance(value, (bool, np.bool_)) - ) - if invalid_incumbent.any(): - offending_types = sorted( - { - f"{type(value).__module__}.{type(value).__qualname__}" - for value in incumbent.dropna().loc[invalid_incumbent] - } - ) - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"physical booleans for {entity}.{column}, but the pool " - "materialized observed non-boolean values with " - f"dtype {incumbent.dtype!s}: {offending_types}." - ) - merged_boolean = pd.Series( - pd.array(incumbent, dtype="boolean"), - index=target.index, - name=column, - ) - merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array - target[column] = merged_boolean - continue - if column in target and pd.api.types.is_bool_dtype(target[column].dtype): - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"non-boolean values for boolean-materialized " - f"{entity}.{column}; source dtype={source_values.dtype!s}." - ) - if column not in target: - target[column] = aligned.to_numpy() - else: - positions = np.flatnonzero(eligible.to_numpy()) - target.loc[target.index[positions], column] = aligned.iloc[ - positions - ].to_numpy() - merged_rows[entity] = int(eligible.sum()) + tables[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target, + source, + columns, + entity=entity, + entity_id=entity_id, + operator_name=operator_name, + ) merged = Frame( tables, @@ -2832,6 +3725,95 @@ def _merge_source_operator_outputs( return merged, merged_rows +def _merge_source_person_or_group_outputs( + target: pd.DataFrame, + source: pd.DataFrame, + columns: frozenset[str], + *, + entity: str, + entity_id: str, + operator_name: str, +) -> tuple[pd.DataFrame, int]: + """The shared source-boundary ID merge; callers supply an isolated target.""" + if entity_id not in target or entity_id not in source: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot align " + f"{entity!r} without {entity_id!r}." + ) + if source[entity_id].duplicated().any(): + raise ValueError( + f"Multispine source operator {operator_name!r} returned duplicate " + f"{entity_id!r} values." + ) + missing_outputs = sorted(set(columns) - set(source.columns)) + if missing_outputs: + raise ValueError( + f"Multispine source operator {operator_name!r} did not emit its " + f"declared {entity!r} output(s): {missing_outputs}." + ) + source_by_id = source.set_index(entity_id) + target_ids = target[entity_id] + eligible = target_ids.isin(source_by_id.index) + if int(eligible.sum()) != len(source): + raise ValueError( + f"Multispine source operator {operator_name!r} output IDs do not " + f"align one-to-one with the {entity!r} pool." + ) + for column in sorted(columns): + source_values = source_by_id[column] + aligned = source_values.reindex(target_ids) + source_is_boolean = _is_physical_boolean_series(source_values) + if source_is_boolean: + positions = np.flatnonzero(eligible.to_numpy()) + aligned_boolean = pd.Series( + pd.array(aligned, dtype="boolean"), + index=target.index, + name=column, + ) + if column not in target: + target[column] = aligned_boolean + continue + incumbent = target[column] + invalid_incumbent = incumbent.dropna().map( + lambda value: not isinstance(value, (bool, np.bool_)) + ) + if invalid_incumbent.any(): + offending_types = sorted( + { + f"{type(value).__module__}.{type(value).__qualname__}" + for value in incumbent.dropna().loc[invalid_incumbent] + } + ) + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"physical booleans for {entity}.{column}, but the pool " + "materialized observed non-boolean values with " + f"dtype {incumbent.dtype!s}: {offending_types}." + ) + merged_boolean = pd.Series( + pd.array(incumbent, dtype="boolean"), + index=target.index, + name=column, + ) + merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array + target[column] = merged_boolean + continue + if column in target and pd.api.types.is_bool_dtype(target[column].dtype): + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"non-boolean values for boolean-materialized " + f"{entity}.{column}; source dtype={source_values.dtype!s}." + ) + if column not in target: + target[column] = aligned.to_numpy() + else: + positions = np.flatnonzero(eligible.to_numpy()) + target.loc[target.index[positions], column] = aligned.iloc[ + positions + ].to_numpy() + return target, int(eligible.sum()) + + def _is_physical_boolean_series(values: pd.Series) -> bool: """Recognize boolean values without treating numeric 0/1 as booleans.""" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index e3e18961e..310b9953c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -14,7 +14,6 @@ import pandas as pd -from microcosm.build.us_runtime.acs_transfer import ACS_DERIVED_TRANSFER_INPUTS from microcosm.build.us_runtime.adult_care import US_ADULT_CARE_OUTPUT_COLUMNS from microcosm.build.us_runtime.child_support import ( US_CHILD_SUPPORT_OUTPUT_COLUMNS, @@ -59,20 +58,6 @@ US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, ) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, -) -from microcosm.build.us_runtime.puf_support import ( - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, -) from microcosm.build.us_runtime.qbi_inputs import US_QBI_RECONCILED_PERSON_COLUMNS from microcosm.build.us_runtime.relationship_inputs import ( US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, @@ -96,6 +81,20 @@ ) from microcosm.frame import Frame +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS, + PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, + PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) + __all__ = [ "FORMULA_OWNED_SOURCE_COLUMNS", "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py index ab3e56d6f..7983db5bf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py @@ -45,7 +45,7 @@ support_role_series, without_support_role_metadata, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -63,6 +63,7 @@ "US_PRIOR_YEAR_INCOME_STAGE_NAME", "derive_us_prior_year_income_from_manifest", "impute_us_prior_year_income_to_puf_support_from_manifest", + "prepare_us_prior_year_person", "us_prior_year_income_signal_gate", "us_prior_year_income_source_reconciliation_gate", "us_prior_year_income_stage_spec", @@ -213,11 +214,7 @@ def _previous_year_availability_match_survival_factor(frame: Frame) -> float: f"spine manifest version {version!r}." ) factor = manifest.get("sample_fraction") - if ( - type(factor) is not float - or not np.isfinite(factor) - or not 0.0 < factor <= 1.0 - ): + if type(factor) is not float or not np.isfinite(factor) or not 0.0 < factor <= 1.0: raise ValueError( "US prior-year-income availability requires a finite production " "stacked sample_fraction in (0, 1]." @@ -695,34 +692,57 @@ def _replace_person_table(frame: Frame, person: pd.DataFrame) -> Frame: ) -def with_us_prior_year_income_inputs( - frame: Frame, +def _prior_year_person_is_complete(person: pd.DataFrame) -> bool: + return ( + not has_support_role_metadata(person, entity="person") + and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) + and not all( + column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS + ) + ) + + +def prepare_us_prior_year_person( + person: pd.DataFrame, *, + weights: Weights, seed: int, time_period: int, -) -> Frame: - """Materialize adjacent-year earnings and PUF-support replacements.""" + predictors: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Run the actual prior-year stage on a real person table. - if frame.schema != US_SCHEMA: - raise ValueError("US prior-year income requires the US schema.") - person = frame.table("person") - has_support_roles = has_support_role_metadata(person, entity="person") - has_raw_sources = all( - column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS - ) - if ( - not has_support_roles - and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) - and not has_raw_sources - ): - return frame + ``weights`` must be resolved for this person order by the caller. The + source-only, pre-clone path requires no other entity table. The legacy + post-clone path supplies the actual person/tax-unit predictor projection; + it is never reconstructed from placeholder entities here. Monetary values + retain the existing source-period treatment; this helper does not uprate. + """ + if not isinstance(weights, Weights): + raise TypeError("US prior-year person weights require canonical Weights.") + if len(weights) != len(person): + raise ValueError("US prior-year weights must match the person row order.") + if _prior_year_person_is_complete(person): + if predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") + return person + has_support_roles = has_support_role_metadata(person, entity="person") stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values if has_support_roles: - predictors = _person_prior_year_income_predictors(frame) + if ( + not isinstance(predictors, pd.DataFrame) + or not predictors.index.equals(person.index) + or tuple(predictors.columns) != _PUF_PREDICTORS + ): + raise ValueError( + "Prior-year support inputs require the actual aligned PUF predictors." + ) for column in _PUF_PREDICTORS: stage_person[_PUF_PREDICTOR_PREFIX + column] = predictors[column].to_numpy() + elif predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") output = run_source_stage( us_prior_year_income_stage_spec(), tables={"person": stage_person}, @@ -742,6 +762,34 @@ def with_us_prior_year_income_inputs( ) if has_support_roles: output = output.drop(columns=[_FORMULA_OWNED_OUTPUT]) + return output + + +def with_us_prior_year_income_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize adjacent-year earnings and PUF-support replacements.""" + if frame.schema != US_SCHEMA: + raise ValueError("US prior-year income requires the US schema.") + person = frame.table("person") + if _prior_year_person_is_complete(person): + return frame + weights = frame.resolve_weights("person") + predictors = ( + _person_prior_year_income_predictors(frame) + if has_support_role_metadata(person, entity="person") + else None + ) + output = prepare_us_prior_year_person( + person, + weights=weights, + seed=seed, + time_period=time_period, + predictors=predictors, + ) return _replace_person_table(frame, output) @@ -892,12 +940,10 @@ def us_prior_year_income_signal_gate(frame: Frame) -> GateResult: if match_survival_factor != 1.0: authored_lower, authored_upper = summary[availability_band_key] applied_floor = float(authored_lower) * match_survival_factor - availability_band_key = ( - "previous_year_income_available_applied_share_band" + availability_band_key = "previous_year_income_available_applied_share_band" + summary["previous_year_income_available_sampled_match_survival_factor"] = ( + match_survival_factor ) - summary[ - "previous_year_income_available_sampled_match_survival_factor" - ] = match_survival_factor summary["previous_year_income_available_applied_floor"] = applied_floor summary[availability_band_key] = [applied_floor, authored_upper] failures: list[str] = [] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_capital_gains_tail.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_capital_gains_tail.py index 995795bf5..ee1c8ea56 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_capital_gains_tail.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_capital_gains_tail.py @@ -37,6 +37,31 @@ ) from microcosm.frame import US_SCHEMA, Frame, Weights +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN as PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN as PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN as PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN as PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN as PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS as PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS as PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, +) +from .operator_column_contracts import ( + PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN as PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, +) + __all__ = [ "PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", "PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", @@ -91,13 +116,6 @@ PUF_CAPITAL_GAINS_TAIL_REFERENCE_QUANTILE = 0.999 PUF_CAPITAL_GAINS_TAIL_ASEC_CAPITAL_GAINS_TOPCODE = 1_999_998.0 -PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS = ( - "short_term_capital_gains", - "long_term_capital_gains_before_response", - "long_term_capital_gains_on_collectibles", - "non_sch_d_capital_gains", -) -PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS = ("unrecaptured_section_1250_gain",) _JOINT_VECTOR_COLUMNS = ( *PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, *PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, @@ -120,18 +138,6 @@ _TAIL_AGI_BAND_LABEL_COLUMN = "_puf_capital_gains_tail_agi_band_label" _TAIL_SYNTHETIC_COLUMN = "_puf_capital_gains_tail_is_synthetic" -PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN = "puf_capital_gains_tail_transfer_applied" -PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN = "puf_capital_gains_tail_donor_source_id" -PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN = ( - "puf_capital_gains_tail_donor_is_synthetic" -) -PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN = ( - "puf_capital_gains_tail_donor_filing_status_code" -) -PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN = ( - "puf_capital_gains_tail_donor_agi_band_index" -) -PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN = "puf_capital_gains_tail_transfer_weight" _FILING_STATUS_BY_CODE = { 1: "SINGLE", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py index 13f5c35c5..ca5dc186b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py @@ -40,6 +40,22 @@ from microcosm.build.us_runtime.take_up_contract import load_take_up_contract from microcosm.frame import US_SCHEMA, Frame +from .operator_column_contracts import ( + _GENERAL_QUALIFICATION_FLAGS as _GENERAL_QUALIFICATION_FLAGS, +) +from .operator_column_contracts import ( + _SSTB_QUALIFICATION_FLAG as _SSTB_QUALIFICATION_FLAG, +) +from .operator_column_contracts import ( + US_QBI_BOOLEAN_OUTPUT_COLUMNS as US_QBI_BOOLEAN_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_NONNEGATIVE_OUTPUT_COLUMNS as US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_OUTPUT_COLUMNS as US_QBI_OUTPUT_COLUMNS, +) + __all__ = [ "QBI_ARCHIVED_ASSUMPTIONS_URL", "QBI_ARCHIVED_CLONE_URL", @@ -87,40 +103,6 @@ US_QBI_STAGE_NAME = "puf_tax_detail" -_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( - "estate_income_would_be_qualified", - "farm_operations_income_would_be_qualified", - "farm_rent_income_would_be_qualified", - "partnership_s_corp_income_would_be_qualified", - "rental_income_would_be_qualified", - "self_employment_income_would_be_qualified", -) -_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" -US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( - *_GENERAL_QUALIFICATION_FLAGS, - _SSTB_QUALIFICATION_FLAG, - # Keep the classifier last so the chained QRF can condition the SSTB draw - # on the qualification flags it must agree with. - "business_is_sstb", -) -US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) -US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( - *US_QBI_BOOLEAN_OUTPUT_COLUMNS, - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_self_employment_income_before_lsr", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) US_QBI_NONCONSTANT_PERSON_COLUMNS = US_QBI_OUTPUT_COLUMNS _SELF_EMPLOYMENT_COLUMN = "self_employment_income_before_lsr" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py index 3695df002..f4f391509 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py @@ -15,6 +15,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -32,6 +33,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -41,6 +45,11 @@ "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_RELATIONSHIP_INPUTS_STAGE_NAME", "derive_us_relationship_inputs_from_manifest", + "prepare_us_relationship_person", + "us_relationship_inputs_person_carries_signal", + "us_relationship_inputs_gate_from_summary", + "us_relationship_inputs_person_gate", + "us_relationship_inputs_person_summary", "us_relationship_inputs_signal_gate", "us_relationship_inputs_stage_spec", "us_relationship_inputs_summary", @@ -182,8 +191,13 @@ def derive_us_relationship_inputs_from_manifest( return result -def _relationship_surface_carries_signal(frame: Frame) -> bool: - person = frame.table("person") +def us_relationship_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the exact incumbent pass-through decision from the real person table. + + All three outputs must be present and each must have multiple observed + values. Preserve the legacy null/constant rule; this does not validate raw + source columns or resolve weights, and is not a scientific signal gate. + """ if any(column not in person for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS): return False return all( @@ -192,22 +206,70 @@ def _relationship_surface_carries_signal(frame: Frame) -> bool: ) -def with_us_relationship_inputs( - frame: Frame, +def _relationship_surface_carries_signal(frame: Frame) -> bool: + return us_relationship_inputs_person_carries_signal(frame.table("person")) + + +def _validated_relationship_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US relationship-input weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US relationship-input weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US relationship-input weights must be nonnegative.") + return array + + +def prepare_us_relationship_person( + person: pd.DataFrame, + weights: np.ndarray, *, seed: int, time_period: int, -) -> Frame: - """Materialize measured ASEC relationship inputs on a US frame.""" - - if frame.schema != US_SCHEMA: - raise ValueError("US relationship inputs require the US schema.") - if _relationship_surface_carries_signal(frame): - return frame - - person = frame.table("person") +) -> pd.DataFrame: + """Derive measured ASEC relationship inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_relationship_inputs`: + it always runs the ``relationship_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that pass-through + decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns (``PH_SEQ``, ``P_SEQ``, ``A_MARITL``), in its own index + and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with ``is_household_head``, ``is_separated``, + and ``is_surviving_spouse`` attached as ``bool`` columns, in + ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_relationship_weights(person, weights) stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values output = run_source_stage( us_relationship_inputs_stage_spec(), tables={"person": stage_person}, @@ -224,9 +286,34 @@ def with_us_relationship_inputs( f"for {column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + result = person.copy(deep=True) for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) + result[column] = aligned[column].to_numpy(dtype=bool) + return result + + +def with_us_relationship_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize measured ASEC relationship inputs on a US frame.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US relationship inputs require the US schema.") + if _relationship_surface_carries_signal(frame): + return frame + + person = frame.table("person") + new_person = prepare_us_relationship_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = new_person return Frame( tables, frame.schema, @@ -237,16 +324,35 @@ def with_us_relationship_inputs( ) -def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: - """Return weighted relationship shares and one-head invariants.""" +def us_relationship_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_relationship_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying + ``is_household_head``, ``is_separated``, and + ``is_surviving_spouse``. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_relationship_inputs_summary`. + """ + + weight_values = _validated_relationship_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(column: str) -> float: values = person[column].fillna(False).astype(bool).to_numpy() - return float(weights[values].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[values].sum()) / total_weight + if total_weight > 0 + else 0.0 + ) household_column = ( "person_household_id" if "person_household_id" in person else "PH_SEQ" @@ -276,24 +382,50 @@ def _share(column: str) -> float: } -def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: - """Require plausible signal and exactly one ASEC head per household.""" +def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants.""" - person = frame.table("person") - missing = [ - column - for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS - if column not in person - ] - if missing: - return GateResult( - name="relationship_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) + return us_relationship_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) - summary = us_relationship_inputs_summary(frame) + +def us_relationship_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check relationship-input plausibility bands and invariants from a summary. + + The pure decision core of :func:`us_relationship_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_relationship_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the three output columns are present; missing + columns are a separate failure mode (see + :func:`us_relationship_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. + """ + + validate_person_signal_summary( + summary, + family="relationship", + outputs=US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "household_head_share": ( + "household_head_share_band", + _HOUSEHOLD_HEAD_SHARE_BAND, + ), + "separated_share": ("separated_share_band", _SEPARATED_SHARE_BAND), + "surviving_spouse_share": ( + "surviving_spouse_share_band", + _SURVIVING_SPOUSE_SHARE_BAND, + ), + }, + invariants=("households_without_exactly_one_head", "separated_and_surviving"), + ) failures: list[str] = [] for share_key, band_key, label in ( ( @@ -330,3 +462,51 @@ def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_relationship_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require plausible signal and exactly one ASEC head per household. + + The real-person-table counterpart of + :func:`us_relationship_inputs_signal_gate`, composing + :func:`us_relationship_inputs_person_summary` and + :func:`us_relationship_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_person_summary(person, weights) + return us_relationship_inputs_gate_from_summary(summary) + + +def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: + """Require plausible signal and exactly one ASEC head per household.""" + + person = frame.table("person") + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_summary(frame) + return us_relationship_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py index ce6b6e296..4f39da34b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py @@ -8,14 +8,17 @@ from __future__ import annotations +import hashlib import re from collections.abc import Mapping +from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import Any import numpy as np import pandas as pd -from microcosm.build.us_runtime.puf_support import ( +from microcosm.build.us_runtime.operator_column_contracts import ( PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, ) from microcosm.build.us_runtime.support_provenance import ( @@ -36,7 +39,14 @@ Weights, ) -__all__ = ["assemble_spines"] +__all__ = [ + "SpinePreparation", + "SpineHarmonization", + "assemble_spines", + "prepare_spines", + "stack_survey_spines", + "harmonize_spine_weights", +] _CHANNEL_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _SUPPORT_CLONE_INDEX = 0 @@ -80,11 +90,146 @@ def assemble_spines( provenance, columns, or ID spaces violate the assembly contract. """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + harmonized = _harmonize_spine_values( + prepared.tables["household"], prepared.values, prepared.context + ) + # These are newly composed per-source records, not a carried Frame log. + # Table provenance remains owned by the independent stacking result. + assembled_mass_log = harmonized.mass_log + result = Frame( + prepared.tables, + US_SCHEMA, + {"household": harmonized.weights}, + prepared.strata, + mass_log=assembled_mass_log, + metadata=prepared.metadata, + ) + validate_assembly_provenance(result, boundary="spine assembly output") + anchor_mass = prepared.context["incoming_masses"][mass_anchor_channel] + if not np.isclose( + result.weights_for("household").total, anchor_mass, rtol=_SHARE_RTOL, atol=0.0 + ): + raise RuntimeError("Spine assembly failed to conserve anchor household mass.") + return result + + +@dataclass(frozen=True) +class SpinePreparation: + """Actual DESIGN-weight tables plus an immutable assembly context. + + The context schema determines which subsequent weight operation can use it. + This numerical container does not authenticate source issuance or membership. + """ + + frame: Frame + context: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame): + raise TypeError("SpinePreparation.frame must be a Frame.") + if self.frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Spine preparation requires actual DESIGN weights.") + if not isinstance(self.context, Mapping): + raise ValueError("Spine preparation context must be a mapping.") + object.__setattr__(self, "context", _freeze_context(self.context)) + + +@dataclass(frozen=True) +class SpineHarmonization: + """Computed importance weights and exact ordered legacy operator history.""" + + weights: Weights + mass_log: tuple[MassChangeRecord, ...] + + +@dataclass(frozen=True) +class _StackedTables: + tables: Mapping[str, pd.DataFrame] + strata: pd.Series + values: np.ndarray + context: Mapping[str, object] + metadata: Mapping[str, object] + mass_log: tuple[MassChangeRecord, ...] + + +def _freeze_context(value: object) -> object: + if isinstance(value, Mapping): + return MappingProxyType( + {key: _freeze_context(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_context(item) for item in value) + return value + + +def _numeric_digest(values: np.ndarray, dtype: str) -> str: + array = np.ascontiguousarray(values, dtype=dtype) + return hashlib.sha256(memoryview(array).cast("B")).hexdigest() + + +def _channel_digest(values: np.ndarray) -> str: + digest = hashlib.sha256() + for value in values: + if not isinstance(value, str): + raise ValueError("Prepared household channels must be strings.") + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + return digest.hexdigest() + + +def _stack_spine_tables( + spines: Mapping[str, Frame], + *, + household_mass_shares: Mapping[str, float], + mass_anchor_channel: str, + require_design: bool = False, +) -> _StackedTables: ordered_channels = _validated_channels(spines, mass_anchor_channel) shares = _validated_shares(household_mass_shares, ordered_channels) frames = {channel: spines[channel] for channel in ordered_channels} + prepared = _stack_source_tables(frames, ordered_channels, require_design) + context = dict(prepared.context) + context.update( + schema="microcosm.us.spine-preparation.v1", + household_mass_shares=shares, + mass_anchor_channel=mass_anchor_channel, + incoming_masses={ + channel: float(frames[channel].weights_for("household").total) + for channel in ordered_channels + }, + ) + return _StackedTables( + prepared.tables, + prepared.strata, + prepared.values, + _freeze_context(context), + prepared.metadata, + prepared.mass_log, + ) + + +def _stack_source_tables( + spines: Mapping[str, Frame], + ordered_channels: tuple[str, ...], + require_design: bool, +) -> _StackedTables: + """Preserve source values while remapping only structural collisions.""" + frames = {channel: spines[channel] for channel in ordered_channels} for channel, frame in frames.items(): _validate_source_frame(frame, channel=channel) + if ( + require_design + and frame.weights_for("household").kind is not WeightKind.DESIGN + ): + raise ValueError( + f"Spine {channel!r} preparation requires actual DESIGN weights." + ) _validate_shared_column_dtypes(frames) offsets = _id_offsets(frames, ordered_channels) @@ -106,51 +251,286 @@ def assemble_spines( column_orders=column_orders, ) - anchor_mass = frames[mass_anchor_channel].weights_for("household").total - weights, mass_log = _assembled_household_weights( - frames, - ordered_channels=ordered_channels, - shares=shares, - anchor_mass=anchor_mass, + values = np.concatenate( + [ + frames[channel].weights_for("household").values + for channel in ordered_channels + ] ) - household_order = group_orders.get("household") - if household_order is not None: - weights = Weights(weights.values[household_order], weights.kind) - weights = _with_exact_total(weights, anchor_mass) - + order = group_orders.get("household") + if order is not None: + values = values[order] + household = tables["household"] + source_ids_digest = hashlib.sha256() + for channel in ordered_channels: + source_ids = np.ascontiguousarray( + prepared[channel]["household"]["household_id"].to_numpy(), dtype=" SpinePreparation: + """Stack actual DESIGN sources without performing importance allocation. + + Legacy ``assemble_spines`` continues to accept its existing mixed-kind + inputs. This new graph-facing seam never relabels such inputs as DESIGN. + """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + require_design=True, + ) + return _design_preparation(prepared) + + +def stack_survey_spines(spines: Mapping[str, Frame]) -> SpinePreparation: + """Stack original DESIGN survey weights before domain allocation. + + Channels are ordered lexically, independent of mapping insertion order. + Every household weight, source age and measured value is carried unchanged; + only structural IDs are remapped and source provenance is added. No survey + total is an anchor, no shares are assigned, and no mass is normalized. + + This is the numerical stacking seam. A graph source owner must separately + authenticate the original issuances, complete membership and publisher + anchors. Sampling and population-domain allocation are subsequent declared + operations. The returned v1 survey-stack context is deliberately refused + by the legacy anchor-total harmonizer. + """ + channels = _validated_source_channels(spines) + prepared = _stack_source_tables(spines, channels, require_design=True) + return _design_preparation(prepared) + + +def _design_preparation(prepared: _StackedTables) -> SpinePreparation: + frame = Frame( + prepared.tables, + US_SCHEMA, + {"household": Weights(prepared.values, WeightKind.DESIGN)}, + prepared.strata, + mass_log=prepared.mass_log, + metadata=prepared.metadata, ) - if not np.isclose( - result.weights_for("household").total, - anchor_mass, - rtol=_SHARE_RTOL, - atol=0.0, + validate_assembly_provenance(frame, boundary="spine preparation output") + return SpinePreparation(frame, prepared.context) + + +def harmonize_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + context: Mapping[str, object], +) -> SpineHarmonization: + """Compute importance allocation from declared DESIGN views and context. + + Input digests bind exact normalized values and ordered IDs/channels. + The explicit permutation restores source-order sums and correction rows. + The result includes legacy per-source logs, not an executor graph ledger. + """ + if not isinstance(weights, Weights) or weights.kind is not WeightKind.DESIGN: + raise ValueError("Spine harmonization requires actual DESIGN weights.") + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" ): - raise RuntimeError("Spine assembly failed to conserve anchor household mass.") - return result + raise ValueError("Unsupported spine preparation context.") + kinds = context.get("source_weight_kinds") + ordered = context.get("ordered_channels") + if ( + not isinstance(kinds, Mapping) + or not isinstance(ordered, (tuple, list)) + or set(kinds) != set(ordered) + or any(kind != WeightKind.DESIGN.value for kind in kinds.values()) + ): + raise ValueError("Spine preparation context does not declare DESIGN sources.") + return _harmonize_spine_values(household, weights.values, context) + + +def _validated_spine_inputs( + household: pd.DataFrame, values: np.ndarray, context: Mapping[str, object] +) -> tuple: + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" + ): + raise ValueError("Unsupported spine preparation context.") + if not isinstance(household, pd.DataFrame) or not { + "household_id", + support_channel_column("household"), + }.issubset(household): + raise ValueError("Harmonization requires household IDs and source channels.") + ids = household["household_id"].to_numpy() + channels = household[support_channel_column("household")].to_numpy() + if not np.issubdtype(ids.dtype, np.integer) or len(values) != len(ids): + raise ValueError("Prepared household IDs/weights are malformed.") + bindings = { + "household_ids_sha256": _numeric_digest(ids, " SpineHarmonization: + ordered, shares, counts, source_values, source_channels, order, anchor_mass = ( + _validated_spine_inputs(household, values, context) + ) + result, logs, allocated, start = [], [], 0.0, 0 + for index, channel in enumerate(ordered): + count = counts[channel] + existing_values = source_values[start : start + count] + if not np.all(source_channels[start : start + count] == channel): + raise ValueError( + "Prepared permutation does not restore source channel order." + ) + start += count + existing_mass = float(existing_values.sum()) + if existing_mass != float(context["incoming_masses"][channel]): + raise ValueError("Prepared source-order mass changed.") + target = ( + anchor_mass - allocated + if index == len(ordered) - 1 + else anchor_mass * shares[channel] + ) + scaled = _values_to_total(existing_values, target) + result.append(scaled) + allocated += float(scaled.sum()) + logs.extend( + MassChangeRecord(**dict(record)) + for record in context["source_mass_logs"][channel] + ) + logs.append( + MassChangeRecord( + entity="household", + old_total=existing_mass, + new_total=float(scaled.sum()), + declared_factor=target / existing_mass, + reason=f"allocated {channel!r} source mass in pre-operator spine assembly", + ) + ) + combined = np.concatenate(result) + if order is not None: + combined = combined[order] + return SpineHarmonization( + _with_exact_total(Weights(combined, WeightKind.IMPORTANCE), anchor_mass), + tuple(logs), + ) def _validated_channels( spines: Mapping[str, Frame], mass_anchor_channel: str, ) -> tuple[str, ...]: + channels = _validated_source_channels(spines) + if ( + not isinstance(mass_anchor_channel, str) + or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None + ): + raise ValueError( + "mass_anchor_channel must be a stable lower-snake-case identifier." + ) + if mass_anchor_channel not in channels: + raise ValueError( + f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." + ) + return ( + mass_anchor_channel, + *(channel for channel in channels if channel != mass_anchor_channel), + ) + + +def _validated_source_channels(spines: Mapping[str, Frame]) -> tuple[str, ...]: if not isinstance(spines, Mapping): raise TypeError(f"spines must be a mapping, got {type(spines).__name__}.") if len(spines) < 2: @@ -171,21 +551,7 @@ def _validated_channels( f"{PUF_TAX_DETAIL_SUPPORT_CHANNEL!r} is a clone operator channel, " "not a peer household spine." ) - if ( - not isinstance(mass_anchor_channel, str) - or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None - ): - raise ValueError( - "mass_anchor_channel must be a stable lower-snake-case identifier." - ) - if mass_anchor_channel not in spines: - raise ValueError( - f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." - ) - return ( - mass_anchor_channel, - *sorted(channel for channel in channels if channel != mass_anchor_channel), - ) + return tuple(sorted(channels)) def _validated_shares( @@ -394,6 +760,11 @@ def _id_offsets( ) offsets[channel][entity] = offset remapped = ids if offset == 0 else ids + offset + if int(remapped.max()) > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID: + raise ValueError( + f"Spine {channel!r} {entity!r} collision remapping exceeds " + f"the clone-safe bound {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}." + ) accumulated[entity] = np.concatenate([used, remapped]) return offsets @@ -493,45 +864,6 @@ def _combined_tables( return tables, group_orders -def _assembled_household_weights( - frames: Mapping[str, Frame], - *, - ordered_channels: tuple[str, ...], - shares: Mapping[str, float], - anchor_mass: float, -) -> tuple[Weights, tuple[MassChangeRecord, ...]]: - values: list[np.ndarray] = [] - mass_log: list[MassChangeRecord] = [] - allocated = 0.0 - for index, channel in enumerate(ordered_channels): - existing = frames[channel].weights_for("household") - target = ( - anchor_mass - allocated - if index == len(ordered_channels) - 1 - else anchor_mass * shares[channel] - ) - scaled = _values_to_total(existing.values, target) - values.append(scaled) - allocated += float(scaled.sum()) - factor = target / existing.total - mass_log.extend(frames[channel].mass_log) - mass_log.append( - MassChangeRecord( - entity="household", - old_total=existing.total, - new_total=float(scaled.sum()), - declared_factor=factor, - reason=( - f"allocated {channel!r} source mass in pre-operator spine assembly" - ), - ) - ) - return ( - Weights(np.concatenate(values), WeightKind.IMPORTANCE), - tuple(mass_log), - ) - - def _values_to_total(values: np.ndarray, target: float) -> np.ndarray: result = np.asarray(values, dtype=np.float64) * ( target / float(np.asarray(values, dtype=np.float64).sum()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311..1e62b8c2d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -26,10 +26,11 @@ ``w_i' = w_i * share_s * M_anchor / M_s`` where ``M_s`` is arm ``s``'s incoming household mass and ``M_anchor`` is the -mass-anchor arm's incoming mass. For the seeded ACS sample, -``M_acs_sample ~= fraction * M_acs_full``, so the allocation factor contains -the inverse-sampling upweighting ``1 / fraction`` automatically; the realized -per-arm scale factors are receipted rather than assumed. +mass-anchor arm's incoming mass. Production sampling first normalizes each +source by its full-source mass divided by its realized sampled mass. For +unequal source weights, that ratio is not generally the inverse realized +household inclusion count ``N / n``. Legacy pilot controls allocate directly +from the sampled ACS mass. Both paths receipt their actual scale factors. """ from __future__ import annotations @@ -40,7 +41,6 @@ import math import os import pickle -import struct import sys from collections import Counter from collections.abc import Callable, Mapping, Sequence @@ -80,6 +80,12 @@ ) from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes from microcosm.build.source_manifest import load_source_manifest +from microcosm.build.table_identity import ( + TABLE_VALUES_DIGEST_CODEC as _LATE_TABLE_DIGEST_CODEC, +) +from microcosm.build.table_identity import ( + table_values_sha256 as _late_table_values_sha256, +) from microcosm.build.us_runtime import ( post_transfer_calibration as post_transfer_calibration_runtime, ) @@ -183,7 +189,13 @@ puf_tax_detail_tail_bound_quantiles_identity, validate_puf_clone_attachment, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.build.us_runtime.spine_assembly import ( + _freeze_context, + _validated_spine_inputs, + assemble_spines, + harmonize_spine_weights, + prepare_spines, +) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_CLONE_INDEX, @@ -220,7 +232,7 @@ us_late_producer_schedule_receipt, ) from microcosm.fit import Regime -from microcosm.frame import US_SCHEMA, Frame, WeightKind +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord, WeightKind, Weights __all__ = [ "ACS_STACKED_SUPPORT_CHANNEL", @@ -249,6 +261,11 @@ "StackedLateProducerResult", "StackedPostPufTransferResult", "StackedSpineResult", + "StackedSpinePreparation", + "StackedSpineHarmonization", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + "finish_stacked_spine", "assemble_stacked_spine", "assert_stacked_tail_cells_preserved", "by_origin_battery", @@ -553,7 +570,47 @@ def _acs_native_group_quarters_receipt( } -def assemble_stacked_spine( +@dataclass(frozen=True) +class _SampledStack: + spines: Mapping[str, Frame] + shares: Mapping[str, float] + incoming_masses: Mapping[str, float] + sampling: Mapping[str, object] + manifest_version: int + mass_anchor_channel: str + + +@dataclass(frozen=True) +class StackedSpinePreparation: + """DESIGN preparation and a distinct pre-harmonization receipt.""" + + frame: Frame + receipt: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame) or not isinstance(self.receipt, Mapping): + raise TypeError( + "StackedSpinePreparation requires a Frame and receipt mapping." + ) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + _validate_stacked_preparation(self.frame, self.receipt) + + +@dataclass(frozen=True) +class StackedSpineHarmonization: + """Actual weights and completed context, with ordered legacy history.""" + + weights: Weights + metadata: Mapping[str, object] + receipt: Mapping[str, object] + legacy_mass_log: tuple[MassChangeRecord, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_context(self.metadata)) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + + +def _sample_stacked_inputs( asec: Frame, acs: Frame, *, @@ -563,27 +620,8 @@ def assemble_stacked_spine( sample_seed: int | None = None, household_mass_shares: Mapping[str, float] | None = None, mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, -) -> StackedSpineResult: - """Assemble uniformly sampled ASEC and ACS survey arms into one spine. - - Production callers provide the single ``sample_fraction`` and - ``sample_seed`` scale-ladder controls. The exact same fraction is applied - independently to both survey arms at whole-household grain; each sampled - arm is then normalized back to its full-source household mass before the - reviewed :func:`assemble_spines` harmonization. This preserves each arm's - composition and prevents the anchor population from shrinking with the - rung. PUF donors are not accepted here and therefore remain unsampled. - - ``acs_sample_fraction``/``acs_sample_seed`` retain the reviewed version-1 - pilot contract for reproducibility only: ASEC remains full and ACS alone - is sampled. Supplying pilot and production controls together fails - closed. - - Returns: - A validated :class:`StackedSpineResult` whose receipt mirrors the - frozen manifest as a JSON-ready mapping. - """ - +) -> _SampledStack: + """Run the unchanged pilot or production sampling/normalization controls.""" shares = ( dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES) if household_mass_shares is None @@ -688,6 +726,287 @@ def assemble_stacked_spine( }, } + return _SampledStack( + { + BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, + ACS_STACKED_SUPPORT_CHANNEL: sampled_acs, + }, + shares, + incoming_masses, + sampling_manifest, + manifest_version, + mass_anchor_channel, + ) + + +def assemble_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpineResult: + """Compose actual sampling, stacking and importance harmonization. + + Production controls normalize each sampled source by M_full/M_sample. + This ratio is not generally inverse realized household inclusion N/n. + Legacy pilot controls and accepted source weight kinds remain unchanged. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + if all( + frame.weights_for("household").kind is WeightKind.DESIGN + for frame in sampled.spines.values() + ): + return finish_stacked_spine(_prepare_sampled_stack(sampled)) + # Existing direct callers may supply calibrated or mixed source kinds. + # They keep their historical path, never relabeled as a DESIGN anchor. + return _finish_legacy_sampled_stack(sampled) + + +def prepare_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpinePreparation: + """Sample/normalize actual DESIGN sources and stack them before allocation. + + Returns a dedicated preparation receipt, never an incomplete completed + stacked-spine manifest. No harmonized weights are precomputed. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + return _prepare_sampled_stack(sampled) + + +def _prepare_sampled_stack(sampled: _SampledStack) -> StackedSpinePreparation: + prepared = prepare_spines( + sampled.spines, + household_mass_shares=sampled.shares, + mass_anchor_channel=sampled.mass_anchor_channel, + ) + samples = sampled.sampling.get( + "survey_samples", + {ACS_STACKED_SUPPORT_CHANNEL: sampled.sampling.get("acs_sample")}, + ) + full_masses = { + channel: float(samples[channel]["incoming_household_mass"]) + if channel in samples + else float(sampled.incoming_masses[channel]) + for channel in sampled.spines + } + receipt = { + "schema": "microcosm.us.stacked-spine-preparation.v1", + "stacked_manifest_version": sampled.manifest_version, + "sampling": sampled.sampling, + "full_source_masses": full_masses, + "household_mass_shares": sampled.shares, + "mass_anchor_channel": sampled.mass_anchor_channel, + "assembly_context": prepared.context, + "assembly_metadata": prepared.frame.metadata, + "acs_native_group_quarters": _acs_native_group_quarters_receipt( + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], prepared.frame + ), + } + return StackedSpinePreparation(prepared.frame, receipt) + + +def _validate_stacked_preparation(frame: Frame, receipt: Mapping[str, object]) -> None: + boundary = "stacked spine preparation" + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Stacked spine preparation requires actual DESIGN weights.") + if STACKED_SPINE_MANIFEST_KEY in frame.metadata: + raise ValueError( + "A preparation must not carry a completed stacked spine manifest." + ) + validate_assembly_provenance(frame, boundary=boundary) + _validate_preparation_receipt(receipt) + if _json_ready(receipt["assembly_metadata"]) != _json_ready(frame.metadata): + raise ValueError("Prepared assembly metadata differs from the actual frame.") + _validated_spine_inputs( + frame.table("household"), + frame.weights_for("household").values, + receipt["assembly_context"], + ) + version, sampling = receipt["stacked_manifest_version"], receipt["sampling"] + production = version == _STACKED_SPINE_MANIFEST_VERSION + fraction = sampling["sample_fraction" if production else "acs_sample_fraction"] + seed = sampling["sample_seed" if production else "acs_sample_seed"] + _validate_fraction(fraction) + _validate_seed(seed) + samples = ( + sampling["survey_samples"] + if production + else {ACS_STACKED_SUPPORT_CHANNEL: sampling["acs_sample"]} + ) + expected = ( + {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + if production + else {ACS_STACKED_SUPPORT_CHANNEL} + ) + if set(samples) != expected: + raise ValueError( + "Prepared sampling receipts do not cover their declared sources." + ) + for channel, sample in samples.items(): + _validate_survey_sample_receipt( + frame, + channel=channel, + fraction=fraction, + seed=seed, + sample=sample, + boundary=boundary, + require_normalization=production, + ) + if float(receipt["full_source_masses"][channel]) != float( + sample["incoming_household_mass"] + ): + raise ValueError( + "Prepared full-source mass differs from its sampling receipt." + ) + _validate_stacked_clone_role_lifecycle(frame, boundary=boundary) + _validated_acs_native_group_quarters_masks(frame, receipt, boundary=boundary) + + +def _validate_preparation_receipt(preparation: Mapping[str, object]) -> None: + if ( + not isinstance(preparation, Mapping) + or preparation.get("schema") != "microcosm.us.stacked-spine-preparation.v1" + ): + raise ValueError("Unsupported stacked-spine preparation receipt.") + if ( + preparation.get("stacked_manifest_version") + not in _SUPPORTED_STACKED_SPINE_MANIFEST_VERSIONS + ): + raise ValueError("Unsupported prepared stacked-spine manifest version.") + for field_name in ( + "sampling", + "full_source_masses", + "household_mass_shares", + "assembly_context", + "assembly_metadata", + "acs_native_group_quarters", + ): + if not isinstance(preparation.get(field_name), Mapping): + raise ValueError( + f"Prepared stacked-spine field {field_name!r} must be a mapping." + ) + context = preparation["assembly_context"] + if preparation.get("mass_anchor_channel") != context.get( + "mass_anchor_channel" + ) or preparation["household_mass_shares"] != context.get("household_mass_shares"): + raise ValueError("Prepared stack controls disagree with the assembly context.") + if set(preparation["full_source_masses"]) != { + BASE_ASEC_SUPPORT_CHANNEL, + ACS_STACKED_SUPPORT_CHANNEL, + }: + raise ValueError("Prepared full-source masses must name ASEC and ACS.") + + +def harmonize_stacked_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + preparation: Mapping[str, object], +) -> StackedSpineHarmonization: + """Compute importance weights and completed metadata from declared views. + + The preparation producer validates full source/person lineage. This helper + binds its household views and performs allocation without restacking or + reading a precomputed final-weight vector. The returned legacy mass log is + distinct from a graph executor's structural transition history. + """ + _validate_preparation_receipt(preparation) + result = harmonize_spine_weights( + household=household, weights=weights, context=preparation["assembly_context"] + ) + shares = preparation["household_mass_shares"] + incoming = preparation["assembly_context"]["incoming_masses"] + anchor = preparation["mass_anchor_channel"] + harmonization = _harmonization_receipt_from_views( + household, + result.weights, + shares=shares, + anchor_mass=incoming[anchor], + incoming_masses=incoming, + ) + manifest = { + "version": preparation["stacked_manifest_version"], + **preparation["sampling"], + "household_mass_shares": { + channel: float(share) for channel, share in shares.items() + }, + "mass_anchor_channel": anchor, + "weight_harmonization": harmonization, + "acs_native_group_quarters": preparation["acs_native_group_quarters"], + } + metadata = { + **preparation["assembly_metadata"], + STACKED_SPINE_MANIFEST_KEY: manifest, + } + return StackedSpineHarmonization( + result.weights, metadata, _json_ready(manifest), result.mass_log + ) + + +def finish_stacked_spine(prepared: StackedSpinePreparation) -> StackedSpineResult: + """Complete a validated preparation, preserving exact legacy Frame history.""" + if not isinstance(prepared, StackedSpinePreparation): + raise TypeError("finish_stacked_spine requires a StackedSpinePreparation.") + _validate_stacked_preparation(prepared.frame, prepared.receipt) + result = harmonize_stacked_spine_weights( + household=prepared.frame.table("household"), + weights=prepared.frame.weights_for("household"), + preparation=prepared.receipt, + ) + frame = Frame( + {entity: prepared.frame.table(entity) for entity in prepared.frame.entities}, + prepared.frame.schema, + {"household": result.weights}, + prepared.frame.strata, + mass_log=result.legacy_mass_log, + metadata=result.metadata, + ) + validated = validate_stacked_spine_frame( + frame, boundary="stacked spine assembly output" + ) + return StackedSpineResult(frame, _json_ready(validated)) + + +def _finish_legacy_sampled_stack(sampled: _SampledStack) -> StackedSpineResult: + sampled_asec, sampled_acs = ( + sampled.spines[BASE_ASEC_SUPPORT_CHANNEL], + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], + ) + shares, incoming_masses = sampled.shares, sampled.incoming_masses + sampling_manifest, manifest_version = sampled.sampling, sampled.manifest_version + mass_anchor_channel = sampled.mass_anchor_channel assembled = assemble_spines( { BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, @@ -1632,9 +1951,25 @@ def _harmonization_receipt( anchor_mass: float, incoming_masses: Mapping[str, float], ) -> dict[str, dict[str, float]]: - household = assembled.table("household") + return _harmonization_receipt_from_views( + assembled.table("household"), + assembled.weights_for("household"), + shares=shares, + anchor_mass=anchor_mass, + incoming_masses=incoming_masses, + ) + + +def _harmonization_receipt_from_views( + household: pd.DataFrame, + household_weights: Weights, + *, + shares: Mapping[str, float], + anchor_mass: float, + incoming_masses: Mapping[str, float], +) -> dict[str, dict[str, float]]: channel_values = household[support_channel_column("household")].astype(str) - weights = np.asarray(assembled.weights_for("household").values, dtype=np.float64) + weights = np.asarray(household_weights.values, dtype=np.float64) receipt: dict[str, dict[str, float]] = {} for channel, share in shares.items(): incoming = float(incoming_masses[channel]) @@ -5104,7 +5439,6 @@ def validate_stacked_post_puf_transfer_receipt( ) -_LATE_TABLE_DIGEST_CODEC = "canonical_scalar_v1" _LATE_PRIMARY_QRF_INPUT_BINDING_ARTIFACT_KIND = ( "populace_us_stacked_late_primary_qrf_input_binding" ) @@ -5112,259 +5446,6 @@ def validate_stacked_post_puf_transfer_receipt( _LATE_RESOURCE_SEMANTICS_ARTIFACT_KIND = ( "populace_us_stacked_late_producer_resource_semantics" ) -_LATE_TABLE_DIGEST_CHUNK_ROWS = 65_536 - - -def _late_digest_part( - digest, - *, - domain: str, - payload: bytes | bytearray | memoryview | np.ndarray, -) -> None: - """Append one length-framed, domain-separated byte field to a digest.""" - - domain_bytes = domain.encode("utf-8") - payload_view = memoryview(payload) - if payload_view.format != "B" or payload_view.ndim != 1: - payload_view = payload_view.cast("B") - digest.update(struct.pack(" np.ndarray: - """Return a contiguous, explicitly little-endian numeric byte source.""" - - array = np.asarray(values) - array = array.astype(array.dtype.newbyteorder("<"), copy=False) - return np.ascontiguousarray(array) - - -def _late_scalar_bytes(value: object) -> bytes: - """Encode one supported object scalar without lossy intermediary hashes.""" - - missing = pd.isna(value) - if isinstance(missing, (bool, np.bool_)) and bool(missing): - return b"null" - if isinstance(value, (bool, np.bool_)): - return b"bool\x01" if bool(value) else b"bool\x00" - if isinstance(value, (int, np.integer)): - return b"integer\x00" + str(int(value)).encode("ascii") - if isinstance(value, (float, np.floating)): - if isinstance(value, np.floating) and value.dtype.itemsize > 8: - raise TypeError( - "US late-producer content digest does not support object " - f"floating scalar {value.dtype!s}." - ) - return b"float64\x00" + struct.pack(" 16: - raise TypeError( - "US late-producer content digest does not support object " - f"complex scalar {value.dtype!s}." - ) - numeric = complex(value) - return b"complex128\x00" + struct.pack(" None: - """Stream framed string or object scalars in bounded-memory chunks.""" - - for chunk_index, start in enumerate( - range(0, len(values), _LATE_TABLE_DIGEST_CHUNK_ROWS) - ): - stop = min(start + _LATE_TABLE_DIGEST_CHUNK_ROWS, len(values)) - lengths = np.zeros(stop - start, dtype=" None: - """Hash one ordered logical Series with explicit dtype and null domains.""" - - dtype = series.dtype - missing = series.isna().to_numpy(dtype=bool) - _late_digest_part( - digest, - domain=f"{domain}/dtype", - payload=str(dtype).encode("utf-8"), - ) - _late_digest_part( - digest, - domain=f"{domain}/row_count", - payload=struct.pack(" str: - """Hash ordered table scalars directly with typed, null-aware framing.""" - - values = ( - canonicalize_table_string_dtypes( - table, - boundary="late-producer content digest", - table_name="declared_surface", - ) - if normalize_strings - else table - ) - if isinstance(values.index, pd.MultiIndex): - index_levels = [ - pd.Series(values.index.get_level_values(level), copy=False) - for level in range(values.index.nlevels) - ] - else: - index_levels = [pd.Series(values.index, copy=False)] - header = { - "codec": _LATE_TABLE_DIGEST_CODEC, - "columns": [str(column) for column in values.columns], - "dtypes": [ - str(values.iloc[:, index].dtype) for index in range(values.shape[1]) - ], - "index_type": type(values.index).__name__, - "index_dtype": str(values.index.dtype), - "index_level_dtypes": [str(level.dtype) for level in index_levels], - "index_names": [ - None if name is None else str(name) for name in values.index.names - ], - } - digest = hashlib.sha256() - _late_digest_part( - digest, - domain="late_table_digest_codec", - payload=_LATE_TABLE_DIGEST_CODEC.encode("ascii"), - ) - _late_digest_part( - digest, - domain="late_table_header_json", - payload=json.dumps( - header, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8"), - ) - for level_index, level in enumerate(index_levels): - _late_digest_series_values( - digest, - level, - domain=f"index_level/{level_index}", - ) - for column_index in range(values.shape[1]): - _late_digest_series_values( - digest, - values.iloc[:, column_index], - domain=f"column/{column_index}", - ) - return digest.hexdigest() def _late_virtual_resource_kind(column: str) -> str: diff --git a/packages/microcosm-build/tests/test_us_acs_pums.py b/packages/microcosm-build/tests/test_us_acs_pums.py index 48b656d96..d663d21ac 100644 --- a/packages/microcosm-build/tests/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/test_us_acs_pums.py @@ -8,13 +8,13 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.us_runtime import acs_pums from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, AcsPumsSource, build_acs_pums_unit_frame, load_acs_pums_tables, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -126,6 +126,15 @@ def _source(tmp_path: Path) -> AcsPumsSource: return AcsPumsSource(household_zip=household_zip, person_zip=person_zip) +def _axis_household(kinds: list[int]) -> pd.DataFrame: + return pd.DataFrame( + { + "SERIALNO": [f"2024HU000000{index}" for index in range(len(kinds))], + "TYPEHUGQ": kinds, + } + ) + + def _asec_shaped_frame() -> Frame: """Return one ASEC-like row sharing only structural and lineage fields.""" @@ -264,6 +273,8 @@ def test_built_acs_lineage_assembles_with_asec_without_measured_coercion( measured_wages = acs.table("person")["WAGP"].copy() raw_serials = acs.table("household")["SERIALNO"].copy() + from microcosm.build.us_runtime.spine_assembly import assemble_spines + assembled = assemble_spines( {"asec": _asec_shaped_frame(), "acs": acs}, household_mass_shares={"asec": 0.5, "acs": 0.5}, @@ -595,3 +606,167 @@ def test_load_acs_pums_tables_requires_native_mapping_columns( load_acs_pums_tables( AcsPumsSource(household_zip=household_zip, person_zip=person_zip) ) + + +def test_load_acs_pums_tables_rejects_group_quarters_with_positive_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + { + "psam_husa.csv": [ + _household( + "2024GQ0000001", + WGTP=7, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ) + ] + }, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_load_acs_pums_tables_rejects_housing_unit_with_zero_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + {"psam_husa.csv": [_household("2024HU0000001", WGTP=0)]}, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_build_acs_pums_unit_frame_reports_household_axis_composition( + tmp_path: Path, +) -> None: + pytest.importorskip("microunit") # sanctioned tax-unit constructor (us extra) + household_zip = tmp_path / "csv_hus.zip" + person_zip = tmp_path / "csv_pus.zip" + _write_csv_zip( + household_zip, + { + "psam_husa.csv": [ + _household("2024HU0000001", WGTP=10, NP=2), + _household("2024HU0000002", WGTP=20), + _household( + "2024GQ0000001", + WGTP=0, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ), + _household( + "2024GQ0000002", + WGTP=0, + TYPEHUGQ=3, + TEN=None, + TAXAMT=None, + ), + _household("2024HU0000003", WGTP=40, NP=0), + ] + }, + ) + _write_csv_zip( + person_zip, + { + "psam_pusa.csv": [ + _person("2024HU0000001", 1, 20, PWGTP=11), + _person("2024HU0000001", 2, 25, MAR=5, PWGTP=12), + _person("2024HU0000002", 1, 20, PWGTP=13), + _person("2024GQ0000001", 1, 37, MAR=5, PWGTP=99), + _person("2024GQ0000002", 1, 38, MAR=5, PWGTP=55), + ] + }, + ) + + frame, metadata = build_acs_pums_unit_frame( + AcsPumsSource(household_zip=household_zip, person_zip=person_zip) + ) + + composition = metadata["household_axis_composition"] + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + # The dropped vacant housing unit is outside the described axis entirely. + assert metadata["vacant_household_rows_dropped"] == 1 + # Units conserve, and design mass conserves separately from units. + assert ( + composition["occupied_housing_unit_rows"] + + composition["institutional_gq_person_rows"] + + composition["noninstitutional_gq_person_rows"] + == metadata["household_rows"] + == frame.n("household") + == 4 + ) + assert composition["occupied_housing_unit_design_weight_total"] + composition[ + "institutional_gq_person_design_weight_total" + ] + composition["noninstitutional_gq_person_design_weight_total"] == pytest.approx( + metadata["weighted_household_population"] + ) + + +# Reached through the module, not imported by name, so this file still imports +# against a build without the helper: the four checks below then fail on their +# own instead of erroring the whole module at collection. +def test_household_axis_composition_separates_units_and_design_mass() -> None: + household = _axis_household([1, 1, 2, 3]) + weights = Weights( + np.asarray([10.0, 20.0, 99.0, 55.0]), + WeightKind.DESIGN, + ) + + composition = acs_pums._household_axis_composition(household, weights) + + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + assert sum( + value for key, value in composition.items() if key.endswith("_rows") + ) == len(household) + assert sum( + value + for key, value in composition.items() + if key.endswith("_design_weight_total") + ) == pytest.approx(weights.total) + + +def test_household_axis_composition_requires_typehugq() -> None: + household = _axis_household([1]).drop(columns=["TYPEHUGQ"]) + weights = Weights(np.asarray([10.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="requires the TYPEHUGQ column"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_unpartitioned_rows() -> None: + household = _axis_household([1, 4]) + weights = Weights(np.asarray([10.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="does not partition the loaded rows"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_misaligned_weights() -> None: + household = _axis_household([1, 2]) + weights = Weights(np.asarray([10.0, 99.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="one DESIGN weight per loaded"): + acs_pums._household_axis_composition(household, weights) diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index aa2a1338a..8ea3569cc 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -7,6 +7,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.asec_checkpoint as checkpoint_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.outer_stage_runtime import ( OUTER_STAGE_CONTEXT_SCHEMA_VERSION, @@ -26,6 +27,309 @@ ) from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights + +def _v4_frame() -> Frame: + frame = _raw_us_frame() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + frame.table("person")[column] = np.asarray([1, 2], dtype=np.int64) + return frame + + +def _v4_binding(frame: Frame) -> dict: + metadata = _raw_binding(frame) + metadata["schema_version"] = 4 + metadata["source_receipt"]["sources"].append( + dict(metadata["source_receipt"]["sources"][0], year=2023) + ) + pins = checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in (2022, 2023) + ], + # Audit describes the full official member, not this two-row subset. + "audit": { + str(year): { + "rows": pins[year].rows, + "yes_rows": 1, + "no_rows": pins[year].rows - 1, + "weighted_yes_share": 0.25, + } + for year in (2022, 2023) + }, + } + return metadata + + +def test_v4_roundtrip_requires_explicit_codec(tmp_path: Path) -> None: + frame = _v4_frame() + path = tmp_path / "v4.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + loaded, binding = checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + assert binding["schema_version"] == 4 + assert loaded.table("person")["NOW_MCAID"].tolist() == [1, 2] + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + load_asec_raw_stage_checkpoint(path) + path3 = tmp_path / "v3.h5" + legacy = _raw_us_frame() + _write_checkpoint(path3, legacy, metadata=_raw_binding(legacy)) + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path3) + + +def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> None: + from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs + + frame = _raw_us_frame() + person = frame.table("person") + person["NOW_GRP"] = [1, 2] + person["NOW_MRK"] = [2, 1] + person["OI_VAL"] = [0.0, 0.0] + person["OI_OFF"] = [0, 0] + sidecar = person[["source_year", "PERIDNUM"]].copy() + sidecar["PH_SEQ"] = [101, 102] + sidecar["P_SEQ"] = [1, 1] + sidecar["A_LINENO"] = [1, 1] + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + sidecar[column] = [1, 2] + result = derive_us_cps_carried_inputs(frame, reported_coverage_source=sidecar) + assert "NOW_MCAID" not in frame.table("person") + assert result.table("person")[ + "has_medicaid_health_coverage_at_interview" + ].tolist() == [True, False] + assert result.table("person")["has_esi"].tolist() == [True, False] + assert result.table("person")[ + "has_marketplace_health_coverage_at_interview" + ].tolist() == [False, True] + assert result.weights_for("household").kind is WeightKind.DESIGN + pd.testing.assert_frame_equal( + result.table("person"), derive_us_cps_carried_inputs(result).table("person") + ) + + +def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import hashlib + from dataclasses import replace + + from microcosm.build.us_runtime import reported_coverage_source + from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 + + legacy = _raw_us_frame() + binding = _raw_binding(legacy) + binding["source_receipt"]["sources"].append( + dict(binding["source_receipt"]["sources"][0], year=2023) + ) + input_path = tmp_path / "v3.h5" + _write_checkpoint(input_path, legacy, metadata=binding) + input_sha = hashlib.sha256(input_path.read_bytes()).hexdigest() + pins = {} + paths = {} + for year, key in ( + (2022, "0000000000000000000001"), + (2023, "0000000000000000000002"), + ): + path = tmp_path / f"person-{year}.csv" + row = { + "PERIDNUM": key, + "PH_SEQ": 1, + "P_SEQ": 1, + "A_LINENO": 1, + "A_FNLWGT": 100.0, + } + row.update( + { + column: 1 if year == 2022 else 2 + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS + } + ) + pd.DataFrame([row]).to_csv(path, index=False) + pins[year] = replace( + checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year], + rows=1, + member=path.name, + member_size_bytes=path.stat().st_size, + member_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + paths[year] = path + monkeypatch.setattr(checkpoint_module, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins) + monkeypatch.setattr( + reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins + ) + output_dir = tmp_path / "restored" + receipt = restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + assert receipt["input_sha256"] == input_sha + output = output_dir / "asec_raw_stage.checkpoint.h5" + assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha + restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] + assert metadata["source_receipt"] == binding["source_receipt"] + assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt + with pytest.raises(FileExistsError): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + with pytest.raises(ValueError, match="SHA-256"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256="f" * 64, + coverage_paths=paths, + output_dir=tmp_path / "bad", + ) + with pytest.raises(ValueError, match="local coverage paths"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths={2022: paths[2022]}, + output_dir=tmp_path / "missing", + ) + paths[2022].write_text("tampered") + with pytest.raises(ValueError, match="byte length mismatch"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=tmp_path / "tampered", + ) + assert not (tmp_path / "bad").exists() + assert not (tmp_path / "missing").exists() + assert not (tmp_path / "tampered").exists() + + +@pytest.mark.parametrize( + "mutation", + ( + "pin_sha", + "pin_member", + "pin_year_missing", + "pin_year_duplicate", + "audit_year", + "audit_rows", + "audit_counts", + "audit_nan", + "audit_bool", + "receipt_year", + "missing_mapping", + "float_version", + ), +) +def test_v4_refuses_inconsistent_attestation(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + metadata = _v4_binding(frame) + mapping = metadata["raw_source_mappings"]["NOW_MCAID"] + if mutation == "pin_sha": + mapping["source_pins"][0]["sha256"] = "f" * 64 + elif mutation == "pin_member": + mapping["source_pins"][0]["member"] = "other.csv" + elif mutation == "pin_year_missing": + mapping["source_pins"].pop() + elif mutation == "pin_year_duplicate": + mapping["source_pins"].append(dict(mapping["source_pins"][0])) + elif mutation == "audit_year": + del mapping["audit"]["2023"] + elif mutation == "audit_rows": + mapping["audit"]["2022"]["rows"] = 2 + elif mutation == "audit_counts": + mapping["audit"]["2022"]["no_rows"] = 0 + elif mutation == "audit_nan": + mapping["audit"]["2022"]["weighted_yes_share"] = float("nan") + elif mutation == "audit_bool": + mapping["audit"]["2022"]["yes_rows"] = True + elif mutation == "receipt_year": + metadata["source_receipt"]["sources"].pop() + elif mutation == "missing_mapping": + del metadata["raw_source_mappings"]["NOW_MCAID"] + elif mutation == "float_version": + metadata["schema_version"] = 4.0 + path = tmp_path / "invalid.h5" + # Exercise the same metadata validators directly for NaN because canonical + # checkpoint serialization correctly refuses it before the loader does. + if mutation == "audit_nan": + with pytest.raises(ValueError, match="audit counts/share"): + checkpoint_module._validate_coverage_v4_binding(frame, metadata, path=path) + else: + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize("value", (0, 3, 1.5, float("inf"), float("nan"), True)) +def test_v4_refuses_bad_measured_coverage(tmp_path: Path, value: object) -> None: + frame = _v4_frame() + frame.table("person")["NOW_MCAID"] = [value, value] + path = tmp_path / "bad-code.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + with pytest.raises(ValueError, match="NOW_MCAID must be complete integer recodes"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize( + "mutation", ("missing_column", "peridnum", "duplicate", "weights") +) +def test_v4_refuses_wrong_source_boundary(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + person = frame.table("person") + if mutation == "missing_column": + person.drop(columns="NOW_IHSFLG", inplace=True) + elif mutation == "peridnum": + person["PERIDNUM"] = ["1", "2"] + elif mutation == "duplicate": + person["source_year"] = [2022, 2022] + person["PERIDNUM"] = ["0" * 22, "0" * 22] + elif mutation == "weights": + frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": Weights(np.asarray([2.0, 3.0]), WeightKind.CALIBRATED)}, + frame.strata, + ) + metadata = _v4_binding(frame) + if mutation == "duplicate": + metadata["source_receipt"]["sources"].pop() + path = tmp_path / "boundary.h5" + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +def test_v4_duplicate_guard_compares_normalized_identity() -> None: + frame = _v4_frame() + person = frame.table("person") + person["source_year"] = pd.Series([2022, "2022"], dtype=object) + person["PERIDNUM"] = pd.Series(["0" * 22, b"0" * 22], dtype=object) + metadata = _v4_binding(frame) + metadata["source_receipt"]["sources"].pop() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column]["source_pins"].pop() + del metadata["raw_source_mappings"][column]["audit"]["2023"] + with pytest.raises(ValueError, match="repeats a source-year/PERIDNUM"): + checkpoint_module._validate_coverage_v4_binding( + frame, metadata, path=Path("fixture.h5") + ) + + _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py index 5aa9db083..ce9073cf7 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py @@ -95,6 +95,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: assert_exact_k_support, select_exact_k, ) +from microcosm.calibrate.group_bounds import GroupedUpperBounds # noqa: E402 from microcosm.calibrate.matrix import ( # noqa: E402 - after the compat gate CalibrationProblem, SkippedTarget, @@ -135,6 +136,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", "CONSERVE_MASS", "FREE_MASS", + "GroupedUpperBounds", "TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE", "TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE", "TARGET_LOSS_ATTRIBUTION_WARNING_CODES", diff --git a/packages/microcosm-fit/src/microcosm/fit/qrf.py b/packages/microcosm-fit/src/microcosm/fit/qrf.py index a39c57d2a..e8bb2c956 100644 --- a/packages/microcosm-fit/src/microcosm/fit/qrf.py +++ b/packages/microcosm-fit/src/microcosm/fit/qrf.py @@ -1034,6 +1034,36 @@ def _draw_target_with_rng( return values +def _draw_target_from_uniforms( + features: pd.DataFrame, + model: _TargetModel, + quantiles: np.ndarray, + sign_uniforms: np.ndarray, +) -> np.ndarray: + """Evaluate a target without consuming or replacing any model RNG state.""" + if model is _RELEASED: + raise RuntimeError("This target's fitted forests were released; refit to draw.") + if model.regime == Regime.DEGENERATE_ZERO: + return np.zeros(len(features), dtype=np.float64) + if model.regime == Regime.POSITIVE_ONLY: + return model.positive.draw(features, quantiles) + if model.regime == Regime.NEGATIVE_ONLY: + return model.negative.draw(features, quantiles) + x = features.loc[:, list(model.columns)].to_numpy(dtype=np.float64) + cumulative = np.cumsum(model.gate.predict_proba(x), axis=1) + # Uniforms occupy [0, 1): strict comparison skips zero-probability classes + # even at u=0. Close the final CDF bin against floating-point roundoff. + cumulative[:, -1] = 1.0 + chosen = (cumulative > sign_uniforms[:, None]).argmax(axis=1) + signs = np.asarray(model.gate.classes_)[chosen] + values = np.zeros(len(features), dtype=np.float64) + for sign, forest in ((1, model.positive), (-1, model.negative)): + mask = signs == sign + if mask.any() and forest is not None: + values[mask] = forest.draw(features.loc[mask], quantiles[mask]) + return values + + class RegimeGatedQRF: """The canonical :class:`~microcosm.fit.model.ConditionalModel`. @@ -1564,6 +1594,60 @@ def predict( self._target_models[target] = _RELEASED return out + def predict_from_uniforms( + self, + frame_or_df: Frame | pd.DataFrame, + *, + quantiles: Mapping[str, np.ndarray], + sign_uniforms: Mapping[str, np.ndarray], + ) -> pd.DataFrame: + """Draw using caller-supplied per-row uniforms, without advancing RNG. + + Each mapping must contain exactly the fitted targets, with one finite + one-dimensional array in ``[0, 1)`` per target, aligned to input rows. + Supply both arrays even for single-sign or all-zero targets. Later + targets condition on earlier draws, just as in :meth:`predict`. + + Pairing uniforms with stable entity IDs makes results invariant to + recipient ordering and batching. Fitted forests remain reusable. The + legacy :meth:`predict` stream and its consumption order are unchanged. + """ + features = self._predictor_frame(frame_or_df) + arrays = {} + for name, supplied in ( + ("quantiles", quantiles), + ("sign_uniforms", sign_uniforms), + ): + if not isinstance(supplied, Mapping) or set(supplied) != set(self.targets): + raise ValueError(f"{name} must contain exactly the fitted targets.") + arrays[name] = {} + for target in self.targets: + values = np.asarray(supplied[target], dtype=np.float64) + if values.shape != (len(features),): + raise ValueError( + f"{name}[{target!r}] must have shape ({len(features)},)." + ) + if ( + not np.isfinite(values).all() + or ((values < 0) | (values >= 1)).any() + ): + raise ValueError(f"{name}[{target!r}] uniforms must be in [0, 1).") + arrays[name][target] = values + out = pd.DataFrame(index=features.index) + if features.empty: + return out.reindex(columns=self.targets).astype(np.float64) + augmented = features.copy() + for target in self.targets: + drawn = _draw_target_from_uniforms( + augmented, + self._target_models[target], + arrays["quantiles"][target], + arrays["sign_uniforms"][target], + ) + out[target] = drawn + augmented[target] = drawn + return out + def _predictor_frame(self, frame_or_df: Frame | pd.DataFrame) -> pd.DataFrame: """Extract the predictor columns from a Frame or DataFrame input.""" if isinstance(frame_or_df, Frame): From d250c863009f751f88240bdad4366def522713b4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 006/255] Stage integration layer 04: PUF-SUPPORT-MERGE.patch --- PROGRESS.md | 2 + .../microcosm/build/us_runtime/puf_support.py | 90 +++++-------------- 2 files changed, 26 insertions(+), 66 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 4852c7f73..8150c10b0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -743,3 +743,5 @@ Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation h Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index d78c70a23..77178b006 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -29,13 +29,9 @@ puf_capital_gains_joint_metrics, puf_processed_capital_gains_stage, ) -from microcosm.build.us_runtime.puf_interest_components import ( - split_us_puf_e19200_by_agi_band, -) from microcosm.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, @@ -54,6 +50,19 @@ from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights, wquantile from microcosm.frame.schema import EntitySchema +from .operator_column_contracts import ( + PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID as PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) +from .operator_column_contracts import ( + US_PUF_SUPPORT_STAGE_NAME as US_PUF_SUPPORT_STAGE_NAME, +) + QRF: Any | None = None __all__ = [ @@ -91,7 +100,6 @@ "validate_puf_clone_attachment", ] -US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" #: Frozen receipt binding a seeded clone attachment (microcosm#578 revision #: item 3) to the live rows: fraction, seed, the floor-rule counts, and the @@ -207,54 +215,6 @@ class _PredictorSourcePlan: "puf_predictor_long_term_capital_gains", ) -PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( - "employment_income_before_lsr", - "self_employment_income_before_lsr", - "taxable_interest_income", - "qualified_dividend_income", - "non_qualified_dividend_income", - "tax_exempt_interest_income", - "short_term_capital_gains", - "long_term_capital_gains_before_response", - "long_term_capital_gains_on_collectibles", - "non_sch_d_capital_gains", - "taxable_private_pension_income", - "taxable_ira_distributions", - "social_security_retirement", - "social_security_disability", - "social_security_dependents", - "social_security_survivors", - "alimony_income", - "alimony_expense", - "salt_refund_income", - "charitable_cash_donations", - "charitable_non_cash_donations", - "real_estate_taxes", - "home_mortgage_interest", - "investment_interest_expense", - "investment_income_elected_form_4952", - "student_loan_interest", - "educator_expense", - "qualified_tuition_expenses", - "casualty_loss", - "unreimbursed_business_employee_expenses", - # The engine owns the realized contribution amounts through the - # IRA-limit scale and self-employment caps; the persistable leaves are - # the desired contributions, equal to the PUF's observed deductions at - # baseline (issue #278). - "traditional_ira_contributions_desired", - "self_employed_pension_contributions_desired", - "rental_income", - "estate_income", - "farm_income", - "farm_operations_income", - "farm_rent_income", - "miscellaneous_income", - "partnership_income", - "s_corp_income", - "partnership_self_employment_net_earnings", - *US_QBI_OUTPUT_COLUMNS, -) PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS = ( "social_security_retirement", @@ -263,17 +223,6 @@ class _PredictorSourcePlan: "social_security_survivors", ) -PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( - "domestic_production_ald", - "unrecaptured_section_1250_gain", - "first_home_mortgage_balance", - "second_home_mortgage_balance", - "first_home_mortgage_interest", - "second_home_mortgage_interest", - "first_home_mortgage_origination_year", - "second_home_mortgage_origination_year", - "health_savings_account_ald", -) _PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS = frozenset( { @@ -1505,6 +1454,16 @@ def _quarantine_us_puf_mortgage_fields( donor_build_summary["mortgage_field_quarantine"] = quarantine +def split_us_puf_e19200_by_agi_band( + total_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray]: + """Load the published component resource only for actual decomposition.""" + from .puf_interest_components import split_us_puf_e19200_by_agi_band as split + + return split(total_interest_paid, adjusted_gross_income) + + def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: """Split raw E19200 into mortgage and modeled non-mortgage components.""" @@ -2368,7 +2327,6 @@ def _id_multiplier_for_frame(frame: Frame) -> int: # indices up to 921 before int64 overflow — orders beyond any configured # clone count. Assembly enforces this bound; _remap_ids re-checks it so a # violation is a governed ValueError, never an OverflowError. -PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 _INT64_MAX = 2**63 - 1 @@ -2563,7 +2521,7 @@ def _formula_owned_engine() -> Any | None: except ImportError: return None try: - return PolicyEngineUSVariableMetadataIndex() + return PolicyEngineUSVariableMetadataIndex(include_consumers=False) except ImportError: return None From 76d07d74acfd87585e0e21689806c30e90648ccd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 007/255] Stage integration layer 05: SOLVE-MERGE-PROPOSAL.patch --- PROGRESS.md | 2 + .../src/microcosm/calibrate/solve.py | 372 +++++++++++++++++- 2 files changed, 368 insertions(+), 6 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 8150c10b0..0365a082e 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -745,3 +745,5 @@ Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation h Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index c37ed85c9..762eccda6 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -67,6 +67,7 @@ from __future__ import annotations +import hashlib import math import warnings from collections.abc import Callable, Mapping @@ -77,6 +78,7 @@ from microcosm.calibrate.exact_k import assert_exact_k_support from microcosm.calibrate.gates import HardConcrete +from microcosm.calibrate.group_bounds import GroupedUpperBounds from microcosm.calibrate.matrix import ( CalibrationProblem, SkippedTarget, @@ -284,7 +286,15 @@ def realized_max_weight_ratio(self) -> float: """ weights = np.asarray(self.weights, dtype=np.float64) initial = np.asarray(self.initial_weights, dtype=np.float64) - return float((weights / initial).max()) + if not self.options.get("grouped_preserve_zeros", {}).get("enabled", False): + return float((weights / initial).max()) + if np.any((initial == 0) & (weights > 0)): + return float("inf") + ratios = np.zeros_like(weights) + np.divide( + weights, initial, out=ratios, where=~((initial == 0) & (weights == 0)) + ) + return float(ratios.max()) @property def top_1pct_weight_share(self) -> float: @@ -772,6 +782,9 @@ def _optimize( progress_callback: Callable[[dict[str, object]], None] | None = None, progress_context: Mapping[str, object] | None = None, return_gate_open_probabilities: bool = False, + grouped_upper_bounds: GroupedUpperBounds | None = None, + grouped_preserve_zeros: bool = False, + _post_projection_observer: Callable[[dict[str, object]], None] | None = None, selection_receipt: dict[str, object] | None = None, ) -> tuple[np.ndarray, np.ndarray] | tuple[np.ndarray, np.ndarray, np.ndarray | None]: """Run the torch optimization and return weights plus its trajectory. @@ -794,6 +807,41 @@ def _optimize( callers. The third value is aligned per-record ``pi_i`` for an L0 run and ``None`` when no gates were active. """ + _validate_grouped_zero_option(grouped_upper_bounds, grouped_preserve_zeros) + if grouped_upper_bounds is not None: + _validate_grouped_mode( + grouped_upper_bounds, + method="adam", + conserve_mass=conserve_mass, + max_weight_ratio=max_weight_ratio, + l0_lambda=l0_lambda, + l1_lambda=0.0, + target_records=target_records, + ) + final, trajectory = _optimize_grouped( + matrix, + targets, + target_loss_weights, + target_loss_scales, + target_loss_cap, + initial_weights, + grouped_upper_bounds, + warm_start_weights=warm_start_weights, + epochs=epochs, + learning_rate=learning_rate, + l2_lambda=l2_lambda, + l2_anchor=l2_anchor, + l2_anchor_weights=l2_anchor_weights, + progress_callback=progress_callback, + progress_context=progress_context, + observer=_post_projection_observer, + preserve_zeros=grouped_preserve_zeros, + ) + if return_gate_open_probabilities: + return final, trajectory, None + return final, trajectory + if _post_projection_observer is not None: + raise ValueError("post-projection observer requires grouped upper bounds") w0 = np.asarray(initial_weights, dtype=np.float64) start = _prepare_warm_start_weights( w0, @@ -975,6 +1023,211 @@ def _optimize( return final, trajectory +def _validate_grouped_mode( + groups: GroupedUpperBounds, + *, + method: str, + conserve_mass: bool, + max_weight_ratio: float | None, + l0_lambda: float, + l1_lambda: float, + target_records: int | None, +) -> None: + if not isinstance(groups, GroupedUpperBounds): + raise ValueError("grouped_upper_bounds must be GroupedUpperBounds") + if max_weight_ratio is not None: + raise ValueError( + "grouped upper bounds refuse simultaneous scalar max_weight_ratio" + ) + if ( + method != "adam" + or conserve_mass + or l0_lambda != 0 + or l1_lambda != 0 + or target_records is not None + ): + raise ValueError( + "grouped upper bounds require Adam free mass without prox, L0, or exact-k selection" + ) + + +def _validate_grouped_zero_option( + groups: GroupedUpperBounds | None, preserve_zeros: bool +) -> None: + if type(preserve_zeros) is not bool: + raise ValueError("grouped_preserve_zeros must be a boolean") + if preserve_zeros and groups is None: + raise ValueError("grouped_preserve_zeros requires grouped upper bounds") + + +def _check_fixed_support( + weights: np.ndarray, initial: np.ndarray, groups: GroupedUpperBounds +) -> np.ndarray: + """Validate the complete vector, preserving the initial zero coordinates.""" + totals = groups.check(weights) + active = initial > 0 + if not active.any(): + raise ValueError("grouped fixed support requires a positive initial weight") + if weights[~active].tobytes() != initial[~active].tobytes(): + raise ValueError("grouped weights must preserve the frozen zero support") + if (weights[active] <= 0).any(): + raise ValueError("grouped active support must remain strictly positive") + return totals + + +def _check_grouped_household_ids(frame: Frame, expected: tuple[int | str, ...]) -> None: + actual = tuple( + frame.table("household")[frame.schema.entity_id_column("household")].tolist() + ) + if actual != expected: + raise ValueError("stored group household IDs must preserve the ordered IDs") + + +def _optimize_grouped( + matrix: torch.Tensor, + targets: torch.Tensor, + target_loss_weights: torch.Tensor | None, + target_loss_scales: torch.Tensor, + target_loss_cap: float, + initial_weights: np.ndarray, + groups: GroupedUpperBounds, + *, + warm_start_weights: np.ndarray | None, + epochs: int, + learning_rate: float, + l2_lambda: float, + l2_anchor: str, + l2_anchor_weights: np.ndarray | None, + progress_callback: Callable[[dict[str, object]], None] | None, + progress_context: Mapping[str, object] | None, + observer: Callable[[dict[str, object]], None] | None, + preserve_zeros: bool = False, +) -> tuple[np.ndarray, np.ndarray]: + """Free-mass Adam on positive coordinates, with full float64 accepted state.""" + if ( + matrix.dtype != torch.float32 + or targets.dtype != torch.float32 + or target_loss_scales.dtype != torch.float32 + ): + raise ValueError("grouped loss matrix, targets and scales must remain float32") + w0 = np.array(initial_weights, dtype=np.float64, copy=True) + groups.check(w0, positive=not preserve_zeros) + if preserve_zeros: + _check_fixed_support(w0, w0, groups) + active = w0 > 0 + accepted = ( + w0.copy() + if warm_start_weights is None + else np.array(warm_start_weights, dtype=np.float64, copy=True) + ) + _check_fixed_support(accepted, w0, groups) + active_indices = torch.tensor(np.flatnonzero(active), dtype=torch.int64) + zero_template = w0.copy() + zero_template[active] = 0.0 + zero_template_t = torch.tensor(zero_template, dtype=torch.float64) + else: + active = np.ones(w0.shape, dtype=bool) + accepted = _prepare_warm_start_weights( + w0, + warm_start_weights, + conserve_mass=False, + max_weight_ratio=None, + ) + groups.check(accepted, positive=True) + + def observe(epoch: int, corrected: int) -> None: + totals = ( + _check_fixed_support(accepted, w0, groups) + if preserve_zeros + else groups.check(accepted, positive=True) + ) + if observer is not None: + observer( + { + "kind": "grouped_accepted_weights", + "epoch": epoch, + "weights": accepted.copy(), + "household_ids": groups.household_ids, + "group_indices": groups.group_indices.copy(), + "absolute_bounds": groups.absolute_bounds.copy(), + "group_totals": totals.copy(), + "corrected_group_count": corrected, + } + ) + + # Admission and initial observer precede even optimizer construction. + observe(0, 0) + log_w = torch.tensor( + np.log(accepted[active]), dtype=torch.float64, requires_grad=True + ) + optimizer = torch.optim.Adam([log_w], lr=learning_rate) + anchor_t = None + if l2_lambda > 0: + anchor = ( + l2_anchor_weights + if l2_anchor_weights is not None + else (w0 if l2_anchor == "initial" else np.full_like(w0, w0.mean())) + ) + anchor_t = torch.tensor(anchor[active], dtype=torch.float64) + trajectory = np.empty(epochs, dtype=np.float64) + for epoch in range(epochs): + optimizer.zero_grad() + # This is an internal candidate, not an accepted population vector. + active_weights = torch.exp(log_w) + weights = ( + zero_template_t.index_copy(0, active_indices, active_weights) + if preserve_zeros + else active_weights + ) + estimate = _apply_constraint(matrix, weights.to(dtype=torch.float32)) + loss = _relative_error_loss( + estimate, + targets, + target_loss_weights, + target_loss_scales, + target_loss_cap, + ) + trajectory[epoch] = float(loss.item()) + if anchor_t is None: + total_loss = loss + elif preserve_zeros: + total_loss = loss + l2_lambda * ( + (active_weights / anchor_t) ** 2 + ).sum() / len(w0) + else: + total_loss = loss + l2_lambda * ((weights / anchor_t) ** 2).mean() + if progress_callback is not None: + progress_callback( + { + **dict(progress_context or {}), + "kind": "calibration_epoch", + "epoch": epoch + 1, + "epochs": epochs, + "loss": trajectory[epoch], + } + ) + total_loss.backward() + optimizer.step() + with torch.no_grad(): + active_candidate = torch.exp(log_w).detach().cpu().numpy().copy() + if preserve_zeros: + candidate = zero_template.copy() + candidate[active] = active_candidate + accepted, corrected = groups.project(candidate, positive=False) + _check_fixed_support(accepted, w0, groups) + else: + accepted, corrected = groups.project(active_candidate) + # Exactly one overwrite for the next update; no reconstruction loop. + log_w.copy_(torch.from_numpy(np.log(accepted[active]))) + observe(epoch + 1, corrected) + closing, _ = groups.project(accepted, positive=not preserve_zeros) + if preserve_zeros: + _check_fixed_support(closing, w0, groups) + if closing.tobytes() != accepted.tobytes(): + raise ValueError("final grouped projection must preserve accepted weight bytes") + return accepted, trajectory + + def _optimize_proximal( matrix: torch.Tensor, targets: torch.Tensor, @@ -1399,6 +1652,9 @@ def calibrate( target_loss_cap: float = _DEFAULT_TARGET_LOSS_CAP, warm_start_weights: np.ndarray | None = None, progress_callback: Callable[[dict[str, object]], None] | None = None, + grouped_upper_bounds: GroupedUpperBounds | None = None, + grouped_preserve_zeros: bool = False, + _post_projection_observer: Callable[[dict[str, object]], None] | None = None, ) -> CalibrationResult: """Calibrate ``weight_entity``'s weights to ``targets`` over ``frame``. @@ -1414,8 +1670,9 @@ def calibrate( ``"household"``). method: Optimization method. ``"adam"`` (default) runs the torch Adam optimizer on the log-weights described above (weights stay strictly - positive by construction). Unregularized free-mass runs retain their - best feasible fit within the requested epoch budget. ``"prox"`` runs proximal + positive by construction). Ungrouped unregularized free-mass runs + retain their best feasible fit within the requested epoch budget. + ``"prox"`` runs proximal gradient (ISTA) on raw non-negative weight ratios with a soft-threshold step, the optimizer required for the nonsmooth ``l1_lambda`` penalty: it drives unneeded records to exact zero, so @@ -1437,6 +1694,23 @@ def calibrate( appends no record. max_weight_ratio: If given, a hard per-record cap: no calibrated weight exceeds ``max_weight_ratio * initial_weight``. The landmine guard. + grouped_upper_bounds: Optional exhaustive ordered household grouping + with frozen absolute bounds. This positive-only Adam/free-mass path + refuses scalar row caps, prox, L0 and record selection. Both input + and effective warm-start weights must already be feasible. Accepted + float64 vectors use stable-ID ``math.fsum`` and direct weight-space + projection; float32 matrix/loss arithmetic remains differentiable. + grouped_preserve_zeros: Opt in to fixed initial zero support for grouped + Adam. Zero rows remain in the full Frame at their original zero + values; only positive coordinates receive log parameters. Both + warm starts and every accepted/stored vector must preserve that + support exactly. An entirely zero initial vector is refused. The + L2 penalty sums positive-coordinate contributions divided by the + full row count; zero coordinates contribute zero. The default + retains the existing strictly positive grouped contract. + _post_projection_observer: Private grouped proof seam, called with + independent copies of the validated initial and every accepted + post-update vector, IDs, group map, bounds and exact group totals. target_records: If given, enable L0 pruning with **budget control**: the solver searches ``l0_lambda`` (a bisection on its log, ``budget_iters`` optimizations) so the achieved non-zero count tracks this budget, and @@ -1522,6 +1796,40 @@ def calibrate( ``method="prox"`` is combined with L0/budget pruning or ``l2_lambda``, or if no targets compile (from the matrix build). """ + _validate_grouped_zero_option(grouped_upper_bounds, grouped_preserve_zeros) + if grouped_upper_bounds is not None: + _validate_grouped_mode( + grouped_upper_bounds, + method=method, + conserve_mass=(mass != FREE_MASS), + max_weight_ratio=max_weight_ratio, + l0_lambda=l0_lambda, + l1_lambda=l1_lambda, + target_records=target_records, + ) + if weight_entity != "household": + raise ValueError("grouped upper bounds require household weights") + actual_ids = tuple( + frame.table(weight_entity)[ + frame.schema.entity_id_column(weight_entity) + ].tolist() + ) + if actual_ids != grouped_upper_bounds.household_ids: + raise ValueError( + "group household IDs must exactly match the frame's ordered IDs" + ) + elif _post_projection_observer is not None: + raise ValueError("post-projection observer requires grouped upper bounds") + last_group_correction = 0 + + def grouped_observer(payload: dict[str, object]) -> None: + nonlocal last_group_correction + _check_grouped_household_ids(frame, actual_ids) + last_group_correction = int(payload["corrected_group_count"]) + if _post_projection_observer is not None: + _post_projection_observer(payload) + _check_grouped_household_ids(frame, actual_ids) + if method == "apg": warnings.warn( "method='apg' is deprecated and now aliases method='adam'; result " @@ -1780,6 +2088,11 @@ def calibrate( progress_callback=progress_callback, return_gate_open_probabilities=True, selection_receipt=iterate_selection_receipt, + grouped_upper_bounds=grouped_upper_bounds, + grouped_preserve_zeros=grouped_preserve_zeros, + _post_projection_observer=grouped_observer + if grouped_upper_bounds is not None + else None, ) n_nonzero = int((final_weights > prune_atol).sum()) if effective_l0 > 0.0 and n_nonzero == 0: @@ -1795,6 +2108,9 @@ def calibrate( "control, which adapts the penalty to a survivor count." ) + accepted_weight_bytes = ( + final_weights.tobytes() if grouped_upper_bounds is not None else None + ) calibrated = initial.with_values(final_weights, kind=WeightKind.CALIBRATED) new_frame = _apply_weights( frame, @@ -1805,7 +2121,6 @@ def calibrate( targets, mass_reason=mass_reason, ) - diagnostics = _build_diagnostics(problem, frame, w0, final_weights) # One closing eval-mode loss on the RETURNED weights (Finding 8): the same # capped weighted-MAPE loss the optimizer minimizes, evaluated after the closing @@ -1824,7 +2139,7 @@ def calibrate( else target_loss_weights_np.copy() ) - return CalibrationResult( + result = CalibrationResult( frame=new_frame, weight_entity=weight_entity, weights=final_weights, @@ -1840,12 +2155,35 @@ def calibrate( target_loss_scales=target_loss_scales_np.copy(), target_loss_cap=target_loss_cap, options={ + **( + { + "grouped_preserve_zeros": { + "enabled": True, + "fixed_zero_count": int(np.count_nonzero(w0 == 0)), + "ordered_zero_mask_sha256": hashlib.sha256( + np.asarray(w0 == 0, dtype=np.uint8).tobytes() + ).hexdigest(), + } + } + if grouped_preserve_zeros + else {} + ), + **( + { + "grouped_upper_bounds": grouped_upper_bounds.diagnostics( + final_weights, last_group_correction + ) + } + if grouped_upper_bounds is not None + else {} + ), "method": method, "epochs": epochs, "learning_rate": learning_rate, "iterate_selection": ( "best_feasible_loss" - if method == "adam" + if grouped_upper_bounds is None + and method == "adam" and mass == FREE_MASS and l0_lambda == 0.0 and l2_lambda == 0.0 @@ -1886,6 +2224,28 @@ def calibrate( }, gate_open_probabilities=gate_open_probabilities, ) + if grouped_upper_bounds is not None: + # Admission is last, after diagnostics/options and their possible hooks. + # Frozen accepted bytes precede materialization, so changing two live + # vector aliases cannot make a changed result validate itself. + stored = result.frame.resolve_weights(weight_entity).values + closing, _ = grouped_upper_bounds.project( + stored, positive=not grouped_preserve_zeros + ) + if grouped_preserve_zeros: + _check_fixed_support(stored, w0, grouped_upper_bounds) + _check_fixed_support(closing, w0, grouped_upper_bounds) + if ( + result.weights.tobytes() != accepted_weight_bytes + or stored.tobytes() != accepted_weight_bytes + or closing.tobytes() != accepted_weight_bytes + ): + raise ValueError( + "stored grouped weights must preserve exact final accepted bytes" + ) + _check_grouped_household_ids(frame, actual_ids) + _check_grouped_household_ids(result.frame, actual_ids) + return result def _apply_weights( From 036503c47b6fd134100afc5b91a5e2514db2499d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 008/255] Stage integration layer 06: J-GRAPH-COMPATIBILITY.patch --- PROGRESS.md | 2 + .../microcosm/build/us_runtime/__init__.py | 3106 +++++++++-------- .../microcosm/build/us_runtime/puma_ladder.py | 319 +- .../build/us_runtime/puma_ladder_sources.py | 33 +- .../tests/test_us_acs_multispine.py | 18 +- .../test_us_acs_multispine_legacy_builder.py | 18 +- .../tests/test_us_base_pool.py | 18 +- .../tests/test_us_multispine_pool_tool.py | 18 +- .../tests/test_us_puma_ladder.py | 177 +- .../tests/test_us_puma_ladder_sources.py | 202 +- tools/build_us_puma_ladder_artifact.py | 99 +- 11 files changed, 2448 insertions(+), 1562 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 0365a082e..483ac5b64 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -747,3 +747,5 @@ Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and pres Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 9437b99e8..60b2d5d28 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -24,7 +24,9 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from importlib import import_module from importlib.resources import files +from threading import RLock from microcosm.build.plan import DonorSpec, Stage, StagePlan from microcosm.build.source_manifest import ( @@ -35,1061 +37,12 @@ load_source_manifest, load_support_spine_manifest, ) -from microcosm.build.us_runtime.adult_care import ( - US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT, - US_ADULT_CARE_EARNED_INCOME_SOURCES, - US_ADULT_CARE_OUTPUT_COLUMNS, - US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS, - US_ADULT_CARE_STAGE_NAME, - derive_us_adult_care_from_manifest, - us_adult_care_signal_gate, - us_adult_care_stage_spec, - us_adult_care_summary, - with_us_adult_care_inputs, -) -from microcosm.build.us_runtime.alimony import ( - ALIMONY_ASEC_ARCHIVED_DERIVATION_URL, - ALIMONY_PUF_ARCHIVED_DERIVATION_URL, - STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL, - US_ALIMONY_NONCONSTANT_PERSON_COLUMNS, - US_ALIMONY_OUTPUT_COLUMNS, - US_ALIMONY_STAGE_NAME, - US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, - derive_us_alimony_from_asec, - derive_us_alimony_from_puf, - us_alimony_signal_gate, - us_alimony_stage_spec, - us_alimony_summary, -) -from microcosm.build.us_runtime.asec_checkpoint import ( - ASEC_RAW_STAGE_ARTIFACT_KIND, - ASEC_RAW_STAGE_CHECKPOINT_FILENAME, - ASEC_RAW_STAGE_OPERATOR_STATUS, - ASEC_RAW_STAGE_SCHEMA_VERSION, - ASEC_RAW_STAGE_STAGE, - load_asec_pre_clone_checkpoint, - load_asec_raw_stage_checkpoint, -) -from microcosm.build.us_runtime.asec_pool import ( - AsecSource, - build_pooled_asec_unit_frame, - load_asec_h5_tables, - pool_asec_sources, -) -from microcosm.build.us_runtime.capital_gain_details import ( - CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_STAGE_NAME, - derive_us_capital_gain_details_from_puf, - us_capital_gain_details_signal_gate, - us_capital_gain_details_stage_spec, - us_capital_gain_details_summary, -) -from microcosm.build.us_runtime.casualty_losses import ( - US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS, - US_CASUALTY_LOSS_OUTPUT_COLUMNS, - US_CASUALTY_LOSS_STAGE_NAME, - derive_us_casualty_loss_from_puf, - us_casualty_loss_signal_gate, - us_casualty_loss_stage_spec, - us_casualty_loss_summary, -) -from microcosm.build.us_runtime.child_support import ( - CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL, - CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL, - CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL, - CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL, - US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS, - US_CHILD_SUPPORT_OUTPUT_COLUMNS, - US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS, - US_CHILD_SUPPORT_STAGE_NAME, - derive_us_child_support_from_asec, - derive_us_child_support_from_manifest, - impute_us_child_support_to_puf_support_from_manifest, - us_child_support_signal_gate, - us_child_support_stage_spec, - us_child_support_summary, - with_us_child_support_inputs, -) -from microcosm.build.us_runtime.childcare import ( - US_CHILDCARE_OUTPUT_COLUMNS, - US_CHILDCARE_REQUIRED_SOURCE_COLUMNS, - US_CHILDCARE_STAGE_NAME, - derive_us_childcare_from_manifest, - impute_us_childcare_to_puf_support_from_manifest, - us_childcare_signal_gate, - us_childcare_stage_spec, - us_childcare_summary, - with_us_childcare_inputs, -) -from microcosm.build.us_runtime.congressional_district_geography import ( - CONGRESSIONAL_DISTRICT_GEOID_COLUMN, - SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID, - assign_congressional_districts_to_households, - congressional_district_assignment_summary, - congressional_district_distribution_from_ledger_facts, - with_household_congressional_districts, -) -from microcosm.build.us_runtime.congressional_district_vintage import ( - CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, - CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, - CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, - CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, - DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE, - SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, - default_congressional_district_vintage_crosswalk_path, - load_congressional_district_vintage_crosswalk, - load_default_congressional_district_vintage_crosswalk, - translate_congressional_district_facts_to_current_vintage, -) -from microcosm.build.us_runtime.congressional_district_vintage_crosswalk import ( - CROSSWALK_BASIS_BLOCK_POPULATION, - build_cd_vintage_crosswalk_rows, - normalize_district_code, - parse_baf_cd_layer, - parse_national_cd_bef_districts, -) -from microcosm.build.us_runtime.cps_carried import ( - CPS_CARRIED_FORMULA_OWNED_COLUMNS, - CPS_CARRIED_PERSON_INPUTS, - CPS_CARRIED_SPM_UNIT_INPUTS, - CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN, - CPS_REPORTED_TANF_TYPE_RAW_COLUMN, - CPS_REPORTED_WIC_RAW_COLUMN, - US_REPORTED_COVERAGE_PERSON_INPUTS, - US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS, - WIC_CARRIER_ADJUDICATION_URL, - derive_us_cps_carried_inputs, - reported_tanf_enrollment_by_spm_unit, - reported_wic_receipt_carrier, -) -from microcosm.build.us_runtime.demographics import ( - AGE_BANDS, - DEMOGRAPHICS_SCHEMA_VERSION, - AgeBand, - compute_age_distribution, - demographics_payload, - write_demographics, -) -from microcosm.build.us_runtime.disability_benefits import ( - DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL, - DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL, - US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS, - US_DISABILITY_BENEFITS_OUTPUT_COLUMNS, - US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS, - US_DISABILITY_BENEFITS_STAGE_NAME, - derive_us_disability_benefits_from_asec, - derive_us_disability_benefits_from_manifest, - impute_us_disability_benefits_to_puf_support_from_manifest, - us_disability_benefits_signal_gate, - us_disability_benefits_stage_spec, - us_disability_benefits_summary, - with_us_disability_benefits, -) -from microcosm.build.us_runtime.domestic_production import ( - DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL, - US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME, - derive_us_domestic_production_ald_from_puf, - us_domestic_production_ald_signal_gate, - us_domestic_production_ald_stage_spec, - us_domestic_production_ald_summary, -) -from microcosm.build.us_runtime.education_assistance_source import ( - ASEC_EDUCATION_ASSISTANCE_ARCHIVES, - ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, - fetch_asec_education_assistance_source, - fill_asec_education_assistance_source, - load_asec_education_assistance_sources, -) -from microcosm.build.us_runtime.education_inputs import ( - US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATION_INPUTS_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_EDUCATION_INPUTS_STAGE_NAME, - derive_us_education_inputs_from_manifest, - us_education_inputs_signal_gate, - us_education_inputs_stage_spec, - us_education_inputs_summary, - with_us_education_inputs, -) -from microcosm.build.us_runtime.educator_expenses import ( - EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL, - EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL, - US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, - US_EDUCATOR_EXPENSE_STAGE_NAME, - derive_us_educator_expense_from_puf, - us_educator_expense_signal_gate, - us_educator_expense_stage_spec, - us_educator_expense_summary, -) -from microcosm.build.us_runtime.eligibility_inputs import ( - US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, - US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - derive_us_eligibility_inputs_from_manifest, - us_eligibility_inputs_signal_gate, - us_eligibility_inputs_stage_spec, - us_eligibility_inputs_summary, - with_us_eligibility_inputs, -) -from microcosm.build.us_runtime.energy_subsidy import ( - ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL, - ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL, - US_ENERGY_SUBSIDY_OUTPUT_COLUMNS, - US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS, - US_ENERGY_SUBSIDY_STAGE_NAME, - derive_us_energy_subsidy_from_manifest, - impute_us_energy_subsidy_to_puf_support_from_manifest, - us_energy_subsidy_signal_gate, - us_energy_subsidy_stage_spec, - us_energy_subsidy_summary, - with_us_energy_subsidy_input, -) -from microcosm.build.us_runtime.farm_business_income import ( - FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL, - FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL, - FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL, - FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL, - US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, - US_FARM_BUSINESS_INCOME_STAGE_NAME, - derive_us_farm_business_income_from_puf, - us_farm_business_income_signal_gate, - us_farm_business_income_stage_spec, - us_farm_business_income_summary, -) -from microcosm.build.us_runtime.fiscal_targets import ( - SOI_VARIABLE_MAP, - US_FISCAL_LEDGER_PARITY_REGISTRY, - US_FISCAL_LEDGER_PARITY_REPORT, - US_FISCAL_MACRO_REALISM_BANDS, - US_FISCAL_TARGET_COVERAGE_REQUIREMENTS, - US_FISCAL_TARGET_LEDGER_REFERENCES, - US_FISCAL_TARGET_REFERENCES, - US_FISCAL_TARGET_REGISTRY, - US_FISCAL_TARGET_SPECS, - US_FISCAL_TARGET_SUPPORT_EXCLUSIONS, - US_JCT_TAX_EXPENDITURE_REFORMS, - US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES, - US_JCT_TAX_EXPENDITURE_TARGET_SPECS, - US_SOI_FISCAL_TARGET_REFERENCES, - US_SOI_FISCAL_TARGET_SPECS, - US_STATE_INCOME_TAX_TARGET_REFERENCES, - US_STATE_INCOME_TAX_TARGET_SPECS, - SimpleTaxExpenditureReform, - compile_us_fiscal_target_registry, -) -from microcosm.build.us_runtime.form_4952 import ( - FORM_4952_ARCHIVED_DERIVATION_URL, - FORM_4952_ARCHIVED_EXPORT_URL, - FORM_4952_ARCHIVED_IMPUTATION_URL, - FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL, - FORM_4952_ARCHIVED_PUF_ARTIFACT_URL, - US_FORM_4952_NONCONSTANT_PERSON_COLUMNS, - US_FORM_4952_OUTPUT_COLUMNS, - US_FORM_4952_STAGE_NAME, - derive_us_form_4952_election_from_puf, - us_form_4952_election_signal_gate, - us_form_4952_election_stage_spec, - us_form_4952_election_summary, -) -from microcosm.build.us_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - US_BLOCK_LADDER_DERIVED_LAYERS, - US_BLOCK_LADDER_KIND, - US_BLOCK_LADDER_SCHEMA_VERSION, - US_GEOGRAPHY_LADDER_COLUMNS, - US_NYC_COUNTY_FIPS, - UsBlockLadder, - assign_us_geography_ladder, - load_us_block_ladder, - us_geography_ladder_assignment_summary, - us_geography_ladder_gate, - with_household_us_geography_ladder, -) -from microcosm.build.us_runtime.hours_worked import ( - US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS, - US_HOURS_WORKED_OUTPUT_COLUMNS, - US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, - US_HOURS_WORKED_POOL_OUTPUT_COLUMNS, - US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS, - US_HOURS_WORKED_STAGE_NAME, - derive_us_hours_worked_from_manifest, - us_hours_worked_signal_gate, - us_hours_worked_stage_spec, - us_hours_worked_summary, - with_us_hours_worked_inputs, -) -from microcosm.build.us_runtime.housing_inputs import ( - ACS_2022_RENT_ARTIFACT_SHA256, - HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL, - HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL, - HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL, - HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL, - HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL, - HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL, - HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL, - US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_STAGE_NAME, - US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_HOUSING_NONCONSTANT_PERSON_COLUMNS, - US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS, - US_HOUSING_PERSON_OUTPUT_COLUMNS, - US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS, - US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS, - US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS, - derive_us_housing_inputs, - impute_us_housing_assistance_to_puf_support, - impute_us_pre_subsidy_rent, - load_acs_2022_rent_donor, - us_housing_inputs_signal_gate, - us_housing_inputs_stage_spec, - us_housing_inputs_summary, - with_us_housing_inputs, -) -from microcosm.build.us_runtime.immigration import ( - IMMIGRATION_STATUS_VALUES, - SSN_CARD_TYPE_VALUES, - US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, - US_IMMIGRATION_OUTPUT_COLUMNS, - US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, - US_IMMIGRATION_STAGE_NAME, - UndocumentedControls, - derive_us_immigration_status_from_manifest, - us_immigration_composition_gate, - us_immigration_composition_summary, - us_immigration_stage_spec, - with_us_immigration_inputs, -) -from microcosm.build.us_runtime.input_mass import ( - us_input_mass_totals, -) -from microcosm.build.us_runtime.medicaid_take_up import ( - US_MEDICAID_ENROLLMENT_SUBSTITUTIONS, - US_MEDICAID_ENROLLMENT_TARGET_ROLE, - US_MEDICAID_ENROLLMENT_TARGET_TABLE, - US_MEDICAID_ENROLLMENT_TOLERANCE, - US_MEDICAID_TAKE_UP_ANCHOR, - US_MEDICAID_TAKE_UP_STAGE, - US_MEDICAID_TAKE_UP_VARIABLE, - MedicaidEnrollmentSubstitution, - apply_us_medicaid_enrollment_substitutions, - us_medicaid_source_person_table, - us_medicaid_take_up_diagnostics, - us_medicaid_take_up_gate, - with_us_medicaid_take_up, - write_us_medicaid_take_up_diagnostics, -) -from microcosm.build.us_runtime.medicare_take_up import ( - MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL, - MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL, - MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL, - MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL, - US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS, - US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_MEDICARE_TAKE_UP_STAGE_NAME, - derive_us_medicare_take_up_from_manifest, - us_medicare_take_up_signal_gate, - us_medicare_take_up_stage_spec, - us_medicare_take_up_summary, - with_us_medicare_take_up_input, -) -from microcosm.build.us_runtime.misc_itemized import ( - US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS, - US_MISC_ITEMIZED_OUTPUT_COLUMNS, - US_MISC_ITEMIZED_STAGE_NAME, - derive_us_misc_itemized_from_puf, - us_misc_itemized_signal_gate, - us_misc_itemized_stage_spec, - us_misc_itemized_summary, -) -from microcosm.build.us_runtime.nonzero_shares import ( - nonzero_share, - us_nonzero_shares, -) -from microcosm.build.us_runtime.operator_boundary import ( - FORMULA_OWNED_SOURCE_COLUMNS, - PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, - assert_operator_free_source_frame, -) -from microcosm.build.us_runtime.org_wages import ( - BLS_STATE_UNION_REPRESENTATION_RATE_2024, - FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES, - FLSA_OVERTIME_OCCUPATION_CODES, - ORG_2024_DONOR_CONTENT_SHA256, - ORG_2024_DONOR_FILENAME, - ORG_PREDICTORS, - US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS, - US_ORG_WAGES_OUTPUT_COLUMNS, - US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS, - US_ORG_WAGES_STAGE_NAME, - derive_flsa_overtime_premium, - derive_us_org_occupation_inputs, - fetch_org_2024_donor, - impute_us_org_wages, - load_org_2024_donor, - us_org_wages_signal_gate, - us_org_wages_stage_spec, - us_org_wages_summary, - with_us_org_wages_inputs, -) -from microcosm.build.us_runtime.other_health_insurance import ( - OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL, - US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES, - US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, - US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS, - US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS, - US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS, - US_SE_HEALTH_MEDICARE_AGE_THRESHOLD, - US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES, - attribute_us_se_health_premiums, - attribute_us_se_health_premiums_from_manifest, - derive_us_other_health_insurance_from_asec, - derive_us_other_health_insurance_from_manifest, - impute_us_other_health_insurance_to_puf_support_from_manifest, - us_other_health_insurance_signal_gate, - us_other_health_insurance_stage_spec, - us_other_health_insurance_summary, - with_us_other_health_insurance_inputs, -) -from microcosm.build.us_runtime.parity_reference import ( - ECPS_PARITY_KNOWN_GAPS_RESOURCE, - ECPS_PARITY_REFERENCE_RESOURCE, - EcpsParityReference, - EcpsParitySource, - ParityKnownGap, - load_ecps_parity_known_gaps, - load_ecps_parity_reference, -) -from microcosm.build.us_runtime.pregnancy import ( - US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS, - US_PREGNANCY_OUTPUT_COLUMN, - US_PREGNANCY_REQUIRED_SOURCE_COLUMNS, - US_PREGNANCY_STAGE_NAME, - derive_us_pregnancy_from_manifest, - us_pregnancy_signal_gate, - us_pregnancy_stage_spec, - us_pregnancy_summary, - with_us_pregnancy_inputs, -) -from microcosm.build.us_runtime.prior_year_income import ( - PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL, - US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS, - US_PRIOR_YEAR_INCOME_STAGE_NAME, - derive_us_prior_year_income_from_manifest, - impute_us_prior_year_income_to_puf_support_from_manifest, - us_prior_year_income_signal_gate, - us_prior_year_income_source_reconciliation_gate, - us_prior_year_income_stage_spec, - us_prior_year_income_summary, - with_us_prior_year_income_inputs, -) -from microcosm.build.us_runtime.public_assistance_type_source import ( - ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS, - ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS, - PAW_TYPE_TANF_CODES, - PAW_TYPE_VALID_CODES, - fill_asec_public_assistance_type_source, - load_asec_public_assistance_type_sources, -) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET, - PUF_CAPITAL_GAINS_TAIL_QUANTILE, - PUF_CAPITAL_GAINS_TAIL_STAGE_NAME, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, - assert_puf_capital_gains_tail_survives_selection, - puf_capital_gains_tail_concentration_gate, - puf_capital_gains_tail_support_contract_identity, - puf_capital_gains_tail_terminal_support_receipt, - select_puf_capital_gains_tail_donors, - transfer_puf_capital_gains_tail, - validate_puf_capital_gains_tail_manifest, - validate_puf_capital_gains_tail_terminal_support_receipt, - write_puf_capital_gains_tail_manifest, -) -from microcosm.build.us_runtime.puf_donor_io import load_puf_tax_unit_donor -from microcosm.build.us_runtime.puf_e01000_reconciliation import ( - PUF_E01000_RECONCILIATION_SCHEMA_VERSION, - build_puf_e01000_reconciliation_basis, - finalize_puf_e01000_reconciliation, - puf_capital_gains_joint_metrics, - puf_processed_capital_gains_stage, - puf_raw_e01000_stage, -) -from microcosm.build.us_runtime.puf_interest_components import ( - US_PUF_E19200_AGI_BANDS, - US_PUF_E19200_ALL_RETURNS_COMPONENTS, - PufE19200AgiBand, - PufE19200InterestComponents, - split_us_puf_e19200_by_agi_band, -) -from microcosm.build.us_runtime.puf_source_agi import ( - PUF_AGGREGATE_DISAGGREGATION_SEED, - PUF_AGGREGATE_RECIDS, - PUF_SOURCE_YEAR, - PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, - PUF_SYNTHETIC_RECID_START, - source_year_puf_adjusted_gross_income, -) -from microcosm.build.us_runtime.puf_support import ( - BASE_ASEC_SUPPORT_CHANNEL, - PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN, - PUF_TAX_DETAIL_CLONE_INDEX, - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, - PUF_TAX_DETAIL_SUPPORT_CHANNEL, - US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_SUPPORT_FIT_NAME, - US_PUF_SUPPORT_STAGE_NAME, - PufTaxDetailChainInputs, - clone_us_frame_for_puf_support, - finalize_us_puf_tax_detail_predictions, - has_assembled_support_metadata, - has_support_role_metadata, - impute_us_puf_tax_detail_support, - prepare_us_puf_tax_detail_chain_inputs, - puf_tax_detail_clone_mask, - puf_tax_unit_donor_from_arrays, - spine_source_id_column, - support_channel_column, - support_clone_index_column, - support_role_series, - support_source_id_column, -) -from microcosm.build.us_runtime.puma_ladder import ( - PUMA_LADDER_ARTIFACT_SHA256_ATTR, - PUMA_LADDER_VINTAGES_ATTR, - US_PUMA_LADDER_COLUMNS, - US_PUMA_LADDER_DERIVED_LAYERS, - US_PUMA_LADDER_KIND, - US_PUMA_LADDER_SCHEMA_VERSION, - US_PUMA_LADDER_TRACT_COLUMN, - UsPumaLadder, - assign_us_puma_ladder, - load_us_puma_ladder, - us_puma_ladder_assignment_summary, - us_puma_ladder_gate, - with_household_us_puma_ladder, -) -from microcosm.build.us_runtime.puma_ladder_sources import ( - assemble_us_puma_ladder, - parse_tract_to_puma_relationship, -) -from microcosm.build.us_runtime.qbi_inputs import ( - QBI_ARCHIVED_ASSUMPTIONS_URL, - QBI_ARCHIVED_CLONE_URL, - QBI_ARCHIVED_DERIVATION_URL, - QBI_ARCHIVED_EXPORT_URL, - QBI_ARCHIVED_IMPUTATION_URL, - QBI_ARCHIVED_PUF_ARTIFACT_URL, - QBI_ARCHIVED_SIMULATION_URL, - US_QBI_BOOLEAN_OUTPUT_COLUMNS, - US_QBI_NONCONSTANT_PERSON_COLUMNS, - US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, - US_QBI_STAGE_NAME, - us_qbi_inputs_signal_gate, - us_qbi_inputs_stage_spec, - us_qbi_inputs_summary, - with_us_qbi_input_reconciliation, -) -from microcosm.build.us_runtime.reform_coverage_smoke import ( - us_reform_coverage_smoke_gate, -) -from microcosm.build.us_runtime.reform_validation import ( - REFORM_VALIDATION_SCHEMA_VERSION, - ReformValidationSpec, - in_sample_reform_specs, - load_default_reform_specs, - out_of_sample_reform_specs, - reform_validation_payload, - write_reform_validation, -) -from microcosm.build.us_runtime.register_consistency import ( - us_register_consistency_gate, - us_register_contradictions, -) -from microcosm.build.us_runtime.relationship_inputs import ( - US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, - US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - derive_us_relationship_inputs_from_manifest, - us_relationship_inputs_signal_gate, - us_relationship_inputs_stage_spec, - us_relationship_inputs_summary, - with_us_relationship_inputs, -) -from microcosm.build.us_runtime.release_input_coverage import ( - POST_REFERENCE_ECPS_REQUIRED_INPUTS, - SSI_COUNTABLE_RESOURCE_ASSETS, - US_CGD_ROUTE_REQUIRED_INPUTS, - US_RELEASE_INPUT_COVERAGE_RESOURCE, - ReformCoverageProbe, - ReleaseInputColumn, - ReleaseInputCoverageManifest, - assert_release_input_coverage_manifest_current, - load_release_input_coverage_manifest, - us_release_input_coverage_gate, - us_release_input_coverage_required_columns, - us_release_input_coverage_reviewed_exclusions, - us_release_reform_coverage_probes, -) -from microcosm.build.us_runtime.release_target_parity import ( - RED_LINE_COMPILED_FAMILIES, - US_TARGET_PARITY_FEED_FAMILIES_RESOURCE, - US_TARGET_PARITY_MANIFEST_RESOURCE, - TargetFamily, - TargetFence, - TargetParityManifest, - assert_target_parity_manifest_current, - load_target_parity_feed_families, - load_target_parity_manifest, - registry_target_family_ids, - us_release_target_parity_compiled_families, - us_release_target_parity_gate, - us_release_target_parity_reviewed_exclusions, - us_target_family_id, -) -from microcosm.build.us_runtime.retirement_contributions import ( - US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - derive_us_retirement_contributions_from_manifest, - impute_us_retirement_contributions_to_puf_support_from_manifest, - us_retirement_contributions_signal_gate, - us_retirement_contributions_stage_spec, - us_retirement_contributions_summary, - with_us_retirement_contribution_inputs, -) -from microcosm.build.us_runtime.retirement_distributions import ( - RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL, - RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL, - US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - derive_us_retirement_distributions_from_manifest, - impute_us_retirement_distributions_to_puf_support_from_manifest, - us_retirement_distributions_signal_gate, - us_retirement_distributions_stage_spec, - us_retirement_distributions_summary, - with_us_retirement_distribution_inputs, -) -from microcosm.build.us_runtime.salt_refund_income import ( - SALT_REFUND_ARCHIVED_DERIVATION_URL, - SALT_REFUND_ARCHIVED_EXPORT_URL, - SALT_REFUND_ARCHIVED_IMPUTATION_URL, - SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL, - SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL, - US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, - US_SALT_REFUND_OUTPUT_COLUMNS, - US_SALT_REFUND_STAGE_NAME, - derive_us_salt_refund_income_from_puf, - us_salt_refund_income_signal_gate, - us_salt_refund_income_stage_spec, - us_salt_refund_income_summary, -) -from microcosm.build.us_runtime.scf_auto_loans import ( - QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET, - SCF_2022_FULL_EXTRACT_MEMBER, - SCF_2022_FULL_EXTRACT_MEMBER_SHA256, - SCF_2022_FULL_EXTRACT_URL, - SCF_2022_FULL_EXTRACT_ZIP_SHA256, - SCF_AUTO_LOAN_AMOUNT_COLUMNS, - SCF_AUTO_LOAN_RATE_COLUMNS, - US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_AUTO_LOAN_OUTPUT_COLUMNS, - fetch_scf_2022_full_extract, - impute_us_scf_auto_loans, - load_scf_2022_auto_loan_donor, - qualified_auto_loan_interest_proxy, - us_scf_auto_loans_signal_gate, - us_scf_auto_loans_stage_spec, - us_scf_auto_loans_summary, - with_us_scf_auto_loan_inputs, -) -from microcosm.build.us_runtime.scf_wealth import ( - FINANCIAL_ASSET_BLEND_AUDIT_KEY, - FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY, - SCF_FINANCIAL_ASSET_TARGET_COMPONENTS, - SCF_NET_WORTH_TARGET_COMPONENTS, - SCF_WEALTH_PREDICTORS, - US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS, - US_SCF_NET_WORTH_OUTPUT_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS, - US_SCF_WEALTH_STAGE_NAME, - fetch_scf_2022_summary_extract, - financial_asset_source_is_scf, - impute_us_scf_financial_assets, - impute_us_scf_net_worth, - impute_us_sipp_scf_financial_assets, - load_scf_2022_financial_asset_donor, - us_scf_wealth_signal_gate, - us_scf_wealth_stage_spec, - us_scf_wealth_summary, - with_us_scf_wealth_inputs, -) -from microcosm.build.us_runtime.sipp_financial_assets import ( - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS, - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE, - SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION, - SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256, - SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES, - SIPP_2023_FINANCIAL_ASSET_DONOR_URL, - SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN, - SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS, - SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS, - fetch_sipp_2023_financial_asset_donor, - impute_us_sipp_financial_assets, - load_sipp_2023_financial_asset_donor, -) -from microcosm.build.us_runtime.sipp_head_start import ( - HEAD_START_SIPP_DICTIONARY_URL, - SIPP_2023_HEAD_START_DONOR_REVISION, - SIPP_2023_HEAD_START_DONOR_SHA256, - SIPP_2023_HEAD_START_DONOR_SIZE_BYTES, - SIPP_2023_HEAD_START_DONOR_URL, - SIPP_HEAD_START_FIT_PARAMETERS, - SIPP_HEAD_START_MODEL_PREDICTORS, - SIPP_HEAD_START_READ_PARAMETERS, - SIPP_HEAD_START_SOURCE_COLUMNS, - US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_HEAD_START_OUTPUT_COLUMNS, - US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS, - US_SIPP_HEAD_START_STAGE_NAME, - fetch_sipp_2023_head_start_donor, - impute_us_sipp_head_start, - load_sipp_2023_head_start_donor, - us_sipp_head_start_signal_gate, - us_sipp_head_start_stage_spec, - us_sipp_head_start_summary, - with_us_sipp_head_start_input, -) -from microcosm.build.us_runtime.sipp_tips import ( - CENSUS_OCCUPATION_CODE_TO_TTOC, - SIPP_2023_TIP_DONOR_REVISION, - SIPP_2023_TIP_DONOR_SHA256, - SIPP_2023_TIP_DONOR_URL, - SIPP_TIP_OUTPUT_COLUMNS, - SIPP_TIP_PREDICTORS, - US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_TIPS_OUTPUT_COLUMNS, - US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS, - US_SIPP_TIPS_STAGE_NAME, - derive_treasury_tipped_occupation_code, - fetch_sipp_2023_tip_donor, - impute_us_sipp_tips, - load_sipp_2023_tip_donor, - us_sipp_tips_signal_gate, - us_sipp_tips_stage_spec, - us_sipp_tips_summary, - with_us_sipp_tip_inputs, -) -from microcosm.build.us_runtime.sipp_vehicles import ( - SIPP_2023_VEHICLE_DONOR_REVISION, - SIPP_2023_VEHICLE_DONOR_SHA256, - SIPP_2023_VEHICLE_DONOR_SIZE_BYTES, - SIPP_2023_VEHICLE_DONOR_URL, - US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SIPP_VEHICLE_OUTPUT_COLUMNS, - fetch_sipp_2023_vehicle_donor, - load_sipp_2023_vehicle_donor, - us_sipp_vehicles_signal_gate, - us_sipp_vehicles_stage_spec, - us_sipp_vehicles_summary, - with_us_sipp_vehicle_inputs, -) -from microcosm.build.us_runtime.snap_discretionary_exemption import ( - US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN, - US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - derive_us_snap_discretionary_exemption_from_manifest, - us_snap_discretionary_exemption_signal_gate, - us_snap_discretionary_exemption_stage_spec, - us_snap_discretionary_exemption_summary, - with_us_snap_discretionary_exemption_inputs, -) -from microcosm.build.us_runtime.snap_state_take_up import ( - US_SNAP_CASELOAD_TOLERANCE, - US_SNAP_HOUSEHOLDS_TARGET_TABLE, - US_SNAP_STATE_TAKE_UP_ANCHOR, - US_SNAP_STATE_TAKE_UP_STAGE, - us_snap_state_take_up_diagnostics, - us_snap_state_take_up_gate, - with_us_snap_state_take_up, - write_us_snap_state_take_up_diagnostics, -) -from microcosm.build.us_runtime.snap_take_up import ( - US_SNAP_TAKE_UP_OUTPUT_COLUMN, - US_SNAP_TAKE_UP_RAW_COLUMN, - US_SNAP_TAKE_UP_STAGE_NAME, - derive_us_snap_take_up_from_manifest, - us_snap_take_up_signal_gate, - us_snap_take_up_stage_spec, - us_snap_take_up_summary, - with_us_snap_take_up_inputs, -) -from microcosm.build.us_runtime.source_coverage import ( - CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - US_SOURCE_COVERAGE, - hard_target_package_aliases, - source_gap_family_ids, - us_source_coverage_diagnostics, - us_source_coverage_gate, - validation_only_family_ids, - write_us_source_coverage_diagnostics, -) -from microcosm.build.us_runtime.source_runtime import ( - disaggregate_us_puf_aggregate_records_from_manifest, - us_source_operation_handlers, -) -from microcosm.build.us_runtime.spine_agreement import ( - DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE, - DEFAULT_INCIDENCE_RATIO_BOUNDS, - DEFAULT_QUANTILE_ENVELOPE_TOLERANCE, - DEFAULT_SPINE_AGREEMENT_QUANTILES, - US_SPINE_AGREEMENT_REGISTRY, - SpineAgreementSpec, - default_spine_agreement_registry, - normalize_transfer_family_name, - spine_agreement_gate, - validate_spine_agreement_registry, -) -from microcosm.build.us_runtime.spine_assembly import assemble_spines -from microcosm.build.us_runtime.ssi_disability_criteria import ( - SIPP_2023_SSI_DISABILITY_DONOR_REVISION, - SIPP_2023_SSI_DISABILITY_DONOR_SHA256, - SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES, - SIPP_2023_SSI_DISABILITY_DONOR_URL, - SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS, - SIPP_SSI_DISABILITY_FIT_PARAMETERS, - SIPP_SSI_DISABILITY_MODEL_PREDICTORS, - SIPP_SSI_DISABILITY_READ_PARAMETERS, - SIPP_SSI_DISABILITY_SOURCE_COLUMNS, - SSI_DISABILITY_ARCHIVED_CPS_URL, - SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL, - SSI_DISABILITY_ARCHIVED_SIPP_URL, - SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL, - SSI_DISABILITY_SIPP_DICTIONARY_URL, - US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS, - US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS, - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - fetch_sipp_2023_ssi_disability_donor, - impute_us_ssi_disability_criteria, - load_sipp_2023_ssi_disability_donor, - us_ssi_disability_criteria_signal_gate, - us_ssi_disability_criteria_stage_spec, - us_ssi_disability_criteria_summary, - with_us_ssi_disability_criteria, -) -from microcosm.build.us_runtime.ssi_take_up import ( - SSI_TAKE_UP_ARCHIVED_DERIVATION_URL, - SSI_TAKE_UP_ARCHIVED_EXPORT_URL, - SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL, - SSI_TAKE_UP_ARCHIVED_REPORTER_URL, - SSI_TAKE_UP_ARCHIVED_TARGETS_URL, - SSI_TAKE_UP_SSA_SOURCE_URL, - US_SSI_TAKE_UP_ANCHOR, - US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE, - US_SSI_TAKE_UP_ENFORCED_BAND_KEYS, - US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_SSI_TAKE_UP_OUTPUT_COLUMNS, - US_SSI_TAKE_UP_PHASE_ASSIGNMENT, - US_SSI_TAKE_UP_PHASE_RELEASE_FINAL, - US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME, - US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT, - US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_SSI_TAKE_UP_STAGE_NAME, - US_SSI_TAKE_UP_TARGET_TABLE_NAME, - SSITakeUpBandPriorBasis, - SSITakeUpPriorBasis, - ssi_take_up_prior_basis_from_artifact, - ssi_take_up_prior_basis_from_diagnostics, - us_ssi_take_up_delivery_gate, - us_ssi_take_up_diagnostics, - us_ssi_take_up_gate, - us_ssi_take_up_reporter_source_ids, - with_us_ssi_take_up, - write_us_ssi_take_up_diagnostics, -) -from microcosm.build.us_runtime.support_provenance import ( - us_reported_coverage_vintage_signal_gate, -) -from microcosm.build.us_runtime.take_up import ( - US_TAKE_UP_SHARE_BAND, - SeededTakeUpResult, - us_take_up_participation_diagnostics, - us_take_up_signal_gate, - us_take_up_summary, - with_us_take_up_inputs, - write_us_take_up_participation_diagnostics, -) -from microcosm.build.us_runtime.take_up_contract import ( - TakeUpContract, - TakeUpProgram, - assert_take_up_contract_current, - assert_take_up_treatments_consistent, - count_calibrated_take_up_programs, - load_take_up_contract, - seeded_take_up_programs, -) -from microcosm.build.us_runtime.validation_input_coverage import ( - US_VALIDATION_PROVISION_INPUT_LEAVES, - ValidationInputLeaf, - assert_validation_leaf_registry_current, - us_source_stage_outputs, - us_validation_input_coverage_gate, -) -from microcosm.build.us_runtime.voluntary_filing import ( - SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION, - SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256, - SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES, - SIPP_2023_VOLUNTARY_FILING_DONOR_URL, - SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS, - SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS, - US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS, - US_VOLUNTARY_FILING_OUTPUT_COLUMNS, - US_VOLUNTARY_FILING_STAGE_NAME, - VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL, - VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL, - VOLUNTARY_FILING_SIPP_DICTIONARY_URL, - fetch_sipp_2023_voluntary_filing_donor, - impute_us_voluntary_filing, - load_sipp_2023_voluntary_filing_donor, - us_voluntary_filing_signal_gate, - us_voluntary_filing_stage_spec, - us_voluntary_filing_summary, - with_us_voluntary_filing_input, -) -from microcosm.build.us_runtime.weeks_unemployed import ( - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR, - ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL, - US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS, - US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS, - US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL, - WEEKS_UNEMPLOYED_DERIVE_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_PREDICTORS, - WEEKS_UNEMPLOYED_READ_PARAMETERS, - derive_us_weeks_unemployed_from_manifest, - fetch_asec_2023_weeks_unemployed_source, - fill_asec_2022_weeks_unemployed_source, - impute_us_weeks_unemployed_to_puf_support_from_manifest, - load_asec_2023_weeks_unemployed_source, - us_weeks_unemployed_signal_gate, - us_weeks_unemployed_stage_spec, - us_weeks_unemployed_summary, - with_us_weeks_unemployed, -) -from microcosm.build.us_runtime.wic_claim import ( - US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS, - US_WIC_CLAIM_OUTPUT_COLUMNS, - US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS, - US_WIC_CLAIM_STAGE_NAME, - WIC_CLAIM_ARCHIVED_DERIVATION_URL, - WIC_CLAIM_ARCHIVED_PARAMETERS_URL, - WIC_CLAIM_ARCHIVED_RANDOMNESS_URL, - WIC_CLAIM_FNS_SOURCE_URL, - derive_us_wic_claim_from_manifest, - us_wic_claim_signal_gate, - us_wic_claim_stage_spec, - us_wic_claim_summary, - with_us_wic_claim_input, -) -from microcosm.build.us_runtime.workers_compensation import ( - US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS, - US_WORKERS_COMPENSATION_OUTPUT_COLUMNS, - US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS, - US_WORKERS_COMPENSATION_STAGE_NAME, - WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL, - WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL, - derive_us_workers_compensation_from_asec, - derive_us_workers_compensation_from_manifest, - impute_us_workers_compensation_to_puf_support_from_manifest, - us_workers_compensation_signal_gate, - us_workers_compensation_stage_spec, - us_workers_compensation_summary, - with_us_workers_compensation, -) from microcosm.frame import Frame __all__ = [ "ASEC_RAW_STAGE_ARTIFACT_KIND", "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", @@ -1875,6 +828,7 @@ "load_default_congressional_district_vintage_crosswalk", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", "load_puf_tax_unit_donor", "normalize_district_code", "parse_baf_cd_layer", @@ -1990,9 +944,1067 @@ "parse_tract_to_puma_relationship", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] +# Explicit compatibility targets: no discovery or source parsing at runtime. +_EXPORT_MODULES = { + "microcosm.build.us_runtime.adult_care": ( + "US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT", + "US_ADULT_CARE_EARNED_INCOME_SOURCES", + "US_ADULT_CARE_OUTPUT_COLUMNS", + "US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS", + "US_ADULT_CARE_STAGE_NAME", + "derive_us_adult_care_from_manifest", + "us_adult_care_signal_gate", + "us_adult_care_stage_spec", + "us_adult_care_summary", + "with_us_adult_care_inputs", + ), + "microcosm.build.us_runtime.alimony": ( + "ALIMONY_ASEC_ARCHIVED_DERIVATION_URL", + "ALIMONY_PUF_ARCHIVED_DERIVATION_URL", + "STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL", + "US_ALIMONY_NONCONSTANT_PERSON_COLUMNS", + "US_ALIMONY_OUTPUT_COLUMNS", + "US_ALIMONY_STAGE_NAME", + "US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS", + "derive_us_alimony_from_asec", + "derive_us_alimony_from_puf", + "us_alimony_signal_gate", + "us_alimony_stage_spec", + "us_alimony_summary", + ), + "microcosm.build.us_runtime.asec_checkpoint": ( + "ASEC_RAW_STAGE_ARTIFACT_KIND", + "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_OPERATOR_STATUS", + "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_STAGE", + "load_asec_pre_clone_checkpoint", + "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", + ), + "microcosm.build.us_runtime.asec_pool": ( + "AsecSource", + "build_pooled_asec_unit_frame", + "load_asec_h5_tables", + "pool_asec_sources", + ), + "microcosm.build.us_runtime.capital_gain_details": ( + "CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_STAGE_NAME", + "derive_us_capital_gain_details_from_puf", + "us_capital_gain_details_signal_gate", + "us_capital_gain_details_stage_spec", + "us_capital_gain_details_summary", + ), + "microcosm.build.us_runtime.casualty_losses": ( + "US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS", + "US_CASUALTY_LOSS_OUTPUT_COLUMNS", + "US_CASUALTY_LOSS_STAGE_NAME", + "derive_us_casualty_loss_from_puf", + "us_casualty_loss_signal_gate", + "us_casualty_loss_stage_spec", + "us_casualty_loss_summary", + ), + "microcosm.build.us_runtime.child_support": ( + "CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL", + "CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL", + "CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL", + "CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL", + "US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS", + "US_CHILD_SUPPORT_OUTPUT_COLUMNS", + "US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS", + "US_CHILD_SUPPORT_STAGE_NAME", + "derive_us_child_support_from_asec", + "derive_us_child_support_from_manifest", + "impute_us_child_support_to_puf_support_from_manifest", + "us_child_support_signal_gate", + "us_child_support_stage_spec", + "us_child_support_summary", + "with_us_child_support_inputs", + ), + "microcosm.build.us_runtime.childcare": ( + "US_CHILDCARE_OUTPUT_COLUMNS", + "US_CHILDCARE_REQUIRED_SOURCE_COLUMNS", + "US_CHILDCARE_STAGE_NAME", + "derive_us_childcare_from_manifest", + "impute_us_childcare_to_puf_support_from_manifest", + "us_childcare_signal_gate", + "us_childcare_stage_spec", + "us_childcare_summary", + "with_us_childcare_inputs", + ), + "microcosm.build.us_runtime.congressional_district_geography": ( + "CONGRESSIONAL_DISTRICT_GEOID_COLUMN", + "SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID", + "assign_congressional_districts_to_households", + "congressional_district_assignment_summary", + "congressional_district_distribution_from_ledger_facts", + "with_household_congressional_districts", + ), + "microcosm.build.us_runtime.congressional_district_vintage": ( + "CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR", + "CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR", + "CURRENT_CONGRESSIONAL_DISTRICT_PREFIX", + "CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE", + "DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE", + "SOURCE_CONGRESSIONAL_DISTRICT_PREFIX", + "default_congressional_district_vintage_crosswalk_path", + "load_congressional_district_vintage_crosswalk", + "load_default_congressional_district_vintage_crosswalk", + "translate_congressional_district_facts_to_current_vintage", + ), + "microcosm.build.us_runtime.congressional_district_vintage_crosswalk": ( + "CROSSWALK_BASIS_BLOCK_POPULATION", + "build_cd_vintage_crosswalk_rows", + "normalize_district_code", + "parse_baf_cd_layer", + "parse_national_cd_bef_districts", + ), + "microcosm.build.us_runtime.cps_carried": ( + "CPS_CARRIED_FORMULA_OWNED_COLUMNS", + "CPS_CARRIED_PERSON_INPUTS", + "CPS_CARRIED_SPM_UNIT_INPUTS", + "CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN", + "CPS_REPORTED_TANF_TYPE_RAW_COLUMN", + "CPS_REPORTED_WIC_RAW_COLUMN", + "US_REPORTED_COVERAGE_PERSON_INPUTS", + "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", + "WIC_CARRIER_ADJUDICATION_URL", + "derive_us_cps_carried_inputs", + "reported_tanf_enrollment_by_spm_unit", + "reported_wic_receipt_carrier", + ), + "microcosm.build.us_runtime.demographics": ( + "AGE_BANDS", + "DEMOGRAPHICS_SCHEMA_VERSION", + "AgeBand", + "compute_age_distribution", + "demographics_payload", + "write_demographics", + ), + "microcosm.build.us_runtime.disability_benefits": ( + "DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL", + "DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL", + "US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS", + "US_DISABILITY_BENEFITS_OUTPUT_COLUMNS", + "US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS", + "US_DISABILITY_BENEFITS_STAGE_NAME", + "derive_us_disability_benefits_from_asec", + "derive_us_disability_benefits_from_manifest", + "impute_us_disability_benefits_to_puf_support_from_manifest", + "us_disability_benefits_signal_gate", + "us_disability_benefits_stage_spec", + "us_disability_benefits_summary", + "with_us_disability_benefits", + ), + "microcosm.build.us_runtime.domestic_production": ( + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL", + "US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME", + "derive_us_domestic_production_ald_from_puf", + "us_domestic_production_ald_signal_gate", + "us_domestic_production_ald_stage_spec", + "us_domestic_production_ald_summary", + ), + "microcosm.build.us_runtime.education_assistance_source": ( + "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", + "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", + "fetch_asec_education_assistance_source", + "fill_asec_education_assistance_source", + "load_asec_education_assistance_sources", + ), + "microcosm.build.us_runtime.education_inputs": ( + "US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATION_INPUTS_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_EDUCATION_INPUTS_STAGE_NAME", + "derive_us_education_inputs_from_manifest", + "us_education_inputs_signal_gate", + "us_education_inputs_stage_spec", + "us_education_inputs_summary", + "with_us_education_inputs", + ), + "microcosm.build.us_runtime.educator_expenses": ( + "EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL", + "EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL", + "US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS", + "US_EDUCATOR_EXPENSE_STAGE_NAME", + "derive_us_educator_expense_from_puf", + "us_educator_expense_signal_gate", + "us_educator_expense_stage_spec", + "us_educator_expense_summary", + ), + "microcosm.build.us_runtime.eligibility_inputs": ( + "US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS", + "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_ELIGIBILITY_INPUTS_STAGE_NAME", + "derive_us_eligibility_inputs_from_manifest", + "us_eligibility_inputs_signal_gate", + "us_eligibility_inputs_stage_spec", + "us_eligibility_inputs_summary", + "with_us_eligibility_inputs", + ), + "microcosm.build.us_runtime.energy_subsidy": ( + "ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL", + "ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL", + "US_ENERGY_SUBSIDY_OUTPUT_COLUMNS", + "US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS", + "US_ENERGY_SUBSIDY_STAGE_NAME", + "derive_us_energy_subsidy_from_manifest", + "impute_us_energy_subsidy_to_puf_support_from_manifest", + "us_energy_subsidy_signal_gate", + "us_energy_subsidy_stage_spec", + "us_energy_subsidy_summary", + "with_us_energy_subsidy_input", + ), + "microcosm.build.us_runtime.farm_business_income": ( + "FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL", + "US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS", + "US_FARM_BUSINESS_INCOME_STAGE_NAME", + "derive_us_farm_business_income_from_puf", + "us_farm_business_income_signal_gate", + "us_farm_business_income_stage_spec", + "us_farm_business_income_summary", + ), + "microcosm.build.us_runtime.fiscal_targets": ( + "SOI_VARIABLE_MAP", + "US_FISCAL_LEDGER_PARITY_REGISTRY", + "US_FISCAL_LEDGER_PARITY_REPORT", + "US_FISCAL_MACRO_REALISM_BANDS", + "US_FISCAL_TARGET_COVERAGE_REQUIREMENTS", + "US_FISCAL_TARGET_LEDGER_REFERENCES", + "US_FISCAL_TARGET_REFERENCES", + "US_FISCAL_TARGET_REGISTRY", + "US_FISCAL_TARGET_SPECS", + "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS", + "US_JCT_TAX_EXPENDITURE_REFORMS", + "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", + "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", + "US_SOI_FISCAL_TARGET_REFERENCES", + "US_SOI_FISCAL_TARGET_SPECS", + "US_STATE_INCOME_TAX_TARGET_REFERENCES", + "US_STATE_INCOME_TAX_TARGET_SPECS", + "SimpleTaxExpenditureReform", + "compile_us_fiscal_target_registry", + ), + "microcosm.build.us_runtime.form_4952": ( + "FORM_4952_ARCHIVED_DERIVATION_URL", + "FORM_4952_ARCHIVED_EXPORT_URL", + "FORM_4952_ARCHIVED_IMPUTATION_URL", + "FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL", + "FORM_4952_ARCHIVED_PUF_ARTIFACT_URL", + "US_FORM_4952_NONCONSTANT_PERSON_COLUMNS", + "US_FORM_4952_OUTPUT_COLUMNS", + "US_FORM_4952_STAGE_NAME", + "derive_us_form_4952_election_from_puf", + "us_form_4952_election_signal_gate", + "us_form_4952_election_stage_spec", + "us_form_4952_election_summary", + ), + "microcosm.build.us_runtime.geography_ladder": ( + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + "US_BLOCK_LADDER_DERIVED_LAYERS", + "US_BLOCK_LADDER_KIND", + "US_BLOCK_LADDER_SCHEMA_VERSION", + "US_GEOGRAPHY_LADDER_COLUMNS", + "US_NYC_COUNTY_FIPS", + "UsBlockLadder", + "assign_us_geography_ladder", + "load_us_block_ladder", + "us_geography_ladder_assignment_summary", + "us_geography_ladder_gate", + "with_household_us_geography_ladder", + ), + "microcosm.build.us_runtime.hours_worked": ( + "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", + "US_HOURS_WORKED_OUTPUT_COLUMNS", + "US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS", + "US_HOURS_WORKED_POOL_OUTPUT_COLUMNS", + "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", + "US_HOURS_WORKED_STAGE_NAME", + "derive_us_hours_worked_from_manifest", + "us_hours_worked_signal_gate", + "us_hours_worked_stage_spec", + "us_hours_worked_summary", + "with_us_hours_worked_inputs", + ), + "microcosm.build.us_runtime.housing_inputs": ( + "ACS_2022_RENT_ARTIFACT_SHA256", + "HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL", + "HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL", + "HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL", + "US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_STAGE_NAME", + "US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_HOUSING_NONCONSTANT_PERSON_COLUMNS", + "US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS", + "US_HOUSING_PERSON_OUTPUT_COLUMNS", + "US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS", + "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", + "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", + "derive_us_housing_inputs", + "impute_us_housing_assistance_to_puf_support", + "impute_us_pre_subsidy_rent", + "load_acs_2022_rent_donor", + "us_housing_inputs_signal_gate", + "us_housing_inputs_stage_spec", + "us_housing_inputs_summary", + "with_us_housing_inputs", + ), + "microcosm.build.us_runtime.immigration": ( + "IMMIGRATION_STATUS_VALUES", + "SSN_CARD_TYPE_VALUES", + "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", + "US_IMMIGRATION_OUTPUT_COLUMNS", + "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", + "US_IMMIGRATION_STAGE_NAME", + "UndocumentedControls", + "derive_us_immigration_status_from_manifest", + "us_immigration_composition_gate", + "us_immigration_composition_summary", + "us_immigration_stage_spec", + "with_us_immigration_inputs", + ), + "microcosm.build.us_runtime.input_mass": ("us_input_mass_totals",), + "microcosm.build.us_runtime.medicaid_take_up": ( + "US_MEDICAID_ENROLLMENT_SUBSTITUTIONS", + "US_MEDICAID_ENROLLMENT_TARGET_ROLE", + "US_MEDICAID_ENROLLMENT_TARGET_TABLE", + "US_MEDICAID_ENROLLMENT_TOLERANCE", + "US_MEDICAID_TAKE_UP_ANCHOR", + "US_MEDICAID_TAKE_UP_STAGE", + "US_MEDICAID_TAKE_UP_VARIABLE", + "MedicaidEnrollmentSubstitution", + "apply_us_medicaid_enrollment_substitutions", + "us_medicaid_source_person_table", + "us_medicaid_take_up_diagnostics", + "us_medicaid_take_up_gate", + "with_us_medicaid_take_up", + "write_us_medicaid_take_up_diagnostics", + ), + "microcosm.build.us_runtime.medicare_take_up": ( + "MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL", + "MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL", + "MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL", + "MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL", + "US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS", + "US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_MEDICARE_TAKE_UP_STAGE_NAME", + "derive_us_medicare_take_up_from_manifest", + "us_medicare_take_up_signal_gate", + "us_medicare_take_up_stage_spec", + "us_medicare_take_up_summary", + "with_us_medicare_take_up_input", + ), + "microcosm.build.us_runtime.misc_itemized": ( + "US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS", + "US_MISC_ITEMIZED_OUTPUT_COLUMNS", + "US_MISC_ITEMIZED_STAGE_NAME", + "derive_us_misc_itemized_from_puf", + "us_misc_itemized_signal_gate", + "us_misc_itemized_stage_spec", + "us_misc_itemized_summary", + ), + "microcosm.build.us_runtime.nonzero_shares": ("nonzero_share", "us_nonzero_shares"), + "microcosm.build.us_runtime.operator_boundary": ( + "FORMULA_OWNED_SOURCE_COLUMNS", + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "assert_operator_free_source_frame", + ), + "microcosm.build.us_runtime.org_wages": ( + "BLS_STATE_UNION_REPRESENTATION_RATE_2024", + "FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES", + "FLSA_OVERTIME_OCCUPATION_CODES", + "ORG_2024_DONOR_CONTENT_SHA256", + "ORG_2024_DONOR_FILENAME", + "ORG_PREDICTORS", + "US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS", + "US_ORG_WAGES_OUTPUT_COLUMNS", + "US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS", + "US_ORG_WAGES_STAGE_NAME", + "derive_flsa_overtime_premium", + "derive_us_org_occupation_inputs", + "fetch_org_2024_donor", + "impute_us_org_wages", + "load_org_2024_donor", + "us_org_wages_signal_gate", + "us_org_wages_stage_spec", + "us_org_wages_summary", + "with_us_org_wages_inputs", + ), + "microcosm.build.us_runtime.other_health_insurance": ( + "OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL", + "US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES", + "US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME", + "US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS", + "US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS", + "US_SE_HEALTH_MEDICARE_AGE_THRESHOLD", + "US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES", + "attribute_us_se_health_premiums", + "attribute_us_se_health_premiums_from_manifest", + "derive_us_other_health_insurance_from_asec", + "derive_us_other_health_insurance_from_manifest", + "impute_us_other_health_insurance_to_puf_support_from_manifest", + "us_other_health_insurance_signal_gate", + "us_other_health_insurance_stage_spec", + "us_other_health_insurance_summary", + "with_us_other_health_insurance_inputs", + ), + "microcosm.build.us_runtime.parity_reference": ( + "ECPS_PARITY_KNOWN_GAPS_RESOURCE", + "ECPS_PARITY_REFERENCE_RESOURCE", + "EcpsParityReference", + "EcpsParitySource", + "ParityKnownGap", + "load_ecps_parity_known_gaps", + "load_ecps_parity_reference", + ), + "microcosm.build.us_runtime.pregnancy": ( + "US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS", + "US_PREGNANCY_OUTPUT_COLUMN", + "US_PREGNANCY_REQUIRED_SOURCE_COLUMNS", + "US_PREGNANCY_STAGE_NAME", + "derive_us_pregnancy_from_manifest", + "us_pregnancy_signal_gate", + "us_pregnancy_stage_spec", + "us_pregnancy_summary", + "with_us_pregnancy_inputs", + ), + "microcosm.build.us_runtime.prior_year_income": ( + "PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL", + "US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS", + "US_PRIOR_YEAR_INCOME_STAGE_NAME", + "derive_us_prior_year_income_from_manifest", + "impute_us_prior_year_income_to_puf_support_from_manifest", + "us_prior_year_income_signal_gate", + "us_prior_year_income_source_reconciliation_gate", + "us_prior_year_income_stage_spec", + "us_prior_year_income_summary", + "with_us_prior_year_income_inputs", + ), + "microcosm.build.us_runtime.public_assistance_type_source": ( + "ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS", + "ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS", + "PAW_TYPE_TANF_CODES", + "PAW_TYPE_VALID_CODES", + "fill_asec_public_assistance_type_source", + "load_asec_public_assistance_type_sources", + ), + "microcosm.build.us_runtime.operator_column_contracts": ( + "PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "US_PUF_SUPPORT_STAGE_NAME", + "US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "US_QBI_OUTPUT_COLUMNS", + ), + "microcosm.build.us_runtime.puf_capital_gains_tail": ( + "PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION", + "PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET", + "PUF_CAPITAL_GAINS_TAIL_QUANTILE", + "PUF_CAPITAL_GAINS_TAIL_STAGE_NAME", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION", + "assert_puf_capital_gains_tail_survives_selection", + "puf_capital_gains_tail_concentration_gate", + "puf_capital_gains_tail_support_contract_identity", + "puf_capital_gains_tail_terminal_support_receipt", + "select_puf_capital_gains_tail_donors", + "transfer_puf_capital_gains_tail", + "validate_puf_capital_gains_tail_manifest", + "validate_puf_capital_gains_tail_terminal_support_receipt", + "write_puf_capital_gains_tail_manifest", + ), + "microcosm.build.us_runtime.puf_donor_io": ("load_puf_tax_unit_donor",), + "microcosm.build.us_runtime.puf_e01000_reconciliation": ( + "PUF_E01000_RECONCILIATION_SCHEMA_VERSION", + "build_puf_e01000_reconciliation_basis", + "finalize_puf_e01000_reconciliation", + "puf_capital_gains_joint_metrics", + "puf_processed_capital_gains_stage", + "puf_raw_e01000_stage", + ), + "microcosm.build.us_runtime.puf_interest_components": ( + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "PufE19200AgiBand", + "PufE19200InterestComponents", + "split_us_puf_e19200_by_agi_band", + ), + "microcosm.build.us_runtime.puf_source_agi": ( + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", + "source_year_puf_adjusted_gross_income", + ), + "microcosm.build.us_runtime.puf_support": ( + "BASE_ASEC_SUPPORT_CHANNEL", + "PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN", + "PUF_TAX_DETAIL_CLONE_INDEX", + "PUF_TAX_DETAIL_SUPPORT_CHANNEL", + "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", + "US_PUF_SUPPORT_FIT_NAME", + "PufTaxDetailChainInputs", + "clone_us_frame_for_puf_support", + "finalize_us_puf_tax_detail_predictions", + "has_assembled_support_metadata", + "has_support_role_metadata", + "impute_us_puf_tax_detail_support", + "prepare_us_puf_tax_detail_chain_inputs", + "puf_tax_detail_clone_mask", + "puf_tax_unit_donor_from_arrays", + "spine_source_id_column", + "support_channel_column", + "support_clone_index_column", + "support_role_series", + "support_source_id_column", + ), + "microcosm.build.us_runtime.puma_ladder": ( + "PUMA_LADDER_ARTIFACT_SHA256_ATTR", + "PUMA_LADDER_VINTAGES_ATTR", + "US_PUMA_LADDER_COLUMNS", + "US_PUMA_LADDER_DERIVED_LAYERS", + "US_PUMA_LADDER_KIND", + "US_PUMA_LADDER_SCHEMA_VERSION", + "US_PUMA_LADDER_TRACT_COLUMN", + "UsPumaLadder", + "assign_us_puma_ladder", + "load_us_puma_ladder", + "us_puma_ladder_assignment_summary", + "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", + "with_household_us_puma_ladder", + ), + "microcosm.build.us_runtime.puma_ladder_sources": ( + "assemble_us_puma_ladder", + "parse_tract_to_puma_relationship", + ), + "microcosm.build.us_runtime.qbi_inputs": ( + "QBI_ARCHIVED_ASSUMPTIONS_URL", + "QBI_ARCHIVED_CLONE_URL", + "QBI_ARCHIVED_DERIVATION_URL", + "QBI_ARCHIVED_EXPORT_URL", + "QBI_ARCHIVED_IMPUTATION_URL", + "QBI_ARCHIVED_PUF_ARTIFACT_URL", + "QBI_ARCHIVED_SIMULATION_URL", + "US_QBI_NONCONSTANT_PERSON_COLUMNS", + "US_QBI_STAGE_NAME", + "us_qbi_inputs_signal_gate", + "us_qbi_inputs_stage_spec", + "us_qbi_inputs_summary", + "with_us_qbi_input_reconciliation", + ), + "microcosm.build.us_runtime.reform_coverage_smoke": ( + "us_reform_coverage_smoke_gate", + ), + "microcosm.build.us_runtime.reform_validation": ( + "REFORM_VALIDATION_SCHEMA_VERSION", + "ReformValidationSpec", + "in_sample_reform_specs", + "load_default_reform_specs", + "out_of_sample_reform_specs", + "reform_validation_payload", + "write_reform_validation", + ), + "microcosm.build.us_runtime.register_consistency": ( + "us_register_consistency_gate", + "us_register_contradictions", + ), + "microcosm.build.us_runtime.relationship_inputs": ( + "US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS", + "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_RELATIONSHIP_INPUTS_STAGE_NAME", + "derive_us_relationship_inputs_from_manifest", + "us_relationship_inputs_signal_gate", + "us_relationship_inputs_stage_spec", + "us_relationship_inputs_summary", + "with_us_relationship_inputs", + ), + "microcosm.build.us_runtime.release_input_coverage": ( + "POST_REFERENCE_ECPS_REQUIRED_INPUTS", + "SSI_COUNTABLE_RESOURCE_ASSETS", + "US_CGD_ROUTE_REQUIRED_INPUTS", + "US_RELEASE_INPUT_COVERAGE_RESOURCE", + "ReformCoverageProbe", + "ReleaseInputColumn", + "ReleaseInputCoverageManifest", + "assert_release_input_coverage_manifest_current", + "load_release_input_coverage_manifest", + "us_release_input_coverage_gate", + "us_release_input_coverage_required_columns", + "us_release_input_coverage_reviewed_exclusions", + "us_release_reform_coverage_probes", + ), + "microcosm.build.us_runtime.release_target_parity": ( + "RED_LINE_COMPILED_FAMILIES", + "US_TARGET_PARITY_FEED_FAMILIES_RESOURCE", + "US_TARGET_PARITY_MANIFEST_RESOURCE", + "TargetFamily", + "TargetFence", + "TargetParityManifest", + "assert_target_parity_manifest_current", + "load_target_parity_feed_families", + "load_target_parity_manifest", + "registry_target_family_ids", + "us_release_target_parity_compiled_families", + "us_release_target_parity_gate", + "us_release_target_parity_reviewed_exclusions", + "us_target_family_id", + ), + "microcosm.build.us_runtime.retirement_contributions": ( + "US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME", + "derive_us_retirement_contributions_from_manifest", + "impute_us_retirement_contributions_to_puf_support_from_manifest", + "us_retirement_contributions_signal_gate", + "us_retirement_contributions_stage_spec", + "us_retirement_contributions_summary", + "with_us_retirement_contribution_inputs", + ), + "microcosm.build.us_runtime.retirement_distributions": ( + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL", + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL", + "US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME", + "derive_us_retirement_distributions_from_manifest", + "impute_us_retirement_distributions_to_puf_support_from_manifest", + "us_retirement_distributions_signal_gate", + "us_retirement_distributions_stage_spec", + "us_retirement_distributions_summary", + "with_us_retirement_distribution_inputs", + ), + "microcosm.build.us_runtime.salt_refund_income": ( + "SALT_REFUND_ARCHIVED_DERIVATION_URL", + "SALT_REFUND_ARCHIVED_EXPORT_URL", + "SALT_REFUND_ARCHIVED_IMPUTATION_URL", + "SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL", + "SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL", + "US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS", + "US_SALT_REFUND_OUTPUT_COLUMNS", + "US_SALT_REFUND_STAGE_NAME", + "derive_us_salt_refund_income_from_puf", + "us_salt_refund_income_signal_gate", + "us_salt_refund_income_stage_spec", + "us_salt_refund_income_summary", + ), + "microcosm.build.us_runtime.scf_auto_loans": ( + "QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET", + "SCF_2022_FULL_EXTRACT_MEMBER", + "SCF_2022_FULL_EXTRACT_MEMBER_SHA256", + "SCF_2022_FULL_EXTRACT_URL", + "SCF_2022_FULL_EXTRACT_ZIP_SHA256", + "SCF_AUTO_LOAN_AMOUNT_COLUMNS", + "SCF_AUTO_LOAN_RATE_COLUMNS", + "US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_AUTO_LOAN_OUTPUT_COLUMNS", + "fetch_scf_2022_full_extract", + "impute_us_scf_auto_loans", + "load_scf_2022_auto_loan_donor", + "qualified_auto_loan_interest_proxy", + "us_scf_auto_loans_signal_gate", + "us_scf_auto_loans_stage_spec", + "us_scf_auto_loans_summary", + "with_us_scf_auto_loan_inputs", + ), + "microcosm.build.us_runtime.scf_wealth": ( + "FINANCIAL_ASSET_BLEND_AUDIT_KEY", + "FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY", + "SCF_FINANCIAL_ASSET_TARGET_COMPONENTS", + "SCF_NET_WORTH_TARGET_COMPONENTS", + "SCF_WEALTH_PREDICTORS", + "US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS", + "US_SCF_NET_WORTH_OUTPUT_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS", + "US_SCF_WEALTH_STAGE_NAME", + "fetch_scf_2022_summary_extract", + "financial_asset_source_is_scf", + "impute_us_scf_financial_assets", + "impute_us_scf_net_worth", + "impute_us_sipp_scf_financial_assets", + "load_scf_2022_financial_asset_donor", + "us_scf_wealth_signal_gate", + "us_scf_wealth_stage_spec", + "us_scf_wealth_summary", + "with_us_scf_wealth_inputs", + ), + "microcosm.build.us_runtime.sipp_financial_assets": ( + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES", + "SIPP_2023_FINANCIAL_ASSET_DONOR_URL", + "SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN", + "SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS", + "SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS", + "fetch_sipp_2023_financial_asset_donor", + "impute_us_sipp_financial_assets", + "load_sipp_2023_financial_asset_donor", + ), + "microcosm.build.us_runtime.sipp_head_start": ( + "HEAD_START_SIPP_DICTIONARY_URL", + "SIPP_2023_HEAD_START_DONOR_REVISION", + "SIPP_2023_HEAD_START_DONOR_SHA256", + "SIPP_2023_HEAD_START_DONOR_SIZE_BYTES", + "SIPP_2023_HEAD_START_DONOR_URL", + "SIPP_HEAD_START_FIT_PARAMETERS", + "SIPP_HEAD_START_MODEL_PREDICTORS", + "SIPP_HEAD_START_READ_PARAMETERS", + "SIPP_HEAD_START_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_HEAD_START_OUTPUT_COLUMNS", + "US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_STAGE_NAME", + "fetch_sipp_2023_head_start_donor", + "impute_us_sipp_head_start", + "load_sipp_2023_head_start_donor", + "us_sipp_head_start_signal_gate", + "us_sipp_head_start_stage_spec", + "us_sipp_head_start_summary", + "with_us_sipp_head_start_input", + ), + "microcosm.build.us_runtime.sipp_tips": ( + "CENSUS_OCCUPATION_CODE_TO_TTOC", + "SIPP_2023_TIP_DONOR_REVISION", + "SIPP_2023_TIP_DONOR_SHA256", + "SIPP_2023_TIP_DONOR_URL", + "SIPP_TIP_OUTPUT_COLUMNS", + "SIPP_TIP_PREDICTORS", + "US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_TIPS_OUTPUT_COLUMNS", + "US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_TIPS_STAGE_NAME", + "derive_treasury_tipped_occupation_code", + "fetch_sipp_2023_tip_donor", + "impute_us_sipp_tips", + "load_sipp_2023_tip_donor", + "us_sipp_tips_signal_gate", + "us_sipp_tips_stage_spec", + "us_sipp_tips_summary", + "with_us_sipp_tip_inputs", + ), + "microcosm.build.us_runtime.sipp_vehicles": ( + "SIPP_2023_VEHICLE_DONOR_REVISION", + "SIPP_2023_VEHICLE_DONOR_SHA256", + "SIPP_2023_VEHICLE_DONOR_SIZE_BYTES", + "SIPP_2023_VEHICLE_DONOR_URL", + "US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SIPP_VEHICLE_OUTPUT_COLUMNS", + "fetch_sipp_2023_vehicle_donor", + "load_sipp_2023_vehicle_donor", + "us_sipp_vehicles_signal_gate", + "us_sipp_vehicles_stage_spec", + "us_sipp_vehicles_summary", + "with_us_sipp_vehicle_inputs", + ), + "microcosm.build.us_runtime.snap_discretionary_exemption": ( + "US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN", + "US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME", + "derive_us_snap_discretionary_exemption_from_manifest", + "us_snap_discretionary_exemption_signal_gate", + "us_snap_discretionary_exemption_stage_spec", + "us_snap_discretionary_exemption_summary", + "with_us_snap_discretionary_exemption_inputs", + ), + "microcosm.build.us_runtime.snap_state_take_up": ( + "US_SNAP_CASELOAD_TOLERANCE", + "US_SNAP_HOUSEHOLDS_TARGET_TABLE", + "US_SNAP_STATE_TAKE_UP_ANCHOR", + "US_SNAP_STATE_TAKE_UP_STAGE", + "us_snap_state_take_up_diagnostics", + "us_snap_state_take_up_gate", + "with_us_snap_state_take_up", + "write_us_snap_state_take_up_diagnostics", + ), + "microcosm.build.us_runtime.snap_take_up": ( + "US_SNAP_TAKE_UP_OUTPUT_COLUMN", + "US_SNAP_TAKE_UP_RAW_COLUMN", + "US_SNAP_TAKE_UP_STAGE_NAME", + "derive_us_snap_take_up_from_manifest", + "us_snap_take_up_signal_gate", + "us_snap_take_up_stage_spec", + "us_snap_take_up_summary", + "with_us_snap_take_up_inputs", + ), + "microcosm.build.us_runtime.source_coverage": ( + "CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "US_SOURCE_COVERAGE", + "hard_target_package_aliases", + "source_gap_family_ids", + "us_source_coverage_diagnostics", + "us_source_coverage_gate", + "validation_only_family_ids", + "write_us_source_coverage_diagnostics", + ), + "microcosm.build.us_runtime.source_runtime": ( + "disaggregate_us_puf_aggregate_records_from_manifest", + "us_source_operation_handlers", + ), + "microcosm.build.us_runtime.spine_agreement": ( + "DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE", + "DEFAULT_INCIDENCE_RATIO_BOUNDS", + "DEFAULT_QUANTILE_ENVELOPE_TOLERANCE", + "DEFAULT_SPINE_AGREEMENT_QUANTILES", + "US_SPINE_AGREEMENT_REGISTRY", + "SpineAgreementSpec", + "default_spine_agreement_registry", + "normalize_transfer_family_name", + "spine_agreement_gate", + "validate_spine_agreement_registry", + ), + "microcosm.build.us_runtime.spine_assembly": ("assemble_spines",), + "microcosm.build.us_runtime.ssi_disability_criteria": ( + "SIPP_2023_SSI_DISABILITY_DONOR_REVISION", + "SIPP_2023_SSI_DISABILITY_DONOR_SHA256", + "SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES", + "SIPP_2023_SSI_DISABILITY_DONOR_URL", + "SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS", + "SIPP_SSI_DISABILITY_FIT_PARAMETERS", + "SIPP_SSI_DISABILITY_MODEL_PREDICTORS", + "SIPP_SSI_DISABILITY_READ_PARAMETERS", + "SIPP_SSI_DISABILITY_SOURCE_COLUMNS", + "SSI_DISABILITY_ARCHIVED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_SIPP_URL", + "SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL", + "SSI_DISABILITY_SIPP_DICTIONARY_URL", + "US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME", + "fetch_sipp_2023_ssi_disability_donor", + "impute_us_ssi_disability_criteria", + "load_sipp_2023_ssi_disability_donor", + "us_ssi_disability_criteria_signal_gate", + "us_ssi_disability_criteria_stage_spec", + "us_ssi_disability_criteria_summary", + "with_us_ssi_disability_criteria", + ), + "microcosm.build.us_runtime.ssi_take_up": ( + "SSI_TAKE_UP_ARCHIVED_DERIVATION_URL", + "SSI_TAKE_UP_ARCHIVED_EXPORT_URL", + "SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL", + "SSI_TAKE_UP_ARCHIVED_REPORTER_URL", + "SSI_TAKE_UP_ARCHIVED_TARGETS_URL", + "SSI_TAKE_UP_SSA_SOURCE_URL", + "US_SSI_TAKE_UP_ANCHOR", + "US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE", + "US_SSI_TAKE_UP_ENFORCED_BAND_KEYS", + "US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_TAKE_UP_OUTPUT_COLUMNS", + "US_SSI_TAKE_UP_PHASE_ASSIGNMENT", + "US_SSI_TAKE_UP_PHASE_RELEASE_FINAL", + "US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME", + "US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT", + "US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_SSI_TAKE_UP_STAGE_NAME", + "US_SSI_TAKE_UP_TARGET_TABLE_NAME", + "SSITakeUpBandPriorBasis", + "SSITakeUpPriorBasis", + "ssi_take_up_prior_basis_from_artifact", + "ssi_take_up_prior_basis_from_diagnostics", + "us_ssi_take_up_delivery_gate", + "us_ssi_take_up_diagnostics", + "us_ssi_take_up_gate", + "us_ssi_take_up_reporter_source_ids", + "with_us_ssi_take_up", + "write_us_ssi_take_up_diagnostics", + ), + "microcosm.build.us_runtime.support_provenance": ( + "us_reported_coverage_vintage_signal_gate", + ), + "microcosm.build.us_runtime.take_up": ( + "US_TAKE_UP_SHARE_BAND", + "SeededTakeUpResult", + "us_take_up_participation_diagnostics", + "us_take_up_signal_gate", + "us_take_up_summary", + "with_us_take_up_inputs", + "write_us_take_up_participation_diagnostics", + ), + "microcosm.build.us_runtime.take_up_contract": ( + "TakeUpContract", + "TakeUpProgram", + "assert_take_up_contract_current", + "assert_take_up_treatments_consistent", + "count_calibrated_take_up_programs", + "load_take_up_contract", + "seeded_take_up_programs", + ), + "microcosm.build.us_runtime.validation_input_coverage": ( + "US_VALIDATION_PROVISION_INPUT_LEAVES", + "ValidationInputLeaf", + "assert_validation_leaf_registry_current", + "us_source_stage_outputs", + "us_validation_input_coverage_gate", + ), + "microcosm.build.us_runtime.voluntary_filing": ( + "SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES", + "SIPP_2023_VOLUNTARY_FILING_DONOR_URL", + "SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS", + "SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS", + "US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_VOLUNTARY_FILING_OUTPUT_COLUMNS", + "US_VOLUNTARY_FILING_STAGE_NAME", + "VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL", + "VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL", + "VOLUNTARY_FILING_SIPP_DICTIONARY_URL", + "fetch_sipp_2023_voluntary_filing_donor", + "impute_us_voluntary_filing", + "load_sipp_2023_voluntary_filing_donor", + "us_voluntary_filing_signal_gate", + "us_voluntary_filing_stage_spec", + "us_voluntary_filing_summary", + "with_us_voluntary_filing_input", + ), + "microcosm.build.us_runtime.weeks_unemployed": ( + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR", + "ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL", + "US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS", + "US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS", + "US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS", + "US_WEEKS_UNEMPLOYED_STAGE_NAME", + "WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL", + "WEEKS_UNEMPLOYED_DERIVE_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_PREDICTORS", + "WEEKS_UNEMPLOYED_READ_PARAMETERS", + "derive_us_weeks_unemployed_from_manifest", + "fetch_asec_2023_weeks_unemployed_source", + "fill_asec_2022_weeks_unemployed_source", + "impute_us_weeks_unemployed_to_puf_support_from_manifest", + "load_asec_2023_weeks_unemployed_source", + "us_weeks_unemployed_signal_gate", + "us_weeks_unemployed_stage_spec", + "us_weeks_unemployed_summary", + "with_us_weeks_unemployed", + ), + "microcosm.build.us_runtime.wic_claim": ( + "US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS", + "US_WIC_CLAIM_OUTPUT_COLUMNS", + "US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS", + "US_WIC_CLAIM_STAGE_NAME", + "WIC_CLAIM_ARCHIVED_DERIVATION_URL", + "WIC_CLAIM_ARCHIVED_PARAMETERS_URL", + "WIC_CLAIM_ARCHIVED_RANDOMNESS_URL", + "WIC_CLAIM_FNS_SOURCE_URL", + "derive_us_wic_claim_from_manifest", + "us_wic_claim_signal_gate", + "us_wic_claim_stage_spec", + "us_wic_claim_summary", + "with_us_wic_claim_input", + ), + "microcosm.build.us_runtime.workers_compensation": ( + "US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS", + "US_WORKERS_COMPENSATION_OUTPUT_COLUMNS", + "US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS", + "US_WORKERS_COMPENSATION_STAGE_NAME", + "WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL", + "WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL", + "derive_us_workers_compensation_from_asec", + "derive_us_workers_compensation_from_manifest", + "impute_us_workers_compensation_to_puf_support_from_manifest", + "us_workers_compensation_signal_gate", + "us_workers_compensation_stage_spec", + "us_workers_compensation_summary", + "with_us_workers_compensation", + ), +} +_LAZY_EXPORTS = { + name: (module, name) for module, names in _EXPORT_MODULES.items() for name in names +} + @dataclass(frozen=True) class BuildConfig: @@ -2045,428 +2057,560 @@ def to_manifest(self) -> dict[str, object]: } -#: The US donor graph: every imputation stage's primary survey, with -#: citations. This is the single place the build's sources are declared — -#: the observatory's sources diagram and the dataset card derive from it. -US_DONORS: Mapping[str, DonorSpec] = { - "scf_wealth": DonorSpec( - survey="Fed SCF 2022 + Census SIPP 2023", - source="https://www.federalreserve.gov/econres/scfindex.htm", - notes=( - "SCF anchors signed net worth and one half of household liquid-asset " - "vectors; the immutable SIPP 2023 public-use donor supplies the " - "other half and restores low liquid-asset mass. Auto loans use the " - "full SCF separately." - ), - ), - US_SSI_DISABILITY_CRITERIA_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Latent under-65 SSI disability/blindness criterion from the " - "pinned full 2023 public-use donor. ASEC and PUF-support people " - "are predicted separately, and only direct under-65 ASEC SSI " - "reporters receive the observed-reporter anchor." - ), - ), - US_SIPP_HEAD_START_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Direct December nursery/preschool federally sponsored-program " - "responses train a weighted Head Start take-up proxy for ages " - "3--5; strict reported structural negatives exclude hot-decked " - "answers and the prediction is shared by support clones." - ), - ), - US_SSI_TAKE_UP_STAGE_NAME: DonorSpec( - survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", - source=SSI_TAKE_UP_SSA_SOURCE_URL, - notes=( - "Direct ASEC SSI_VAL reporters anchor person-level take-up; " - "eligible source-person identities are count-calibrated by age to " - "SSA December 2024 Federal-payment recipient counts and fanned to " - "both support clones." - ), - ), - "sipp_tips": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes="Tip income for tipped occupations.", - ), - "org_wages": DonorSpec( - survey="CPS ORG", - source=("https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/"), - notes=( - "Hourly-wage labor-market inputs. Donor load failures abort the " - "build — the silent zero-fallback this stage once had is " - "structurally impossible under StagePlan." - ), - ), - "meps_esi_premiums": DonorSpec( - survey="MEPS-IC", - source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", - notes="Employer-sponsored insurance premium parameters.", - ), - "aca_marketplace_inputs": DonorSpec( - survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", - source="https://www.cms.gov/marketplace/resources/data/public-use-files", - notes=( - "Marketplace take-up and selected-plan inputs: CPS reported " - "Marketplace coverage and premium reports anchor the records; " - "CMS OEP enrollment, APTC, and metal-level tables provide the " - "calibration targets." - ), - ), - "medicaid_take_up": DonorSpec( - survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", - source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", - notes=( - "Medicaid take-up by anchored count-calibration (contract " - "treatment count_calibrated, microcosm #331): CPS-reported " - "Medicaid coverage at interview anchors the flag; the fill is " - "calibrated to CMS December 2024 state enrollment snapshots. " - "Point-in-time semantics per #332; heals the #170 " - "enrollment==eligibility degeneracy." - ), - ), - US_SNAP_STATE_TAKE_UP_STAGE: DonorSpec( - survey=( - "Census CPS ASEC reported receipt + USDA FNS state " - "average-monthly household caseloads" - ), - source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", - notes=( - "SNAP take-up by anchored count-calibration (contract treatment " - "count_calibrated, microcosm #372): reported ASEC receipt anchors " - "the flag; the fill is calibrated per state to FNS FY2024 " - "average-monthly household counts among eligible non-anchored " - "units, replacing the national snap_take_up fill that bakes in " - "state-dependent CPS underreporting." - ), - ), - US_OTHER_HEALTH_INSURANCE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " - "Marketplace, and Medicaid premiums after take-up stages; an " - "ASEC-trained weighted QRF replaces both premium leaves on the " - "PUF support half." - ), - ), - US_PRIOR_YEAR_INCOME_STAGE_NAME: DonorSpec( - survey="CPS ASEC (prior year)", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Adjacent-year PERIDNUM join for measured prior-year earnings, " - "with Census allocation flags and sentinels enforced. A joint " - "eight-predictor weighted QRF replaces both earnings leaves on " - "the PUF support half; signed self-employment losses survive." - ), - ), - US_IMMIGRATION_STAGE_NAME: DonorSpec( - survey="CPS ASEC + published unauthorized-population estimates", - source=( - "https://www.pewresearch.org/short-reads/2024/07/22/" - "what-we-know-about-unauthorized-immigrants-living-in-the-us/" - ), - notes=( - "SSN card type and immigration status from ASEC citizenship, " - "entry-year, nativity, and program-participation fields via the " - "ASEC-UA residual method (SSRN 4662801), targeted to published " - "undocumented population/worker/student control totals." - ), - ), - US_HOURS_WORKED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Hours-worked inputs mapped directly from measured ASEC person " - "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " - "them the engine defaults every person to 40 weekly hours and " - "hours-conditioned rules (SNAP work requirements) become no-ops." - ), - ), - US_SNAP_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + USDA FNS participation rate estimates", - source="https://www.fns.usda.gov/snap/participation-rates", - notes=( - "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " - "up; non-reporting units drawn to the cited FNS participation " - "rate. Without it the engine defaults every eligible unit to " - "100% take-up." - ), - ), - US_ELIGIBILITY_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "SNAP eligibility/exemption inputs mapped directly from " - "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " - "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " - "them disability, student, parent/child, and veteran " - "exemption channels default to False/0." - ), - ), - US_RELATIONSHIP_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured household-head and marital-status flags mapped exactly " - "from P_SEQ and A_MARITL; nothing is imputed." - ), - ), - US_MEDICARE_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured Medicare enrollment mapped exactly from MCARE == 1 and " - "copied onto the PUF support clone; no take-up rate is applied." - ), - ), - US_RETIREMENT_DISTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " - "archived CPS-only QRF replaces the four populated non-IRA leaves " - "on PUF support while preserving IRA channel ownership. Nothing " - "is allocated across accounts." - ), - ), - US_PREGNANCY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", - source="https://www.cdc.gov/nchs/nvss/births.htm", - notes=( - "Pregnancy seeded among women 15-44 at the national " - "point-in-time rate (births x 39/52 over female 15-44 " - "population), matching the retired pipeline's national " - "fallback; state-level rates are follow-up work (#351). " - "The ASEC does not measure pregnancy." - ), - ), - US_WIC_CLAIM_STAGE_NAME: DonorSpec( - survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", - source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", - notes=( - "Stable person-level claim draws use official CY2022 FNS category " - "coverage rates after the pregnancy and parent-input stages. The " - "all-postpartum rate is used because no hermetic source identifies " - "breastfeeding; nutritional risk remains separately excluded." - ), - ), - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", - source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", - notes=( - "ABAWD discretionary exemptions seeded at the statutory cap " - "(8% from FY2024) across potentially covered adults 18-64; " - "the engine intersects with modeled coverage. Assumes full " - "state usage of the cap (#323)." - ), - ), - US_CHILDCARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_CHILDCAREXPNS is validated and carried " - "to the SPM-unit childcare leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_ADULT_CARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " - "qualifying flag is the measured PEDISDRS self-care difficulty " - "item; the expense leg is a seeded, weight-targeted draw from " - "the measured ASEC childcare-expense distribution (the same " - "section 21 expense class), restricted to tax units where the " - "statute can bind, with the 21(d)(2) spouse deeming honored. " - "Neither ASEC nor the SIPP 2023 PUF releases an in-household " - "adult-care expenditure amount, so the level proxy is declared " - "in the stage manifest." - ), - ), - US_ENERGY_SUBSIDY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_ENGVAL is validated and carried to the " - "SPM-unit energy-subsidy leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_CHILD_SUPPORT_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " - "are carried directly. After PUF tax-detail imputation, one " - "ASEC-trained weighted QRF jointly replaces both person leaves " - "on the PUF support half using the archived predictor subset." - ), - ), - US_DISABILITY_BENEFITS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " - "when their source code is not workers' compensation. After PUF " - "tax-detail imputation, an ASEC-trained weighted QRF replaces the " - "CPS-only person leaf on the PUF support half." - ), - ), - US_WORKERS_COMPENSATION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual WC_VAL is carried directly. After PUF tax-detail " - "imputation, an ASEC-trained weighted QRF replaces the CPS-only " - "person leaf on the PUF support half." - ), - ), - US_WEEKS_UNEMPLOYED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured LKWEEKS is carried directly, including an exact " - "identity-keyed repair of the omitted 2022-income-year column " - "from the pinned official 2023 ASEC archive. An ASEC-trained " - "QRF then replaces only the PUF support half, with the archived " - "unemployment-compensation zero rule." - ), - ), - "puf_tax_detail": DonorSpec( - survey="IRS PUF 2015 (uprated)", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Itemized-deduction detail, versioned processed-PUF Section 199A " - "simulation leaves (carried without redrawing), partnership SE, " - "source-year-AGI E19200 mortgage/non-mortgage split, direct " - "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " - "miscellaneous-expense proxy. The pinned processed PUF uprates its " - "raw TY2015 rows before seeded disclosure-record replacement and " - "includes a Forbes-backed 3,900-record open tail; this runtime " - "reconstructs the bounded-record AGI lineage and anchors every " - "Forbes-tail record in the final published AGI band without rerunning " - "Forbes synthesis. Support is clipped to the PUF's realized ranges." - ), - ), - US_EDUCATION_INPUTS_STAGE_NAME: DonorSpec( - survey="IRS PUF 2015 (uprated) + Census CPS ASEC", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Qualified tuition comes from the PUF E03230/E87530 maximum; " - "the published retired path drops the reported AOTC output and its " - "five affirmative factual inputs therefore follow positive tuition; " - "educational assistance carries directly from ASEC ED_VAL." - ), - ), - US_RETIREMENT_CONTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + published retirement-contribution shares", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured ASEC RETCB_VAL is allocated across five desired " - "retirement-contribution leaves using archived IRS/BEA/Vanguard/" - "PSCA shares, then CPS-trained QRF predictions replace the PUF " - "support half. PolicyEngine-US owns all statutory caps." - ), - ), - "capital_gain_distributions": DonorSpec( - survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", - source=( - "https://www.irs.gov/statistics/" - "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" - ), - notes=( - "Schedule D line 13 capital gain distributions split out of " - "long-term gains as a memo component at the national SOCA-derived " - "share; the direct-1040 route is already a PUF-stage output and " - "the two routes are mutually exclusive on a real return." - ), - ), - US_HOUSING_INPUTS_STAGE_NAME: DonorSpec( - survey="Census ACS 2022", - source="https://www.census.gov/programs-surveys/acs", - notes=( - "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " - "rent imputation on household heads; reported housing assistance " - "alone receives the archived second-stage PUF-support QRF." - ), - ), - "vehicle_assets": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Household vehicle count and value from the pinned full 2023 " - "public-use donor. They remain independent policy inputs until " - "the full mixed-source net-worth reconciliation is restored." - ), - ), - US_VOLUNTARY_FILING_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Measured 2023 SIPP filing and expected-filing responses replace " - "the retired uncited demographic probability table. Reciprocal " - "spouses form one source unit, reported dependents are excluded, " - "and one weighted-QRF prediction is shared across support clones." - ), - ), -} +# Annotations preserve the public type surface without initializing lazy values. +US_DONORS: Mapping[str, DonorSpec] +US_STAGE_NAMES: tuple[str, ...] +US_SOURCE_MANIFEST: SourceManifest +US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] +US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] +US_SUPPORT_SPINE_MANIFEST: SupportSpineManifest +US_SUPPORT_SPINE_SPEC: SupportSpineSpec + +_EXPORT_LOCK = RLock() + + +def _resolve_export(name: str): + if name in globals(): + return globals()[name] + module, attribute = _LAZY_EXPORTS[name] + # Never hold our lock across Python's module import locks. + value = getattr(import_module(module), attribute) + with _EXPORT_LOCK: + return globals().setdefault(name, value) + + +def _initialize_source_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SOURCE_EXPORTS): + return + manifest = globals().get("US_SOURCE_MANIFEST") + if manifest is None: + manifest = _load_us_source_manifest() + values = { + "US_SOURCE_MANIFEST": manifest, + "US_SOURCE_STAGE_SPECS": manifest.stages, + "US_NONNEGATIVE_SOURCE_OUTPUTS": frozenset( + output + for stage in manifest.stages + for output in stage.nonnegative_outputs + ), + } + for name, value in values.items(): + globals().setdefault(name, value) + + +def _initialize_support_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SUPPORT_EXPORTS): + return + manifest = globals().get("US_SUPPORT_SPINE_MANIFEST") + if manifest is None: + manifest = _load_us_support_spine_manifest() + values = { + "US_SUPPORT_SPINE_MANIFEST": manifest, + "US_SUPPORT_SPINE_SPEC": manifest.support_spine, + } + for name, value in values.items(): + globals().setdefault(name, value) -#: Stage order of the US build. Derivation stages (no donor) interleave with -#: the donor imputations; the export/calibration stages close the plan. -US_STAGE_NAMES: tuple[str, ...] = ( - "asec_load", - "unit_assignment", - "derive_cps_carried", - US_PRIOR_YEAR_INCOME_STAGE_NAME, - US_IMMIGRATION_STAGE_NAME, - US_HOURS_WORKED_STAGE_NAME, - US_SNAP_TAKE_UP_STAGE_NAME, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - US_MEDICARE_TAKE_UP_STAGE_NAME, - US_HOUSING_INPUTS_STAGE_NAME, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - US_PREGNANCY_STAGE_NAME, - US_WIC_CLAIM_STAGE_NAME, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - US_CHILDCARE_STAGE_NAME, - US_ADULT_CARE_STAGE_NAME, - US_ENERGY_SUBSIDY_STAGE_NAME, - US_PUF_SUPPORT_STAGE_NAME, - "puf_tax_detail", - US_CHILD_SUPPORT_STAGE_NAME, - US_DISABILITY_BENEFITS_STAGE_NAME, - US_WORKERS_COMPENSATION_STAGE_NAME, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - US_EDUCATION_INPUTS_STAGE_NAME, - "capital_gain_distributions", - "scf_wealth", - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - US_SIPP_HEAD_START_STAGE_NAME, - US_SSI_TAKE_UP_STAGE_NAME, - "sipp_tips", - "org_wages", - "meps_esi_premiums", - "mortgage_conversion", - "vehicle_assets", - US_VOLUNTARY_FILING_STAGE_NAME, - "entity_placement", - "aca_marketplace_inputs", - "medicaid_take_up", - US_SNAP_STATE_TAKE_UP_STAGE, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - "export", + +_SOURCE_EXPORTS = ( + "US_SOURCE_MANIFEST", + "US_SOURCE_STAGE_SPECS", + "US_NONNEGATIVE_SOURCE_OUTPUTS", ) +_SUPPORT_EXPORTS = ("US_SUPPORT_SPINE_MANIFEST", "US_SUPPORT_SPINE_SPEC") + + +def __getattr__(name: str): + if name in _LAZY_EXPORTS: + return _resolve_export(name) + if name in ("US_DONORS", "US_STAGE_NAMES"): + _initialize_plan() + elif name in _SOURCE_EXPORTS: + _initialize_source_manifest() + elif name in _SUPPORT_EXPORTS: + _initialize_support_manifest() + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return globals()[name] + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) + + +def _initialize_plan() -> None: + if "US_DONORS" in globals() and "US_STAGE_NAMES" in globals(): + return + ssi_take_up_ssa_source_url = _resolve_export("SSI_TAKE_UP_SSA_SOURCE_URL") + us_adult_care_stage_name = _resolve_export("US_ADULT_CARE_STAGE_NAME") + us_childcare_stage_name = _resolve_export("US_CHILDCARE_STAGE_NAME") + us_child_support_stage_name = _resolve_export("US_CHILD_SUPPORT_STAGE_NAME") + us_disability_benefits_stage_name = _resolve_export( + "US_DISABILITY_BENEFITS_STAGE_NAME" + ) + us_education_inputs_stage_name = _resolve_export("US_EDUCATION_INPUTS_STAGE_NAME") + us_eligibility_inputs_stage_name = _resolve_export( + "US_ELIGIBILITY_INPUTS_STAGE_NAME" + ) + us_energy_subsidy_stage_name = _resolve_export("US_ENERGY_SUBSIDY_STAGE_NAME") + us_hours_worked_stage_name = _resolve_export("US_HOURS_WORKED_STAGE_NAME") + us_housing_inputs_stage_name = _resolve_export("US_HOUSING_INPUTS_STAGE_NAME") + us_immigration_stage_name = _resolve_export("US_IMMIGRATION_STAGE_NAME") + us_medicare_take_up_stage_name = _resolve_export("US_MEDICARE_TAKE_UP_STAGE_NAME") + us_other_health_insurance_stage_name = _resolve_export( + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME" + ) + us_pregnancy_stage_name = _resolve_export("US_PREGNANCY_STAGE_NAME") + us_prior_year_income_stage_name = _resolve_export("US_PRIOR_YEAR_INCOME_STAGE_NAME") + us_puf_support_stage_name = _resolve_export("US_PUF_SUPPORT_STAGE_NAME") + us_relationship_inputs_stage_name = _resolve_export( + "US_RELATIONSHIP_INPUTS_STAGE_NAME" + ) + us_retirement_contribution_stage_name = _resolve_export( + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME" + ) + us_retirement_distribution_stage_name = _resolve_export( + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME" + ) + us_sipp_head_start_stage_name = _resolve_export("US_SIPP_HEAD_START_STAGE_NAME") + us_snap_discretionary_exemption_stage_name = _resolve_export( + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME" + ) + us_snap_state_take_up_stage = _resolve_export("US_SNAP_STATE_TAKE_UP_STAGE") + us_snap_take_up_stage_name = _resolve_export("US_SNAP_TAKE_UP_STAGE_NAME") + us_ssi_disability_criteria_stage_name = _resolve_export( + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME" + ) + us_ssi_take_up_stage_name = _resolve_export("US_SSI_TAKE_UP_STAGE_NAME") + us_voluntary_filing_stage_name = _resolve_export("US_VOLUNTARY_FILING_STAGE_NAME") + us_weeks_unemployed_stage_name = _resolve_export("US_WEEKS_UNEMPLOYED_STAGE_NAME") + us_wic_claim_stage_name = _resolve_export("US_WIC_CLAIM_STAGE_NAME") + us_workers_compensation_stage_name = _resolve_export( + "US_WORKERS_COMPENSATION_STAGE_NAME" + ) + with _EXPORT_LOCK: + us_donors: Mapping[str, DonorSpec] = { + "scf_wealth": DonorSpec( + survey="Fed SCF 2022 + Census SIPP 2023", + source="https://www.federalreserve.gov/econres/scfindex.htm", + notes=( + "SCF anchors signed net worth and one half of household liquid-asset " + "vectors; the immutable SIPP 2023 public-use donor supplies the " + "other half and restores low liquid-asset mass. Auto loans use the " + "full SCF separately." + ), + ), + us_ssi_disability_criteria_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Latent under-65 SSI disability/blindness criterion from the " + "pinned full 2023 public-use donor. ASEC and PUF-support people " + "are predicted separately, and only direct under-65 ASEC SSI " + "reporters receive the observed-reporter anchor." + ), + ), + us_sipp_head_start_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Direct December nursery/preschool federally sponsored-program " + "responses train a weighted Head Start take-up proxy for ages " + "3--5; strict reported structural negatives exclude hot-decked " + "answers and the prediction is shared by support clones." + ), + ), + us_ssi_take_up_stage_name: DonorSpec( + survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", + source=ssi_take_up_ssa_source_url, + notes=( + "Direct ASEC SSI_VAL reporters anchor person-level take-up; " + "eligible source-person identities are count-calibrated by age to " + "SSA December 2024 Federal-payment recipient counts and fanned to " + "both support clones." + ), + ), + "sipp_tips": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes="Tip income for tipped occupations.", + ), + "org_wages": DonorSpec( + survey="CPS ORG", + source=( + "https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/" + ), + notes=( + "Hourly-wage labor-market inputs. Donor load failures abort the " + "build — the silent zero-fallback this stage once had is " + "structurally impossible under StagePlan." + ), + ), + "meps_esi_premiums": DonorSpec( + survey="MEPS-IC", + source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", + notes="Employer-sponsored insurance premium parameters.", + ), + "aca_marketplace_inputs": DonorSpec( + survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", + source="https://www.cms.gov/marketplace/resources/data/public-use-files", + notes=( + "Marketplace take-up and selected-plan inputs: CPS reported " + "Marketplace coverage and premium reports anchor the records; " + "CMS OEP enrollment, APTC, and metal-level tables provide the " + "calibration targets." + ), + ), + "medicaid_take_up": DonorSpec( + survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", + source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", + notes=( + "Medicaid take-up by anchored count-calibration (contract " + "treatment count_calibrated, microcosm #331): CPS-reported " + "Medicaid coverage at interview anchors the flag; the fill is " + "calibrated to CMS December 2024 state enrollment snapshots. " + "Point-in-time semantics per #332; heals the #170 " + "enrollment==eligibility degeneracy." + ), + ), + us_snap_state_take_up_stage: DonorSpec( + survey=( + "Census CPS ASEC reported receipt + USDA FNS state " + "average-monthly household caseloads" + ), + source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", + notes=( + "SNAP take-up by anchored count-calibration (contract treatment " + "count_calibrated, microcosm #372): reported ASEC receipt anchors " + "the flag; the fill is calibrated per state to FNS FY2024 " + "average-monthly household counts among eligible non-anchored " + "units, replacing the national snap_take_up fill that bakes in " + "state-dependent CPS underreporting." + ), + ), + us_other_health_insurance_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " + "Marketplace, and Medicaid premiums after take-up stages; an " + "ASEC-trained weighted QRF replaces both premium leaves on the " + "PUF support half." + ), + ), + us_prior_year_income_stage_name: DonorSpec( + survey="CPS ASEC (prior year)", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Adjacent-year PERIDNUM join for measured prior-year earnings, " + "with Census allocation flags and sentinels enforced. A joint " + "eight-predictor weighted QRF replaces both earnings leaves on " + "the PUF support half; signed self-employment losses survive." + ), + ), + us_immigration_stage_name: DonorSpec( + survey="CPS ASEC + published unauthorized-population estimates", + source=( + "https://www.pewresearch.org/short-reads/2024/07/22/" + "what-we-know-about-unauthorized-immigrants-living-in-the-us/" + ), + notes=( + "SSN card type and immigration status from ASEC citizenship, " + "entry-year, nativity, and program-participation fields via the " + "ASEC-UA residual method (SSRN 4662801), targeted to published " + "undocumented population/worker/student control totals." + ), + ), + us_hours_worked_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Hours-worked inputs mapped directly from measured ASEC person " + "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " + "them the engine defaults every person to 40 weekly hours and " + "hours-conditioned rules (SNAP work requirements) become no-ops." + ), + ), + us_snap_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC + USDA FNS participation rate estimates", + source="https://www.fns.usda.gov/snap/participation-rates", + notes=( + "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " + "up; non-reporting units drawn to the cited FNS participation " + "rate. Without it the engine defaults every eligible unit to " + "100% take-up." + ), + ), + us_eligibility_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "SNAP eligibility/exemption inputs mapped directly from " + "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " + "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " + "them disability, student, parent/child, and veteran " + "exemption channels default to False/0." + ), + ), + us_relationship_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured household-head and marital-status flags mapped exactly " + "from P_SEQ and A_MARITL; nothing is imputed." + ), + ), + us_medicare_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured Medicare enrollment mapped exactly from MCARE == 1 and " + "copied onto the PUF support clone; no take-up rate is applied." + ), + ), + us_retirement_distribution_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " + "archived CPS-only QRF replaces the four populated non-IRA leaves " + "on PUF support while preserving IRA channel ownership. Nothing " + "is allocated across accounts." + ), + ), + us_pregnancy_stage_name: DonorSpec( + survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", + source="https://www.cdc.gov/nchs/nvss/births.htm", + notes=( + "Pregnancy seeded among women 15-44 at the national " + "point-in-time rate (births x 39/52 over female 15-44 " + "population), matching the retired pipeline's national " + "fallback; state-level rates are follow-up work (#351). " + "The ASEC does not measure pregnancy." + ), + ), + us_wic_claim_stage_name: DonorSpec( + survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", + source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", + notes=( + "Stable person-level claim draws use official CY2022 FNS category " + "coverage rates after the pregnancy and parent-input stages. The " + "all-postpartum rate is used because no hermetic source identifies " + "breastfeeding; nutritional risk remains separately excluded." + ), + ), + us_snap_discretionary_exemption_stage_name: DonorSpec( + survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", + source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", + notes=( + "ABAWD discretionary exemptions seeded at the statutory cap " + "(8% from FY2024) across potentially covered adults 18-64; " + "the engine intersects with modeled coverage. Assumes full " + "state usage of the cap (#323)." + ), + ), + us_childcare_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_CHILDCAREXPNS is validated and carried " + "to the SPM-unit childcare leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_adult_care_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " + "qualifying flag is the measured PEDISDRS self-care difficulty " + "item; the expense leg is a seeded, weight-targeted draw from " + "the measured ASEC childcare-expense distribution (the same " + "section 21 expense class), restricted to tax units where the " + "statute can bind, with the 21(d)(2) spouse deeming honored. " + "Neither ASEC nor the SIPP 2023 PUF releases an in-household " + "adult-care expenditure amount, so the level proxy is declared " + "in the stage manifest." + ), + ), + us_energy_subsidy_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_ENGVAL is validated and carried to the " + "SPM-unit energy-subsidy leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_child_support_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " + "are carried directly. After PUF tax-detail imputation, one " + "ASEC-trained weighted QRF jointly replaces both person leaves " + "on the PUF support half using the archived predictor subset." + ), + ), + us_disability_benefits_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " + "when their source code is not workers' compensation. After PUF " + "tax-detail imputation, an ASEC-trained weighted QRF replaces the " + "CPS-only person leaf on the PUF support half." + ), + ), + us_workers_compensation_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual WC_VAL is carried directly. After PUF tax-detail " + "imputation, an ASEC-trained weighted QRF replaces the CPS-only " + "person leaf on the PUF support half." + ), + ), + us_weeks_unemployed_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured LKWEEKS is carried directly, including an exact " + "identity-keyed repair of the omitted 2022-income-year column " + "from the pinned official 2023 ASEC archive. An ASEC-trained " + "QRF then replaces only the PUF support half, with the archived " + "unemployment-compensation zero rule." + ), + ), + "puf_tax_detail": DonorSpec( + survey="IRS PUF 2015 (uprated)", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Itemized-deduction detail, versioned processed-PUF Section 199A " + "simulation leaves (carried without redrawing), partnership SE, " + "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " + "miscellaneous-expense proxy. The pinned processed PUF uprates its " + "raw TY2015 rows before seeded disclosure-record replacement and " + "includes a Forbes-backed 3,900-record open tail; this runtime " + "reconstructs the bounded-record AGI lineage and anchors every " + "Forbes-tail record in the final published AGI band without rerunning " + "Forbes synthesis. Support is clipped to the PUF's realized ranges." + ), + ), + us_education_inputs_stage_name: DonorSpec( + survey="IRS PUF 2015 (uprated) + Census CPS ASEC", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Qualified tuition comes from the PUF E03230/E87530 maximum; " + "the published retired path drops the reported AOTC output and its " + "five affirmative factual inputs therefore follow positive tuition; " + "educational assistance carries directly from ASEC ED_VAL." + ), + ), + us_retirement_contribution_stage_name: DonorSpec( + survey="Census CPS ASEC + published retirement-contribution shares", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured ASEC RETCB_VAL is allocated across five desired " + "retirement-contribution leaves using archived IRS/BEA/Vanguard/" + "PSCA shares, then CPS-trained QRF predictions replace the PUF " + "support half. PolicyEngine-US owns all statutory caps." + ), + ), + "capital_gain_distributions": DonorSpec( + survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", + source=( + "https://www.irs.gov/statistics/" + "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" + ), + notes=( + "Schedule D line 13 capital gain distributions split out of " + "long-term gains as a memo component at the national SOCA-derived " + "share; the direct-1040 route is already a PUF-stage output and " + "the two routes are mutually exclusive on a real return." + ), + ), + us_housing_inputs_stage_name: DonorSpec( + survey="Census ACS 2022", + source="https://www.census.gov/programs-surveys/acs", + notes=( + "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " + "rent imputation on household heads; reported housing assistance " + "alone receives the archived second-stage PUF-support QRF." + ), + ), + "vehicle_assets": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Household vehicle count and value from the pinned full 2023 " + "public-use donor. They remain independent policy inputs until " + "the full mixed-source net-worth reconciliation is restored." + ), + ), + us_voluntary_filing_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Measured 2023 SIPP filing and expected-filing responses replace " + "the retired uncited demographic probability table. Reciprocal " + "spouses form one source unit, reported dependents are excluded, " + "and one weighted-QRF prediction is shared across support clones." + ), + ), + } + us_stage_names: tuple[str, ...] = ( + "asec_load", + "unit_assignment", + "derive_cps_carried", + us_prior_year_income_stage_name, + us_immigration_stage_name, + us_hours_worked_stage_name, + us_snap_take_up_stage_name, + us_relationship_inputs_stage_name, + us_medicare_take_up_stage_name, + us_housing_inputs_stage_name, + us_retirement_distribution_stage_name, + us_eligibility_inputs_stage_name, + us_pregnancy_stage_name, + us_wic_claim_stage_name, + us_snap_discretionary_exemption_stage_name, + us_retirement_contribution_stage_name, + us_childcare_stage_name, + us_adult_care_stage_name, + us_energy_subsidy_stage_name, + us_puf_support_stage_name, + "puf_tax_detail", + us_child_support_stage_name, + us_disability_benefits_stage_name, + us_workers_compensation_stage_name, + us_weeks_unemployed_stage_name, + us_education_inputs_stage_name, + "capital_gain_distributions", + "scf_wealth", + us_ssi_disability_criteria_stage_name, + us_sipp_head_start_stage_name, + us_ssi_take_up_stage_name, + "sipp_tips", + "org_wages", + "meps_esi_premiums", + "mortgage_conversion", + "vehicle_assets", + us_voluntary_filing_stage_name, + "entity_placement", + "aca_marketplace_inputs", + "medicaid_take_up", + us_snap_state_take_up_stage, + us_other_health_insurance_stage_name, + "export", + ) + globals().setdefault("US_DONORS", us_donors) + globals().setdefault("US_STAGE_NAMES", us_stage_names) def _load_us_source_manifest() -> SourceManifest: @@ -2481,15 +2625,6 @@ def _load_us_support_spine_manifest() -> SupportSpineManifest: ) -US_SOURCE_MANIFEST = _load_us_source_manifest() -US_SUPPORT_SPINE_MANIFEST = _load_us_support_spine_manifest() -US_SUPPORT_SPINE_SPEC: SupportSpineSpec = US_SUPPORT_SPINE_MANIFEST.support_spine -US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] = US_SOURCE_MANIFEST.stages -US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] = frozenset( - output for stage in US_SOURCE_STAGE_SPECS for output in stage.nonnegative_outputs -) - - def us_plan( implementations: Mapping[str, Callable[[Frame], Frame]], ) -> StagePlan: @@ -2510,23 +2645,26 @@ def us_plan( ValueError: If any declared stage lacks an implementation, or an implementation is supplied for an undeclared stage. """ - missing = [name for name in US_STAGE_NAMES if name not in implementations] + _initialize_plan() + stage_names = globals()["US_STAGE_NAMES"] + donors = globals()["US_DONORS"] + missing = [name for name in stage_names if name not in implementations] if missing: raise ValueError( f"us_plan needs an implementation for every declared stage; " f"missing {missing}. There are no stubs or fallbacks by design." ) - unknown = sorted(set(implementations) - set(US_STAGE_NAMES)) + unknown = sorted(set(implementations) - set(stage_names)) if unknown: raise ValueError( f"Unknown stage implementation(s) {unknown}; declared stages " - f"are {list(US_STAGE_NAMES)}." + f"are {list(stage_names)}." ) return StagePlan( Stage( name=name, transform=implementations[name], - donor=US_DONORS.get(name), + donor=donors.get(name), ) - for name in US_STAGE_NAMES + for name in stage_names ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py index bcbe95cdf..815081c95 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py @@ -4,25 +4,17 @@ Microdata Area — the finest geography the ACS PUMS publishes. A household that already knows its PUMA (an ACS-spine record) keeps it; a household that knows only its state (an ASEC-spine record) is assigned a PUMA within that state, -sampled proportional to 2020 PUMA population. Every coarser layer of the -ladder then derives from the PUMA by a population-weighted draw over the -PUMA's overlap with that layer: - -- **congressional district (119th)** — sampled within the PUMA proportional to - the ``(PUMA, CD)`` block-population overlap (a 2020 PUMA can span several - districts and a district several PUMAs; they do not nest). -- **county** — sampled within the PUMA proportional to the ``(PUMA, county)`` - block-population overlap. -- **tract** (behind ``assign_tract``) — sampled within the PUMA proportional to - the ``(PUMA, tract)`` block-population overlap; when tract is assigned the - county derives structurally from it (``county = tract // 10**6``) so the two - never disagree. Congressional district, county and state are the launch - requirement; tract is the stretch rung. - -One national dataset, filter by geography, at any grain (microcosm #275; no -per-area files, the standing rule): the dense ACS-spine artifact is filtered to -state / congressional district / county for local analysis because every record -carries them. +sampled proportional to 2020 PUMA population. The derived geography is one +population-weighted draw from the actual ``(PUMA, tract, 119th CD)`` block +join. County is the selected tract's prefix, so county and CD always have +joint source support. The retained marginal tables validate this joint +population; assignment never multiplies their probabilities. + +The internal tract identifies a supported crosswalk cell, not an observed +residence. ``assign_tract`` controls whether that assigned tract is exported; +it does not change the PUMA, county or CD draw. National/CD analysis still +requires its separate calibration, vintage and coverage gates. An assigned +county or tract is not certification for analysis at that grain. Vintage discipline follows the country-spec geography-spine schema (``vintage_policy: "error"``): the artifact records one vintage per derived @@ -45,6 +37,7 @@ import json from collections.abc import Mapping from dataclasses import dataclass +from io import BytesIO from pathlib import Path from typing import Any @@ -59,6 +52,7 @@ from microcosm.build.us_runtime.puma_ladder_sources import ( COUNTY_FROM_TRACT_DIVISOR, PUMA_GEOID_STATE_DIVISOR, + US_PUMA_LADDER_SCHEMA_VERSION, ) from microcosm.frame import Frame @@ -81,7 +75,6 @@ #: The finer tract rung, written only when ``assign_tract=True``. US_PUMA_LADDER_TRACT_COLUMN = "tract_geoid" -US_PUMA_LADDER_SCHEMA_VERSION = 1 US_PUMA_LADDER_KIND = "us_puma_ladder" PUMA_LADDER_ARTIFACT_SHA256_ATTR = "populace_puma_ladder_artifact_sha256" @@ -108,14 +101,18 @@ _REQUIRED_ARRAY_KEYS = ( *_ANCHOR_KEYS, *(key for keys in _OVERLAP_KEYS.values() for key in keys), + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", ) @dataclass(frozen=True) class UsPumaLadder: """The national PUMA ladder: one anchor row per populated 2020 PUMA plus - three ``(PUMA, layer_value, population)`` overlap tables sorted by - ``(puma, layer_value)``. + three marginal overlap tables and the actual joint ``(PUMA, tract, CD, + population)`` table. Joint rows are unique and sorted by ``(puma, tract, CD)``. Attributes: puma: 2020 PUMA geoids (``state_fips * 10**5 + PUMA5CE``) as ``int64``, @@ -134,7 +131,10 @@ class UsPumaLadder: the ``(PUMA, tract)`` population overlap; tract geoids are the 11-digit ``state+county+tract`` integer. metadata: the artifact's embedded metadata, including one vintage per - derived layer. + derived layer. Schema v2 requires the joint source table. + joint_overlap_puma / joint_overlap_tract / joint_overlap_cd / + joint_overlap_population: the supported populated-block join, + aggregated to unique PUMA/tract/CD cells; county is a tract prefix. """ puma: np.ndarray @@ -149,6 +149,10 @@ class UsPumaLadder: tract_overlap_tract: np.ndarray tract_overlap_population: np.ndarray metadata: Mapping[str, Any] + joint_overlap_puma: np.ndarray + joint_overlap_tract: np.ndarray + joint_overlap_cd: np.ndarray + joint_overlap_population: np.ndarray def __len__(self) -> int: return len(self.puma) @@ -186,6 +190,17 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: source = Path(path) if not source.exists(): raise FileNotFoundError(f"US PUMA ladder artifact not found: {source}") + return _decode_us_puma_ladder_archive(source) + + +def decode_us_puma_ladder(payload: bytes) -> UsPumaLadder: + """Validate the same NPZ contract from immutable graph artifact bytes.""" + if not isinstance(payload, bytes): + raise TypeError("US PUMA ladder payload must be immutable bytes.") + return _decode_us_puma_ladder_archive(BytesIO(payload)) + + +def _decode_us_puma_ladder_archive(source: Path | BytesIO) -> UsPumaLadder: with np.load(source, allow_pickle=False) as payload: missing = [key for key in _REQUIRED_ARRAY_KEYS if key not in payload.files] if missing: @@ -274,7 +289,7 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: raise ValueError("tract_overlap_tract geoids must be unique (tracts nest).") _assert_conservation(puma, population, tract_puma, tract_pop, layer="tract") - return UsPumaLadder( + ladder = UsPumaLadder( puma=puma, puma_population=population, cd_overlap_puma=cd_puma, @@ -287,7 +302,13 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: tract_overlap_tract=tract_value, tract_overlap_population=tract_pop, metadata=metadata, + joint_overlap_puma=arrays["joint_overlap_puma"], + joint_overlap_tract=arrays["joint_overlap_tract"], + joint_overlap_cd=arrays["joint_overlap_cd"], + joint_overlap_population=arrays["joint_overlap_population"], ) + _validated_joint_support(ladder) + return ladder def assign_us_puma_ladder( @@ -304,10 +325,11 @@ def assign_us_puma_ladder( Records that already carry a valid ``puma`` (the ACS spine) keep it; records that carry none (the ASEC spine) draw a PUMA within their ``state_fips`` - proportional to 2020 PUMA population. Every record then draws a congressional - district and a county within its PUMA proportional to the block-population - overlap, and — when ``assign_tract`` — a tract from which the county derives - structurally. Draws are consumed from one seeded generator in a fixed sorted + proportional to 2020 PUMA population. Every record then draws one supported + joint tract/CD cell within its PUMA proportional to block population. County + derives from that tract. The tract is written only when ``assign_tract`` is + true; changing that flag leaves the same PUMA/county/CD assignments. + Draws are consumed from one seeded generator in a fixed sorted order (states, then PUMAs), so the result is reproducible from ``seed``. A household ``puma`` absent from the ladder, or a household ``state_fips`` @@ -319,6 +341,9 @@ def assign_us_puma_ladder( f"household table must contain {state_fips_column!r} before PUMA-" "ladder assignment." ) + joint_puma, joint_tract, joint_cd, joint_population = _validated_joint_support( + ladder + ) if expected_congressional_district_vintage is not None: ladder_cd_vintage = ladder.layer_vintages["congressional_district"] if ladder_cd_vintage != expected_congressional_district_vintage: @@ -343,23 +368,17 @@ def assign_us_puma_ladder( if unknown_mask.any(): _draw_puma_within_state(row_puma, unknown_mask, state_ints, ladder, rng) - cd_puma, cd_value, cd_pop = ladder._overlap("congressional_district") - cd_values = _draw_layer_values( - row_puma, cd_puma, cd_value, cd_pop, rng, layer="congressional_district" + joint_rows = _draw_layer_values( + row_puma, + joint_puma, + np.arange(len(joint_puma), dtype=np.int64), + joint_population, + rng, + layer="joint tract/congressional_district", ) - - tract_values: np.ndarray | None = None - if assign_tract: - tract_puma, tract_value, tract_pop = ladder._overlap("tract") - tract_values = _draw_layer_values( - row_puma, tract_puma, tract_value, tract_pop, rng, layer="tract" - ) - county_values = tract_values // COUNTY_FROM_TRACT_DIVISOR - else: - county_puma, county_value, county_pop = ladder._overlap("county") - county_values = _draw_layer_values( - row_puma, county_puma, county_value, county_pop, rng, layer="county" - ) + cd_values = joint_cd[joint_rows] + county_values = joint_tract[joint_rows] // COUNTY_FROM_TRACT_DIVISOR + tract_values = joint_tract[joint_rows] if assign_tract else None _assert_row_state_consistency( state_ints, @@ -383,6 +402,213 @@ def assign_us_puma_ladder( return assigned +def _validated_joint_support( + ladder: UsPumaLadder, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Require real joint support and exact agreement with every marginal. + + A frozen dataclass does not make its arrays immutable. Check the current + arrays when assigning or verifying, including normally constructed ladders. + Nothing in this function manufactures joint rows from marginal weights. + """ + + _validate_ladder_metadata(ladder.metadata) + if ladder.metadata.get("puma_vintage") != "2020_puma": + raise ValueError( + "Joint PUMA assignment requires the observed 2020 PUMA vintage." + ) + if any( + ladder.layer_vintages[layer] != "2020_census" for layer in ("county", "tract") + ): + raise ValueError( + "Joint PUMA assignment requires 2020 Census county/tract vintages." + ) + anchor = _int64_array(ladder.puma, label="puma") + if ( + not len(anchor) + or (np.diff(anchor) <= 0).any() + or ( + (anchor // PUMA_GEOID_STATE_DIVISOR < 1) + | (anchor // PUMA_GEOID_STATE_DIVISOR > 56) + ).any() + ): + raise ValueError( + "Joint PUMA anchors must be unique, sorted and within states 01–56." + ) + columns = tuple( + _int64_array(getattr(ladder, name), label=name) + for name in ( + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", + ) + ) + puma, tract, cd, population = columns + if not len(puma) or any(len(value) != len(puma) for value in columns): + raise ValueError("Joint PUMA overlap arrays must be nonempty and aligned.") + if (population <= 0).any(): + raise ValueError( + "Joint PUMA overlap populations must be positive integer counts." + ) + rows = list(zip(puma.tolist(), tract.tolist(), cd.tolist(), strict=True)) + if rows != sorted(set(rows)): + raise ValueError( + "Joint PUMA overlap rows must be unique and sorted by PUMA/tract/CD." + ) + tract_owner: dict[int, int] = {} + for row_puma, row_tract, _ in rows: + if tract_owner.setdefault(row_tract, row_puma) != row_puma: + raise ValueError("Joint PUMA overlap assigns one tract to multiple PUMAs.") + if ((tract < 10**9) | (tract >= 10**11)).any() or ((cd < 100) | (cd > 9999)).any(): + raise ValueError("Joint PUMA overlap contains an invalid tract or CD geoid.") + _assert_layer_state_matches( + puma, tract // COUNTY_FROM_TRACT_DIVISOR // 1000, layer="joint tract" + ) + _assert_layer_state_matches(puma, cd // 100, layer="joint CD") + + def counts(keys: list[tuple[int, ...]], values: np.ndarray, label: str) -> dict: + values = np.asarray(values) + if values.ndim != 1 or len(values) != len(keys): + raise ValueError(f"{label} population must align with its keys.") + result: dict[tuple[int, ...], int] = {} + for key, value in zip(keys, values.tolist(), strict=True): + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + or value <= 0 + or int(value) != value + ): + raise ValueError( + f"{label} population must contain positive integer counts." + ) + result[key] = result.get(key, 0) + int(value) + return result + + marginals = { + "anchor": [(int(key),) for key in puma], + "congressional_district": [ + (int(key), int(value)) for key, value in zip(puma, cd, strict=True) + ], + "county": [ + (int(key), int(value)) + for key, value in zip(puma, tract // COUNTY_FROM_TRACT_DIVISOR, strict=True) + ], + "tract": [ + (int(key), int(value)) for key, value in zip(puma, tract, strict=True) + ], + } + for layer, keys in marginals.items(): + if layer == "anchor": + reference_keys = [(int(key),) for key in anchor] + reference_pop = ladder.puma_population + else: + source_puma, source_value, reference_pop = ladder._overlap(layer) + source_puma = _int64_array(source_puma, label=layer + "_overlap_puma") + source_value = _int64_array(source_value, label=layer + "_overlap_value") + if len(source_puma) != len(source_value): + raise ValueError(f"{layer} overlap keys must align.") + reference_keys = list( + zip(source_puma.tolist(), source_value.tolist(), strict=True) + ) + if counts(keys, population, "joint") != counts( + reference_keys, reference_pop, layer + ): + raise ValueError( + f"Joint PUMA overlap does not exactly reproduce the {layer} marginal." + ) + return columns + + +def us_puma_ladder_joint_support_gate( + household: pd.DataFrame, + ladder: UsPumaLadder, + *, + assign_tract: bool = False, + expected_congressional_district_vintage: str | None = None, +) -> GateResult: + """Check current assigned rows against actual joint Census support. + + This structural support gate is independent of the national NYC mass gate + and of national/CD calibration; zero-weight clone rows must also be valid. + """ + + failures: list[str] = [] + details: dict[str, object] = {"household_rows": len(household)} + try: + puma, tract, cd, _ = _validated_joint_support(ladder) + if ( + expected_congressional_district_vintage is not None + and ladder.layer_vintages["congressional_district"] + != expected_congressional_district_vintage + ): + raise ValueError( + "Joint-support congressional-district vintage does not match the expected vintage." + ) + columns = ["puma", "county_fips", CONGRESSIONAL_DISTRICT_GEOID_COLUMN] + if assign_tract: + columns.append(US_PUMA_LADDER_TRACT_COLUMN) + missing = [name for name in columns if name not in household] + if missing: + raise ValueError(f"Joint support requires geography columns: {missing}.") + values = [] + for name in columns: + numeric = pd.to_numeric(household[name], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + if not ( + np.isfinite(numeric) + & (numeric > 0) + & (numeric < 2**53) + & (numeric == np.floor(numeric)) + ).all(): + raise ValueError( + f"Joint support requires positive integral {name} geoids." + ) + values.append(numeric.astype(np.int64)) + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + strict=True, + ) + ) + if assign_tract: + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + tract.tolist(), + strict=True, + ) + ) + unsupported = [ + index + for index, row in enumerate( + zip(*(value.tolist() for value in values), strict=True) + ) + if row not in support + ] + details["unsupported_rows"] = len(unsupported) + if unsupported: + failures.append( + f"{len(unsupported)} household row(s) have no joint PUMA/county/CD support; row positions {unsupported[:5]}." + ) + details["joint_support_rows"] = len(puma) + details["layer_vintages"] = ladder.layer_vintages + except (TypeError, ValueError) as error: + failures.append(str(error)) + return GateResult( + name="us_puma_ladder_joint_support", + passed=not failures, + failures=tuple(failures), + details=details, + ) + + def with_household_us_puma_ladder( frame: Frame, ladder: UsPumaLadder, @@ -850,8 +1076,12 @@ def _assert_conservation( def _int64_array(values: np.ndarray, *, label: str) -> np.ndarray: array = np.asarray(values) + if array.ndim != 1: + raise ValueError(f"{label} must be a one-dimensional integer array.") if array.dtype.kind not in ("i", "u"): raise ValueError(f"{label} must be an integer array, got dtype {array.dtype}.") + if array.dtype.kind == "u" and (array > np.iinfo(np.int64).max).any(): + raise ValueError(f"{label} contains an integer outside signed int64.") return array.astype(np.int64, copy=False) @@ -898,5 +1128,6 @@ def _household_cd_state(values: Any) -> np.ndarray: "load_us_puma_ladder", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py index bbf252e6b..1982f0b04 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py @@ -17,8 +17,10 @@ - ``block -> 119th CD`` and ``block -> POP100`` reuse the block-ladder parsers :func:`parse_national_cd_bef` and :func:`parse_pl_geo_blocks` unchanged. -Summing block populations by ``(PUMA, CD)``, ``(PUMA, county)`` and -``(PUMA, tract)`` yields the three overlap tables; summing by PUMA yields the +Summing block populations by ``(PUMA, tract, CD)`` preserves the joint support +needed to draw a consistent county/CD pair (county is the tract prefix). The +three marginal overlap tables are retained for validation and diagnostics; +summing by PUMA yields the anchor population used for the ASEC state -> PUMA draw. Every populated block contributes to exactly one PUMA, one CD, one county and one tract, so each overlap table conserves its PUMA's population exactly — a defect (a populated @@ -45,6 +47,7 @@ TRACT_FROM_BLOCK_DIVISOR = 10**4 COUNTY_FROM_BLOCK_DIVISOR = 10**10 COUNTY_FROM_TRACT_DIVISOR = 10**6 +US_PUMA_LADDER_SCHEMA_VERSION = 2 _TRACT_TO_PUMA_HEADER = ("STATEFP", "COUNTYFP", "TRACTCE", "PUMA5CE") @@ -120,16 +123,21 @@ def assemble_us_puma_ladder( For every populated block the block's tract (a structural prefix) must map to a PUMA and the block must carry a congressional district; either gap is - a source defect, not a skippable row, so the three overlap tables each - conserve their PUMA's population exactly. Returns arrays for the anchor - (``puma`` / ``puma_population``) and the three overlap tables, each sorted - by ``(puma, layer_value)`` for a stable, searchsorted-friendly artifact. + a source defect, not a skippable row. Schema v2 retains the actual joint + PUMA/tract/CD population before aggregation is lost; county is the tract + prefix. It also emits the anchor and three marginal tables for exact + conservation checks. Joint rows are sorted by ``(puma, tract, CD)`` and + marginal rows by ``(puma, layer_value)``. """ + if metadata.get("schema_version") != US_PUMA_LADDER_SCHEMA_VERSION: + raise ValueError("Joint PUMA source assembly requires schema_version 2.") + puma_population: dict[int, int] = {} puma_cd_population: dict[tuple[int, int], int] = {} puma_county_population: dict[tuple[int, int], int] = {} puma_tract_population: dict[tuple[int, int], int] = {} + joint_population: dict[tuple[int, int, int], int] = {} missing_puma: list[int] = [] missing_cd: list[int] = [] @@ -157,6 +165,8 @@ def assemble_us_puma_ladder( puma_tract_population[tract_key] = ( puma_tract_population.get(tract_key, 0) + population ) + joint_key = (puma, tract, cd) + joint_population[joint_key] = joint_population.get(joint_key, 0) + population if missing_puma: examples = [f"{block:015d}" for block in sorted(missing_puma)[:5]] @@ -180,6 +190,7 @@ def assemble_us_puma_ladder( cd_puma, cd_value, cd_pop = _overlap_arrays(puma_cd_population) county_puma, county_value, county_pop = _overlap_arrays(puma_county_population) tract_puma, tract_value, tract_pop = _overlap_arrays(puma_tract_population) + joint_keys = sorted(joint_population) _assert_conserves( pumas, anchor_population, cd_puma, cd_pop, layer="congressional_district" @@ -199,6 +210,16 @@ def assemble_us_puma_ladder( "tract_overlap_puma": tract_puma, "tract_overlap_tract": tract_value, "tract_overlap_population": tract_pop, + "joint_overlap_puma": np.asarray( + [key[0] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_tract": np.asarray( + [key[1] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_cd": np.asarray([key[2] for key in joint_keys], dtype=np.int64), + "joint_overlap_population": np.asarray( + [joint_population[key] for key in joint_keys], dtype=np.int64 + ), "metadata_json": np.asarray(json.dumps(dict(metadata), sort_keys=True)), } diff --git a/packages/microcosm-build/tests/test_us_acs_multispine.py b/packages/microcosm-build/tests/test_us_acs_multispine.py index afc38b72d..fbaa393b7 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine.py @@ -356,15 +356,25 @@ def _test_puma_ladder() -> UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601, 3_601], dtype=np.int64), + joint_overlap_population=np.asarray([100, 200], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py index 31742ad61..47e423dcb 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py @@ -193,15 +193,25 @@ def test_main_wires_verified_sources_transfer_audit_export_and_summary( tract_overlap_puma=np.asarray([100_100], dtype=np.int64), tract_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), tract_overlap_population=np.asarray([100.0]), + joint_overlap_puma=np.asarray([100_100], dtype=np.int64), + joint_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_base_pool.py b/packages/microcosm-build/tests/test_us_base_pool.py index e7a543539..814a37e30 100644 --- a/packages/microcosm-build/tests/test_us_base_pool.py +++ b/packages/microcosm-build/tests/test_us_base_pool.py @@ -180,15 +180,25 @@ def _test_puma_ladder() -> UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101, 102], dtype=np.int64), + joint_overlap_population=np.asarray([40, 60], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 91811e9e8..cc3fe91e5 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -320,15 +320,25 @@ def _fixture_puma_ladder(pool_tool: ModuleType): tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_puma_ladder.py b/packages/microcosm-build/tests/test_us_puma_ladder.py index 8d73a0a24..ab009af7a 100644 --- a/packages/microcosm-build/tests/test_us_puma_ladder.py +++ b/packages/microcosm-build/tests/test_us_puma_ladder.py @@ -1,6 +1,7 @@ """US PUMA-anchored geography-ladder tests.""" import json +from dataclasses import replace from pathlib import Path import numpy as np @@ -13,6 +14,7 @@ load_us_puma_ladder, us_puma_ladder_assignment_summary, us_puma_ladder_gate, + us_puma_ladder_joint_support_gate, with_household_us_puma_ladder, ) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -34,7 +36,7 @@ def _ladder_metadata(**overrides: object) -> dict: }, } metadata = { - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", @@ -66,6 +68,14 @@ def _write_ladder(path: Path, **overrides: object) -> Path: [1001000100, 1003000100, 1003000200, 2013000100], dtype=np.int64 ), "tract_overlap_population": np.asarray([900.0, 100.0, 500.0, 400.0]), + "joint_overlap_puma": np.asarray( + [100100, 100100, 100200, 200100], dtype=np.int64 + ), + "joint_overlap_tract": np.asarray( + [1001000100, 1003000100, 1003000200, 2013000100], dtype=np.int64 + ), + "joint_overlap_cd": np.asarray([101, 102, 102, 200], dtype=np.int64), + "joint_overlap_population": np.asarray([900, 100, 500, 400], dtype=np.int64), "metadata_json": np.asarray(json.dumps(_ladder_metadata())), } arrays.update({k: np.asarray(v) for k, v in overrides.items()}) @@ -465,3 +475,168 @@ def _minimal_us_frame() -> Frame: } strata = pd.Series(["acs_2023", "acs_2023"], name="stratum") return Frame(tables, US_SCHEMA, weights, strata) + + +def test_county_cd_pairs_have_actual_joint_support(tmp_path) -> None: + ladder = _ladder(tmp_path) + household = pd.DataFrame({"state_fips": [1] * 1000, "puma": [100100] * 1000}) + assigned = assign_us_puma_ladder(household, ladder, seed=578) + pairs = set( + zip( + assigned["county_fips"], + assigned["congressional_district_geoid"], + strict=True, + ) + ) + assert pairs == {("01001", 101), ("01003", 102)} + assert us_puma_ladder_joint_support_gate(assigned, ladder).passed + # Both marginals and the state prefix are valid; their pairing is not. + assigned.loc[0, ["county_fips", "congressional_district_geoid"]] = ["01001", 102] + result = us_puma_ladder_joint_support_gate(assigned, ladder) + assert not result.passed + assert result.details["unsupported_rows"] == 1 + assert any( + "no joint PUMA/county/CD support" in reason for reason in result.failures + ) + + +def test_tract_flag_preserves_the_complete_coarse_assignment(tmp_path) -> None: + ladder = _ladder(tmp_path) + household = pd.DataFrame( + {"state_fips": [1, 1, 2] * 20, "puma": [100100, np.nan, np.nan] * 20} + ) + before = household.copy(deep=True) + coarse = assign_us_puma_ladder(household, ladder, seed=20260909) + fine = assign_us_puma_ladder(household, ladder, seed=20260909, assign_tract=True) + pd.testing.assert_frame_equal(coarse, fine.drop(columns=["tract_geoid"])) + pd.testing.assert_frame_equal(household, before) + assert coarse.loc[household["puma"].notna(), "puma"].eq("0100100").all() + assert us_puma_ladder_joint_support_gate(fine, ladder, assign_tract=True).passed + fine.loc[0, "tract_geoid"] = "01003000200" + assert not us_puma_ladder_joint_support_gate(fine, ladder, assign_tract=True).passed + + +def test_old_marginal_only_artifact_requires_rebuild(tmp_path) -> None: + path = _write_ladder(tmp_path / "v1.npz") + with np.load(path, allow_pickle=False) as payload: + arrays = { + name: payload[name] + for name in payload.files + if not name.startswith("joint_") + } + arrays["metadata_json"] = np.asarray(json.dumps(_ladder_metadata(schema_version=1))) + np.savez_compressed(path, **arrays) + with pytest.raises(ValueError, match="missing required key"): + load_us_puma_ladder(path) + + +@pytest.mark.parametrize( + "overrides, reason", + [ + ( + {"joint_overlap_population": np.asarray([899, 100, 500, 400])}, + "exactly reproduce the anchor", + ), + ( + {"joint_overlap_cd": np.asarray([102, 101, 102, 200])}, + "exactly reproduce the congressional_district", + ), + ( + { + "joint_overlap_tract": np.asarray( + [1003000100, 1003000100, 1003000200, 2013000100] + ) + }, + "exactly reproduce the county", + ), + ( + {"joint_overlap_population": np.asarray([900.0, 100.0, 500.0, 400.0])}, + "integer array", + ), + ( + {"joint_overlap_population": np.asarray([900, 0, 500, 400])}, + "positive integer", + ), + ( + {"joint_overlap_population": np.asarray([[900, 100, 500, 400]])}, + "one-dimensional", + ), + ( + {"joint_overlap_cd": np.asarray([201, 102, 102, 200])}, + "state prefix must match", + ), + ( + { + "joint_overlap_population": np.asarray( + [900, 100, 500, 2**63], dtype=np.uint64 + ) + }, + "outside signed int64", + ), + ( + {"joint_overlap_puma": np.asarray([100100, 100200, 100100, 200100])}, + "unique and sorted", + ), + ], +) +def test_load_rejects_invalid_joint_support(tmp_path, overrides, reason) -> None: + path = _write_ladder(tmp_path / "invalid-joint.npz", **overrides) + with pytest.raises(ValueError, match=reason): + load_us_puma_ladder(path) + + +def test_mutated_ladder_arrays_refuse_before_assignment(tmp_path) -> None: + ladder = _ladder(tmp_path) + ladder.joint_overlap_population[0] -= 1 + household = pd.DataFrame({"state_fips": [1], "puma": [100100]}) + with pytest.raises(ValueError, match="exactly reproduce the anchor"): + assign_us_puma_ladder(household, ladder) + assert not us_puma_ladder_joint_support_gate(household, ladder).passed + + +def test_joint_support_gate_rejects_vintage_and_nonintegral_geoids(tmp_path) -> None: + ladder = _ladder(tmp_path) + household = pd.DataFrame({"state_fips": [1], "puma": [100100]}) + assigned = assign_us_puma_ladder(household, ladder) + assert not us_puma_ladder_joint_support_gate( + assigned, ladder, expected_congressional_district_vintage="118th_congress" + ).passed + assigned["congressional_district_geoid"] = [101.5] + assert not us_puma_ladder_joint_support_gate(assigned, ladder).passed + + +def test_direct_ladder_duplicate_anchor_is_rejected(tmp_path) -> None: + ladder = _ladder(tmp_path) + # Aggregate counts still match, but duplicate anchors would bias the ASEC draw. + ladder = replace( + ladder, + puma=np.asarray([100100, 100100, 100200, 200100], dtype=np.int64), + puma_population=np.asarray([500.0, 500.0, 500.0, 400.0]), + ) + with pytest.raises(ValueError, match="anchors must be unique"): + assign_us_puma_ladder(pd.DataFrame({"state_fips": [1]}), ladder) + + +def test_live_joint_support_requires_tracts_to_nest_in_one_puma(tmp_path) -> None: + ladder = _ladder(tmp_path) + household = pd.DataFrame({"state_fips": [1], "puma": [100100]}) + assigned = assign_us_puma_ladder(household, ladder) + # Counts and every marginal remain exact, but this corrupts tract nesting. + ladder.joint_overlap_tract[2] = ladder.joint_overlap_tract[1] + ladder.tract_overlap_tract[2] = ladder.tract_overlap_tract[1] + with pytest.raises(ValueError, match="one tract to multiple PUMAs"): + assign_us_puma_ladder(household, ladder) + result = us_puma_ladder_joint_support_gate(assigned, ladder) + assert not result.passed + assert any("one tract to multiple PUMAs" in reason for reason in result.failures) + + +@pytest.mark.parametrize("layer", ["county", "tract"]) +def test_joint_assignment_rejects_wrong_census_vintage(tmp_path, layer) -> None: + metadata = _ladder_metadata() + metadata["layers"][layer]["vintage"] = "2010_census" + path = _write_ladder( + tmp_path / "puma_ladder.npz", metadata_json=np.asarray(json.dumps(metadata)) + ) + with pytest.raises(ValueError, match="2020 Census county/tract"): + load_us_puma_ladder(path) diff --git a/packages/microcosm-build/tests/test_us_puma_ladder_sources.py b/packages/microcosm-build/tests/test_us_puma_ladder_sources.py index c8ca89872..832f12199 100644 --- a/packages/microcosm-build/tests/test_us_puma_ladder_sources.py +++ b/packages/microcosm-build/tests/test_us_puma_ladder_sources.py @@ -1,6 +1,9 @@ """Tests for the pure PUMA-ladder source parsers and assembler.""" +import hashlib +import importlib.util import json +from pathlib import Path import numpy as np import pytest @@ -33,7 +36,7 @@ 2013000100: 200100, } _METADATA = { - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", @@ -189,3 +192,200 @@ def test_assemble_ignores_zero_population_blocks() -> None: # change any PUMA's conserved total. assert payload["puma_population"].tolist() == [1000, 500, 400] assert np.asarray(payload["cd_overlap_population"]).sum() == 1900 + + +def test_assembly_retains_joint_cells_without_crossing_marginals() -> None: + payload = assemble_us_puma_ladder( + block_population=_BLOCK_POPULATION, + cd_by_block=_CD_BY_BLOCK, + tract_to_puma=_TRACT_TO_PUMA, + metadata=_METADATA, + ) + assert list( + zip( + payload["joint_overlap_puma"].tolist(), + payload["joint_overlap_tract"].tolist(), + payload["joint_overlap_cd"].tolist(), + payload["joint_overlap_population"].tolist(), + strict=True, + ) + ) == [ + (100100, 1001000100, 101, 900), + (100100, 1003000100, 102, 100), + (100200, 1003000200, 102, 500), + (200100, 2013000100, 200, 400), + ] + + +def test_assembly_preserves_a_tract_split_between_districts() -> None: + block_population = {**_BLOCK_POPULATION, 10010001001001: 80} + cd_by_block = {**_CD_BY_BLOCK, 10010001001001: 102} + payload = assemble_us_puma_ladder( + block_population=block_population, + cd_by_block=cd_by_block, + tract_to_puma=_TRACT_TO_PUMA, + metadata=_METADATA, + ) + assert payload["joint_overlap_tract"][:2].tolist() == [1001000100, 1001000100] + assert payload["joint_overlap_cd"][:2].tolist() == [101, 102] + assert payload["joint_overlap_population"][:2].tolist() == [900, 80] + assert payload["puma_population"].tolist() == [1080, 500, 400] + + +def test_assembly_refuses_legacy_schema_label_for_joint_payload() -> None: + with pytest.raises(ValueError, match="requires schema_version 2"): + assemble_us_puma_ladder( + block_population=_BLOCK_POPULATION, + cd_by_block=_CD_BY_BLOCK, + tract_to_puma=_TRACT_TO_PUMA, + metadata={**_METADATA, "schema_version": 1}, + ) + + +@pytest.fixture(scope="module") +def ladder_tool(): + import microcosm.build + + # Resolve the actual source checkout even when ordinary tests are copied + # to isolated scratch. Wheel tests retain the repository-side tool fixture. + path = ( + Path(microcosm.build.__file__).resolve().parents[5] + / "tools/build_us_puma_ladder_artifact.py" + ) + if not path.is_file(): + path = ( + Path(__file__).resolve().parents[3] + / "tools/build_us_puma_ladder_artifact.py" + ) + spec = importlib.util.spec_from_file_location("puma_ladder_tool_fixture", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _invented_pinned_sources(tmp_path, tool): + import zipfile + + inputs = tmp_path / "inputs" + inputs.mkdir() + tract = inputs / "tract.txt" + tract.write_text("STATEFP,COUNTYFP,TRACTCE,PUMA5CE\n01,001,000100,00100\n") + cd = inputs / "cd119.zip" + with zipfile.ZipFile(cd, "w") as archive: + archive.writestr( + "NationalCD119.txt", + "GEOID,CDFP\n010010001001000,01\n010010001001001,02\n", + ) + pl = inputs / "al2020.pl.zip" + + def line(level, geocode, population): + fields = [""] * 97 + fields[2], fields[9], fields[90] = level, geocode, str(population) + return "|".join(fields) + "\n" + + with zipfile.ZipFile(pl, "w") as archive: + archive.writestr( + "algeo2020.pl", + line("040", "01", 100) + + line("750", "010010001001000", 90) + + line("750", "010010001001001", 10), + ) + locators = { + "tract_to_puma": (tract, tool.TRACT_TO_PUMA_URL), + "cd119_bef": (cd, tool.CD119_BEF_URL), + "pl94171_al": ( + pl, + tool.PL94171_URL_TEMPLATE.format(dirname="Alabama", usps_lower="al"), + ), + } + manifest = tmp_path / "sources.json" + manifest.write_text( + json.dumps( + { + "schema_version": 1, + "sources": { + name: { + "path": str(path), + "url": url, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "bytes": path.stat().st_size, + } + for name, (path, url) in locators.items() + }, + } + ) + ) + return manifest, tract + + +def _pinned_cli_args(tmp_path, manifest): + return [ + "--out", + str(tmp_path / "ladder.npz"), + "--states", + "01", + "--source-manifest", + str(manifest), + "--cache-dir", + str(tmp_path / "unused-cache"), + ] + + +def test_pinned_offline_tool_builds_actual_joint_npz(tmp_path, ladder_tool) -> None: + from microcosm.build.us_runtime import load_us_puma_ladder + + manifest, _ = _invented_pinned_sources(tmp_path, ladder_tool) + ladder_tool.main(_pinned_cli_args(tmp_path, manifest)) + ladder = load_us_puma_ladder(tmp_path / "ladder.npz") + # One observed source tract can cross two districts within the same PUMA. + assert ladder.joint_overlap_tract.tolist() == [1001000100, 1001000100] + assert ladder.joint_overlap_cd.tolist() == [101, 102] + assert ladder.joint_overlap_population.tolist() == [90, 10] + assert ( + ladder.metadata["source_manifest_sha256"] + == hashlib.sha256(manifest.read_bytes()).hexdigest() + ) + assert not (tmp_path / "unused-cache").exists() + + +@pytest.mark.parametrize("defect", ["sha256", "bytes", "url", "missing", "extra"]) +def test_pinned_offline_tool_refuses_bad_envelope( + tmp_path, ladder_tool, defect +) -> None: + manifest, _ = _invented_pinned_sources(tmp_path, ladder_tool) + document = json.loads(manifest.read_text()) + row = document["sources"]["tract_to_puma"] + if defect == "sha256": + row["sha256"] = "0" * 64 + elif defect == "bytes": + row["bytes"] += 1 + elif defect == "url": + row["url"] = "https://example.invalid/wrong-source" + elif defect == "missing": + del document["sources"]["pl94171_al"] + else: + document["sources"]["unselected_state"] = dict(row) + manifest.write_text(json.dumps(document)) + with pytest.raises(ValueError, match="[Pp]inned source"): + ladder_tool.main(_pinned_cli_args(tmp_path, manifest)) + assert not (tmp_path / "ladder.npz").exists() + assert not (tmp_path / "unused-cache").exists() + + +@pytest.mark.parametrize("mutation", ["payload", "manifest"]) +def test_pinned_offline_tool_rechecks_after_parsing( + tmp_path, ladder_tool, monkeypatch, mutation +) -> None: + manifest, tract = _invented_pinned_sources(tmp_path, ladder_tool) + original = ladder_tool._text_lines + + def mutate_after_read(path): + yield from original(path) + target = tract if mutation == "payload" else manifest + target.write_bytes(target.read_bytes() + b"\n") + + monkeypatch.setattr(ladder_tool, "_text_lines", mutate_after_read) + with pytest.raises(ValueError, match="Pinned source"): + ladder_tool.main(_pinned_cli_args(tmp_path, manifest)) + assert not (tmp_path / "ladder.npz").exists() diff --git a/tools/build_us_puma_ladder_artifact.py b/tools/build_us_puma_ladder_artifact.py index efe631d5c..5ade1e77a 100644 --- a/tools/build_us_puma_ladder_artifact.py +++ b/tools/build_us_puma_ladder_artifact.py @@ -4,7 +4,7 @@ Census Tract to 2020 PUMA relationship file, the 119th Congressional District block equivalency file, and the 2020 P.L. 94-171 geographic headers (block populations); joins them at 2020-tabulation-block grain and aggregates to the -PUMA anchor plus three ``(PUMA, layer_value, population)`` overlap tables; and +PUMA anchor, actual joint PUMA/tract/CD cells, and three marginal overlap tables; and writes one national NPZ artifact whose embedded metadata records a vintage and source per derived layer (``vintage_policy: error`` — the loader refuses an artifact missing any of them, or one whose overlaps do not conserve each PUMA's @@ -40,11 +40,15 @@ import sys import urllib.request import zipfile +from collections.abc import Iterator from pathlib import Path import numpy as np -from microcosm.build.us_runtime import load_us_puma_ladder +from microcosm.build.us_runtime import ( + US_PUMA_LADDER_SCHEMA_VERSION, + load_us_puma_ladder, +) from microcosm.build.us_runtime.block_ladder_sources import ( US_STATES, parse_national_cd_bef, @@ -117,6 +121,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Path for the build summary. Defaults beside --out.", ) + parser.add_argument( + "--source-manifest", + type=Path, + help=( + "Use only exact local inputs from a schema_version=1 JSON manifest " + "with sources keyed by tract_to_puma, cd119_bef and pl94171_. " + "Each row requires path, url, sha256 and bytes. Disables downloads." + ), + ) return parser.parse_args(argv) @@ -151,15 +164,52 @@ def _download(url: str, cache_dir: Path) -> Path: return destination -def _zip_member_lines(archive_path: Path, member: str) -> list[str]: +def _zip_member_lines(archive_path: Path, member: str) -> Iterator[str]: with zipfile.ZipFile(archive_path) as archive: with archive.open(member) as stream: - return io.TextIOWrapper(stream, encoding="latin-1").readlines() + yield from io.TextIOWrapper(stream, encoding="latin-1") -def _text_lines(path: Path) -> list[str]: +def _text_lines(path: Path) -> Iterator[str]: with path.open(encoding="utf-8-sig") as stream: - return stream.readlines() + yield from stream + + +def _pinned_sources(path: Path, urls: dict[str, str]) -> tuple[dict[str, dict], str]: + """Validate the input envelope without downloading or reading source payloads.""" + + raw = path.read_bytes() + manifest = json.loads(raw) + if not isinstance(manifest, dict) or manifest.get("schema_version") != 1: + raise ValueError("Pinned source manifest requires schema_version 1.") + sources = manifest.get("sources") + if not isinstance(sources, dict) or set(sources) != set(urls): + raise ValueError("Pinned source manifest must match the exact selected inputs.") + for name, url in urls.items(): + row = sources[name] + if not isinstance(row, dict) or set(row) != {"path", "url", "sha256", "bytes"}: + raise ValueError(f"Pinned source {name} requires path/url/sha256/bytes.") + if ( + not isinstance(row["path"], str) + or not Path(row["path"]).is_absolute() + or row["url"] != url + or type(row["bytes"]) is not int + or row["bytes"] <= 0 + or not isinstance(row["sha256"], str) + or len(row["sha256"]) != 64 + or any(char not in "0123456789abcdef" for char in row["sha256"]) + ): + raise ValueError(f"Invalid pinned source envelope for {name}.") + return sources, hashlib.sha256(raw).hexdigest() + + +def _verify_pinned_source(name: str, row: dict) -> Path: + path = Path(row["path"]) + if path.is_symlink() or not path.is_file(): + raise ValueError(f"Pinned source {name} must be a regular local file.") + if path.stat().st_size != row["bytes"] or _sha256(path) != row["sha256"]: + raise ValueError(f"Pinned source identity mismatch for {name}.") + return path def main(argv: list[str] | None = None) -> None: @@ -175,9 +225,29 @@ def main(argv: list[str] | None = None) -> None: allowed_state_fips = frozenset(fips for fips, _, _ in states) _log(f"Building US PUMA ladder for {len(states)} state(s)") + urls = {"tract_to_puma": TRACT_TO_PUMA_URL, "cd119_bef": CD119_BEF_URL} + urls.update( + { + f"pl94171_{usps.lower()}": PL94171_URL_TEMPLATE.format( + dirname=dirname, usps_lower=usps.lower() + ) + for _, usps, dirname in states + } + ) + pinned, manifest_sha256 = ( + _pinned_sources(args.source_manifest, urls) + if args.source_manifest + else (None, None) + ) + + def source_path(name: str) -> Path: + if pinned is not None: + return _verify_pinned_source(name, pinned[name]) + return _download(urls[name], args.cache_dir) + source_files: dict[str, dict[str, str]] = {} - tract_puma_txt = _download(TRACT_TO_PUMA_URL, args.cache_dir) + tract_puma_txt = source_path("tract_to_puma") source_files["tract_to_puma"] = { "url": TRACT_TO_PUMA_URL, "sha256": _sha256(tract_puma_txt), @@ -188,7 +258,7 @@ def main(argv: list[str] | None = None) -> None: ) _log(f" tract-to-PUMA: {len(tract_to_puma):,} tracts") - cd_zip = _download(CD119_BEF_URL, args.cache_dir) + cd_zip = source_path("cd119_bef") source_files["cd119_bef"] = {"url": CD119_BEF_URL, "sha256": _sha256(cd_zip)} _log(" parsing national CD119 BEF") cd_by_block = parse_national_cd_bef( @@ -199,7 +269,7 @@ def main(argv: list[str] | None = None) -> None: for fips, usps, dirname in states: _log(f" state {fips} {usps}") pl_url = PL94171_URL_TEMPLATE.format(dirname=dirname, usps_lower=usps.lower()) - pl_zip = _download(pl_url, args.cache_dir) + pl_zip = source_path(f"pl94171_{usps.lower()}") source_files[f"pl94171_{usps.lower()}"] = { "url": pl_url, "sha256": _sha256(pl_zip), @@ -211,7 +281,7 @@ def main(argv: list[str] | None = None) -> None: block_population.update(state_blocks) metadata = { - "schema_version": 1, + "schema_version": US_PUMA_LADDER_SCHEMA_VERSION, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", @@ -219,6 +289,12 @@ def main(argv: list[str] | None = None) -> None: "states": [fips for fips, _, _ in states], "source_files": source_files, } + if pinned is not None: + for name, row in pinned.items(): + _verify_pinned_source(name, row) + if _sha256(args.source_manifest) != manifest_sha256: + raise ValueError("Pinned source manifest changed during source parsing.") + metadata["source_manifest_sha256"] = manifest_sha256 _log(f" aggregating {len(block_population):,} populated blocks") payload = assemble_us_puma_ladder( block_population=block_population, @@ -238,12 +314,15 @@ def main(argv: list[str] | None = None) -> None: "congressional_district_overlaps": int(len(ladder.cd_overlap_puma)), "county_overlaps": int(len(ladder.county_overlap_puma)), "tract_overlaps": int(len(ladder.tract_overlap_puma)), + "joint_tract_cd_overlaps": int(len(ladder.joint_overlap_puma)), "congressional_districts": int(np.unique(ladder.cd_overlap_cd).size), "counties": int(np.unique(ladder.county_overlap_county).size), "layer_vintages": ladder.layer_vintages, "states": [fips for fips, _, _ in states], "source_files": source_files, } + if manifest_sha256 is not None: + summary["source_manifest_sha256"] = manifest_sha256 summary_path = ( args.summary_json if args.summary_json is not None From 4b79b5bd63178579e6cfe663ac4a285303f3e76e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 009/255] Stage integration layer 07: F-CATALOGUE-OPTIMIZATION.patch --- PROGRESS.md | 2 + .../us-survey-catalogue-digest.changed.md | 1 + .../survey_population_preparation.py | 81 +++++++++- .../tests/test_us_survey_catalogue_digest.py | 144 ++++++++++++++++++ ...st_us_survey_catalogue_digest_authority.py | 114 ++++++++++++++ ...st_us_survey_catalogue_digest_fast_path.py | 68 +++++++++ 6 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 changelog.d/us-survey-catalogue-digest.changed.md create mode 100644 packages/microcosm-build/tests/test_us_survey_catalogue_digest.py create mode 100644 packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py create mode 100644 packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py diff --git a/PROGRESS.md b/PROGRESS.md index 483ac5b64..16b7d46d2 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -749,3 +749,5 @@ Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservati Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/changelog.d/us-survey-catalogue-digest.changed.md b/changelog.d/us-survey-catalogue-digest.changed.md new file mode 100644 index 000000000..96d1c8d13 --- /dev/null +++ b/changelog.d/us-survey-catalogue-digest.changed.md @@ -0,0 +1 @@ +Stream eligible ACS catalogue seals through bounded record JSON encoding while preserving canonical digest bytes, full-value fallback, and repeated source and owner validation. Bind the additional JSON provider calls into the existing preparation authority checks. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 2840e376f..681a16a19 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -115,6 +115,68 @@ def _digest(value): return digest.hexdigest() +def _catalogue_fast_path(value): + """Recognize immutable, bounded raw records without changing admission.""" + if type(value) is not tuple or len(value) != 2: + return False + for group in value: + if type(group) is not tuple: + return False + for record in group: + if type(record) is not tuple or len(record) != 7: + return False + people = record[6] + if type(people) is not tuple or len(people) > 20: + return False + if any(type(person) is not tuple or len(person) != 9 for person in people): + return False + characters = 0 + for row in (record[:6], *people): + for item in row: + if type(item) is str: + if not item.isascii(): + return False + characters += len(item) + if characters > 65_536: + return False + elif type(item) is int: + if not -(2**63) <= item < 2**63: + return False + else: + return False + return True + +def _catalogue_chunks(value): + # Decide for the whole value before encoding. Unexpected shapes retain + # the generic encoder's full-depth scalar pass and error precedence. + if not _catalogue_fast_path(value): + yield from _chunks(value) + return + _check_scalars(value) + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + yield "[" + for group_index, group in enumerate(value): + if group_index: + yield "," + yield "[" + for record_index, record in enumerate(group): + if record_index: + yield "," + # encode uses the ordinary one-shot C provider when available. + # At most 186 leaves and 65,536 ASCII characters bound this + # individual JSON record below 400 KiB, including escaping. + yield encoder.encode(record) + yield "]" + yield "]" + +def _catalogue_digest(value): + digest = hashlib.sha256() + for chunk in _catalogue_chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + def _value(value): if isinstance(value, Enum): return value.value @@ -410,6 +472,23 @@ def _live(): ) result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) result["csv"] = (csv.reader, _csv.reader) + # The bounded catalogue path also uses JSONEncoder.encode and its C + # provider. Bind their actual live identities through this existing seal. + result["json_catalogue"] = ( + json.JSONEncoder, + json.encoder, + tuple( + _function_seal(function) + if isinstance(function, FunctionType) + else _runtime_marker(function) + for function in ( + getattr(json.JSONEncoder, name, None) + for name in ("__init__", "encode", "iterencode") + ) + ), + getattr(json.encoder, "c_make_encoder", None), + getattr(json.encoder, "encode_basestring", None), + ) result["contract"] = ( PROTOCOL, REQUEST_PROTOCOL, @@ -881,7 +960,7 @@ def _nested_seals(catalogues, native): for paths in acs.snapshots ] ), - _digest((acs_cat.records, acs_cat.vacancies)), + _catalogue_digest((acs_cat.records, acs_cat.vacancies)), ] for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): entry = module._ISSUED.get(id(value)) diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py new file mode 100644 index 000000000..db141097d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py @@ -0,0 +1,144 @@ +"""Exact canonical catalogue bytes and unchanged fallback refusals.""" + +import hashlib + +import pytest +from microcosm.build.us_runtime import survey_population_preparation as owner + +chunks = owner._catalogue_chunks +digest = owner._catalogue_digest + + +def _person(): + return ("01", "37", "1", "reported", "0", "reported", "42", "p.csv", 2) + + +def _record(people=()): + return ("2024HU0000001", "1", str(len(people)), "12", "h.csv", 1, people) + + +def _raw(value): + return "".join(chunks(value)).encode("utf-8") + + +@pytest.mark.parametrize( + "value, expected", + [ + (((), ()), b"[[],[]]"), + ( + ((), (_record(),)), + b'[[],[["2024HU0000001","1","0","12","h.csv",1,[]]]]', + ), + ( + ((_record((_person(),)),), (_record(),)), + b'[[["2024HU0000001","1","1","12","h.csv",1,' + b'[["01","37","1","reported","0","reported","42","p.csv",2]]]],' + b'[["2024HU0000001","1","0","12","h.csv",1,[]]]]', + ), + ( + ((("é", '"\\\n', "\x00", "😀", "z", -1, ()),), ()), + '[[["é","\\"\\\\\\n","\\u0000","😀","z",-1,[]]],[]]'.encode("utf-8"), + ), + ], +) +def test_frozen_canonical_bytes(value, expected): + # The literal expected byte sequence is independent of either encoder. + assert _raw(value) == expected + assert "".join(owner._chunks(value)).encode("utf-8") == expected + assert digest(value) == hashlib.sha256(expected).hexdigest() + assert digest(value) == owner._digest(value) + + +def test_order_duplicates_and_per_record_boundaries(): + a, b = _record((_person(),)), ("2024GQ0000002", "3", "0", "0", "g", 7, ()) + values = [((a, b, a), ()), ((b, a, a), ()), ((a, a), (b,))] + actual = [digest(value) for value in values] + assert len(set(actual)) == 3 + for value in values: + assert _raw(value) == "".join(owner._chunks(value)).encode("utf-8") + + +@pytest.mark.parametrize("characters", [65531, 65532, 65533]) +def test_fast_eligibility_character_boundary_does_not_change_acceptance(characters): + # Four remaining one-character strings put 65532 exactly at 65536 total; + # the ordinal is an int. Acceptance must not depend on eligibility. + value = ((("x" * characters, "1", "0", "0", "h", 1, ()),), ()) + assert _raw(value) == "".join(owner._chunks(value)).encode("utf-8") + assert digest(value) == owner._digest(value) + + +@pytest.mark.parametrize("count", [20, 21]) +def test_person_count_fast_boundary_falls_back_without_new_refusal(count): + value = ((_record((_person(),) * count),), ()) + assert _raw(value) == "".join(owner._chunks(value)).encode("utf-8") + + +def test_generic_accepted_shapes_and_scalar_types_keep_old_bytes(): + values = [ + ([list(_record())], []), + (({"z": 1, "a": [None, True, False, 1.0, -0.0, "é"]},), ()), + (((1 << 90, "1", "0", "0", "h", 1, ()),), ()), + ((("x" * owner._MAX_SCALAR_BYTES, "1", "0", "0", "h", 1, ()),), ()), + ] + for value in values: + assert _raw(value) == "".join(owner._chunks(value)).encode("utf-8") + assert digest(value) == owner._digest(value) + + +def _error(function, value): + try: + function(value) + except Exception as exc: + return type(exc), str(exc) + pytest.fail("Malformed corpus member was accepted") + + +def test_malformed_refusal_and_global_scalar_precedence(): + deep = 0 + for _ in range(65): + deep = (deep,) + cycle = [] + cycle.append(cycle) + oversized = "x" * (owner._MAX_SCALAR_BYTES + 1) + cases = [ + (((float("nan"),),), ()), + (((float("inf"),),), ()), + (((float("-inf"),),), ()), + (((b"not-json",),), ()), + ((("\ud800",),), ()), + ((("\udfff",),), ()), + ((({1, 2},),), ()), + ((({1: 1, "a": 2},),), ()), + ((deep,), ()), + ((cycle,), ()), + ((oversized,), ()), + ((float("nan"), oversized), ()), + ((b"bad", deep), ()), + ] + for value in cases: + assert _error(digest, value) == _error(owner._digest, value) + assert _error(digest, ((float("nan"), oversized), ()))[1] == "SCALAR_LIMIT" + assert _error(digest, ((b"bad", deep), ()))[1] == "VALUE_DEPTH" + + +def test_subclasses_use_unchanged_dispatch(): + class Text(str): + pass + + class Integer(int): + pass + + class Sequence(tuple): + pass + + for value in [((Text("é"), Integer(2)), ()), Sequence(((), ()))]: + assert _raw(value) == "".join(owner._chunks(value)).encode("utf-8") + + +def test_digest_is_independent_of_an_explicit_tiny_transport_bound(): + # This small value does not test the production 64 MiB boundary. + # It checks digest parity alongside an independently tiny transport limit. + value = ((_record((_person(),)),) * 1024, ()) + assert digest(value) == owner._digest(value) + with pytest.raises(owner.SurveyPopulationPreparationError, match="PAYLOAD_LIMIT"): + owner._encode(value, 30) diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py new file mode 100644 index 000000000..f1591156c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py @@ -0,0 +1,114 @@ +"""Real invented preparation owners retain final catalogue/provider checks.""" + +import sys +from types import SimpleNamespace + +import pytest +from microcosm.build.us_runtime import survey_population_preparation as owner +from test_us_survey_population_preparation import fixture + + +def _must_not_execute(*_args, **_kwargs): + raise AssertionError("Changed JSON provider executed before owner refusal") + + +def test_changed_catalogue_json_providers_refuse_before_use(tmp_path, monkeypatch): + result = owner.prepare_authenticated_survey_population( + **fixture(tmp_path, monkeypatch) + ) + encoder = owner.json.JSONEncoder + module = owner.json.encoder + mutations = [ + (owner.json, "JSONEncoder", type("ChangedJSONEncoder", (encoder,), {})), + (owner.json, "encoder", SimpleNamespace(**vars(module))), + (module, "c_make_encoder", _must_not_execute), + (module, "encode_basestring", _must_not_execute), + ] + for name in ("__init__", "encode", "iterencode"): + function = getattr(encoder, name) + mutations.extend( + [ + (encoder, name, _must_not_execute), + (function, "__code__", _must_not_execute.__code__), + (function, "__defaults__", ("changed-catalogue-default",)), + (function, "__kwdefaults__", {"changed_catalogue_default": True}), + ] + ) + assert len(mutations) == 16 + assert owner._live() == owner._LIVE + observed = 0 + for target, name, changed in mutations: + before = getattr(target, name) + with monkeypatch.context() as change: + change.setattr(target, name, changed) + assert getattr(target, name) is changed + assert owner._live() != owner._LIVE + with pytest.raises( + owner.SurveyPopulationPreparationError, + match="^FINAL_AUTHORITY_CHANGED$", + ): + result.checked_view() + observed += 1 + assert getattr(target, name) is before + assert owner._live() == owner._LIVE + assert observed == 16 + result.checked_view() + + +@pytest.mark.parametrize("field", ["records", "vacancies", "c_make_encoder"]) +def test_late_catalogue_or_provider_change_refuses_current_borrow( + tmp_path, monkeypatch, field +): + result = owner.prepare_authenticated_survey_population( + **fixture(tmp_path, monkeypatch) + ) + state = owner._ISSUED[id(result)][2] + catalogue = owner.acs_catalogue._lookup(state.catalogues[0]) + if field == "c_make_encoder": + target = owner.json.encoder + original = target.c_make_encoder + assert original is not None + replacement = _must_not_execute + reason = "FINAL_AUTHORITY_CHANGED" + else: + target = catalogue + original = getattr(catalogue, field) + assert original + records = list(original) + record = list(records[-1]) + record[3] += "0" + records[-1] = tuple(record) + replacement = tuple(records) + reason = "NESTED_EVIDENCE_CHANGED" + fired = [] + + def observe(frame, event, _argument): + if ( + event == "return" + and frame.f_code is owner._producer.__code__ + and frame.f_back is not None + and frame.f_back.f_code is owner._validate.__code__ + and not fired + ): + fired.append(True) + if field == "c_make_encoder": + setattr(target, field, replacement) + else: + object.__setattr__(target, field, replacement) + + previous = sys.getprofile() + try: + sys.setprofile(observe) + with pytest.raises( + owner.SurveyPopulationPreparationError, match="^" + reason + "$" + ): + result.checked_view() + assert fired == [True] + finally: + sys.setprofile(previous) + if field == "c_make_encoder": + setattr(target, field, original) + else: + object.__setattr__(target, field, original) + assert getattr(target, field) is original + result.checked_view() diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py new file mode 100644 index 000000000..b557cca6e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py @@ -0,0 +1,68 @@ +"""Candidate-only eligibility controls; never run against the predecessor.""" + +import hashlib + +import pytest +from microcosm.build.us_runtime import survey_population_preparation as owner + + +def _value(*, characters=0, ordinal=1, people=()): + return ((("x" * characters, "1", "0", "0", "h", ordinal, people),), ()) + + +def _person(): + return ("1", "20", "1", "reported", "0", "reported", "1", "p", 1) + + +@pytest.mark.parametrize( + "value, expected", + [ + (((), ()), True), + (_value(characters=65531), True), + (_value(characters=65532), True), + (_value(characters=65533), False), + (_value(ordinal=-(2**63)), True), + (_value(ordinal=2**63 - 1), True), + (_value(ordinal=-(2**63) - 1), False), + (_value(ordinal=2**63), False), + (_value(people=(_person(),) * 20), True), + (_value(people=(_person(),) * 21), False), + (_value(ordinal=True), False), + (_value(ordinal=1.0), False), + (_value(ordinal=None), False), + (_value(ordinal="é"), False), + (_value(ordinal="\ud800"), False), + (([], ()), False), + ([(), ()], False), + (((("short",),), ()), False), + ], +) +def test_exact_eligibility_boundary(value, expected): + assert owner._catalogue_fast_path(value) is expected + + +def test_subclass_ineligibility(): + class Text(str): + pass + + class Integer(int): + pass + + class Sequence(tuple): + pass + + for value in [ + _value(ordinal=Text("x")), + _value(ordinal=Integer(1)), + Sequence(((), ())), + ]: + assert owner._catalogue_fast_path(value) is False + + +def test_ascii_escaping_uses_eligible_canonical_stream(): + value = ((("a", '"\\\n', "\x00", "0", "h", 1, ()),), ()) + expected = b'[[["a","\\"\\\\\\n","\\u0000","0","h",1,[]]],[]]' + assert owner._catalogue_fast_path(value) is True + assert "".join(owner._catalogue_chunks(value)).encode("utf-8") == expected + assert "".join(owner._chunks(value)).encode("utf-8") == expected + assert owner._catalogue_digest(value) == hashlib.sha256(expected).hexdigest() From 60714ba9daee288e4f2e391b3f8f234fb1a51469 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 010/255] Stage integration layer 08: GRAPH-ATTACHMENT-METADATA.patch --- PROGRESS.md | 2 ++ .../src/microcosm/graph/attachments.py | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 16b7d46d2..28b573e0b 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -751,3 +751,5 @@ Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preserv Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-graph/src/microcosm/graph/attachments.py b/packages/microcosm-graph/src/microcosm/graph/attachments.py index 885e23607..fd2e16446 100644 --- a/packages/microcosm-graph/src/microcosm/graph/attachments.py +++ b/packages/microcosm-graph/src/microcosm/graph/attachments.py @@ -3,6 +3,8 @@ These store references never enter node records or portable manifest identity. """ +import hashlib + from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path @@ -21,6 +23,9 @@ _FRAME_FORMAT, ContentStore, StoreCorrupt, + StoreUnavailable, + _canonical_json, + _encode_frame_metadata, _payload_table, _write_frame, ) @@ -80,9 +85,8 @@ def __getitem__(self, version: str) -> PopulationView: "The decoded population attachment changed after execution." ) view = PopulationView(frame) - # The existing Frame store codec omits metadata. Retain the - # original Frame's already-frozen scalar/container descriptor, - # which cannot own tables/arrays, for this live manifest only. + # Retain the original already-frozen metadata object after + # the decoded snapshot has passed the content check above. object.__setattr__(view, "_metadata", reference.metadata) self._cache[version] = view return view @@ -205,6 +209,16 @@ def _snapshot( """ population.frame.revalidate() + metadata_sha256 = hashlib.sha256( + _canonical_json(_encode_frame_metadata(population.frame.metadata)) + ).hexdigest() + + def validate_existing(metadata: Mapping[str, object]) -> None: + if metadata.get("frame_format") != _FRAME_FORMAT: + raise StoreUnavailable("Stored frame predates complete metadata storage.") + if metadata.get("frame_metadata_sha256") != metadata_sha256: + raise StoreCorrupt("The same frame key cannot carry different metadata.") + with TemporaryDirectory( prefix="population-attachment-", dir=store.tmp ) as directory: @@ -233,7 +247,17 @@ def _snapshot( def publish(destination: Path) -> Mapping[str, object]: for child in staging.iterdir(): child.rename(destination / child.name) - return {"frame_format": _FRAME_FORMAT, "node_key": structural.key} + return { + "frame_format": _FRAME_FORMAT, + "node_key": structural.key, + "frame_metadata_sha256": metadata_sha256, + } - store._put(key, "frame", publish, verify_existing=verify_existing) + store._put( + key, + "frame", + publish, + verify_existing=verify_existing, + validate_existing=validate_existing, + ) return key, payload_hash From 6421fa3c6919df39847cf8b31790a26818278857 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 011/255] Stage integration layer 09: F-JOINT-GEOGRAPHY-GATE.patch --- PROGRESS.md | 2 ++ .../build/us_runtime/graph_geography.py | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 28b573e0b..2de8ce0f4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -753,3 +753,5 @@ Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preser Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py index dff2f747e..df10d4bb7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_geography.py @@ -1,8 +1,9 @@ """Declared lookup import, household assignment, and geography validation. -The existing PUMA operator keeps native ACS PUMAs and uses its legacy seeded -draw order for missing PUMAs, districts and counties. Reference import and the -post-assignment gate are separate graph nodes. No target values are fitted here. +The PUMA operator keeps native ACS PUMAs and draws a supported joint tract/CD +cell for each household; county derives from that tract. Reference import and +the assignment-integrity and joint-support gates are separate graph nodes. +No target values are fitted here. """ from __future__ import annotations @@ -61,6 +62,7 @@ decode_us_puma_ladder, us_puma_ladder_assignment_summary, us_puma_ladder_gate, + us_puma_ladder_joint_support_gate, ) US_PUMA_LOOKUP_TYPE = ArtifactType("microcosm.us.puma_ladder_npz", 1) @@ -178,7 +180,7 @@ def run(self, context: KernelContext) -> KernelResult: "assign_tract": False, }: raise ValueError( - "US geography requires the declared legacy seed and county/CD contract." + "US geography requires the declared seed and joint county/CD contract." ) artifact = context.artifacts.get("frame_context") if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: @@ -278,10 +280,19 @@ def run(self, context: KernelContext) -> KernelResult: context.weights["household"].values, assign_tract=False, ) + ladder, _ = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + joint_gate = us_puma_ladder_joint_support_gate( + context.tables["household"], + ladder, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) return KernelResult( receipt={ - "outcome": "pass" if gate.passed else "fail", - "evidence": GateReport((gate,)).to_manifest(), + "outcome": "pass" if gate.passed and joint_gate.passed else "fail", + "evidence": GateReport((gate, joint_gate)).to_manifest(), "scope": "geography_assignment_integrity", } ) @@ -366,6 +377,10 @@ def us_geography_nodes( population=boundary, inputs=(Slice("household", ("state_fips", *US_PUMA_LADDER_COLUMNS)),), params={"phase": GEOGRAPHY_PHASE}, + artifact_inputs=( + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), ), ) From 9fcce2b4e6d7b65665884b11a6aea7fa9303a3f5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 012/255] Stage integration layer 10: SOURCE-CLOSURE.patch --- PROGRESS.md | 2 + .../us_runtime/graph_acs_housing_universe.py | 698 ++++++++++++++++++ .../native_origin_graph_inventory.json | 89 +++ 3 files changed, 789 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json diff --git a/PROGRESS.md b/PROGRESS.md index 2de8ce0f4..d56c3a1b2 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -755,3 +755,5 @@ Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and pre Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py new file mode 100644 index 000000000..2929f225e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py @@ -0,0 +1,698 @@ +"""Typed ACS source evidence and exact whole-household graph selection. + +Only CREATE authenticates source bytes. Selection keeps the original evidence as +an ancestor and emits selected identities/positions; it never mints source +authority or asserts calibrated representation. +""" + +from __future__ import annotations + +import json +import struct +from dataclasses import InitVar, dataclass, replace +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import acs_housing_universe_source as source +from .graph_context import ( + US_FRAME_CONTEXT_TYPE, + encode_us_frame_context, + us_frame_from_context, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ACS_HU_PAYLOAD_MAX_BYTES = 64 * 1024**2 +ACS_HU_HEADER_MAX_BYTES = 65536 +MAGIC = bytes.fromhex("4d43414353485502") +US_ACS_HOUSING_UNIVERSE_TYPE = ArtifactType("microcosm.us.acs_housing_universe", 2) +US_ACS_HOUSING_PREPARATION_TYPE = ArtifactType( + "microcosm.us.acs_housing_preparation", 2 +) +US_ACS_HOUSING_SELECTION_TYPE = ArtifactType("microcosm.us.acs_housing_selection", 2) +CREATE_NODE = "acs_housing_create" +SELECT_NODE = "acs_housing_select" +SOURCE_NAME = "acs_housing_source" +PHASE = "acs_housing_universe_graph_v2" +_TOKEN = object() +_CODES = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) +_HEADER_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_bytes", + "projection_bytes", + "household_rows", + "person_rows", + "code_columns", + "body_bytes", + "source_receipt_sha256", + "projection_sha256", + "prepared_receipt_sha256", + "frame_context_sha256", + "frame_sha256", + "definition_sha256", + "implementation_sha256", + } +) +_ARTIFACT_OUTPUTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ArtifactOutput("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), +) +_PREPARED_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_sha256", + "projection_sha256", + "implementation_sha256", + "same_snapshot_frame_and_observations", + "full_source_frame_before_selection", + "full_source_lexical_projection", + "native_selection_before_person_accumulation", + "selection_kind", + "requested_serialnos", + "full_source_inclusion_probability", + "pre_promotion_frame_sha256", + "frame_sha256", + "dtype_transitions", + "entity_rows", + "weight_kind", + "HU_columns", + } +) + + +def _require(value, code): + source._require(value, "GRAPH_ACS_" + code) + + +@dataclass(frozen=True) +class BoundACSHousingEvidence: + """Immutable graph transport, explicitly not AuthenticatedACSHousingSource.""" + + header_json: bytes + source_receipt_json: bytes + projection_json: bytes + codes: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BOUND_CONSTRUCTOR") + + +def _verify_preparation(prepared, receipt, household, frame_context): + """Check the v2 construction claim without minting source authority.""" + _require( + set(prepared) == _PREPARED_FIELDS + and prepared["format"] == "microcosm.acs_housing_preparation.v2" + and prepared["release_eligible"] is False + and prepared["HU_columns"] == "artifact_only" + and prepared["weight_kind"] == "design" + and prepared["same_snapshot_frame_and_observations"] is True + and prepared["full_source_lexical_projection"] is True + and prepared["full_source_inclusion_probability"] is None, + "PREPARED_SCHEMA", + ) + requested = prepared["requested_serialnos"] + subset = requested is not None + _require( + not subset + or ( + type(requested) is list + and bool(requested) + and all(type(key) is str for key in requested) + and len(set(requested)) == len(requested) + ), + "PREPARED_SELECTION", + ) + chosen = tuple(sorted(household.SERIALNO)) + _require( + prepared["full_source_frame_before_selection"] is (not subset) + and prepared["native_selection_before_person_accumulation"] is subset + and prepared["selection_kind"] + == ("engineering_exact_keys" if subset else "all") + and receipt["selection"] == ("exact_serialnos" if subset else "all") + and (not subset or tuple(sorted(requested)) == chosen) + and receipt["selected_serialnos_sha256"] == source._sha(canonical_json(chosen)), + "PREPARED_SELECTION", + ) + context = source._parse_json(frame_context, ACS_HU_PAYLOAD_MAX_BYTES) + rows = prepared["entity_rows"] + entities = context.get("entities") + _require( + type(rows) is dict + and type(entities) is dict + and set(rows) == set(entities) == set(US_SCHEMA.entities) + and all( + type(rows[entity]) is int + and rows[entity] >= 0 + and type(entities[entity]) is dict + and type(entities[entity].get("rows")) is int + and rows[entity] == entities[entity]["rows"] + for entity in US_SCHEMA.entities + ), + "PREPARED_CONTEXT", + ) + + +def encode_acs_housing_evidence(prepared): + _require( + type(prepared) is source.PreparedACSHousingPopulation + and type(prepared.source) is source.AuthenticatedACSHousingSource, + "PREPARED_TYPE", + ) + receipt = source._parse_json(prepared.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES) + observation = prepared.source + projection = source._parse_json( + observation.projection_json, ACS_HU_PAYLOAD_MAX_BYTES + ) + _h, _p, _t, _lines, _pw, codes = source._tables(projection) + source.verify_acs_frame_projection(prepared.frame, projection) + _require( + receipt["source_receipt_sha256"] == source._sha(observation.receipt_json) + and receipt["projection_sha256"] == source._sha(observation.projection_json) + and receipt["frame_sha256"] == source.frame_content_sha256(prepared.frame), + "PREPARED_BINDING", + ) + context = encode_us_frame_context(prepared.frame) + _verify_preparation( + receipt, + source._parse_json(observation.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES), + _h, + context, + ) + native = b"".join(codes[n].to_numpy(dtype="uint8").tobytes() for n in _CODES) + body = observation.receipt_json + observation.projection_json + native + header = { + "format": "microcosm.acs_housing_graph_evidence.v2", + "release_eligible": False, + "source_receipt_bytes": len(observation.receipt_json), + "projection_bytes": len(observation.projection_json), + "household_rows": len(_h), + "person_rows": len(_p), + "code_columns": list(_CODES), + "body_bytes": len(body), + "source_receipt_sha256": source._sha(observation.receipt_json), + "projection_sha256": source._sha(observation.projection_json), + "prepared_receipt_sha256": source._sha(prepared.receipt_json), + "frame_context_sha256": source._sha(context), + "frame_sha256": receipt["frame_sha256"], + "definition_sha256": source._definition()[1], + "implementation_sha256": source._implementation(), + } + encoded = canonical_json(header) + _require(len(encoded) <= ACS_HU_HEADER_MAX_BYTES, "HEADER_SIZE") + payload = MAGIC + struct.pack(" 0 + and nhouse > 0, + "SECTION_SIZE", + ) + _require( + header["body_bytes"] == nreceipt + nprojection + nhouse * len(_CODES) + and len(payload) == start + size + header["body_bytes"] + 32, + "BODY_SIZE", + ) + cursor = start + size + receipt_bytes = payload[cursor : cursor + nreceipt] + projection_bytes = payload[cursor + nreceipt : cursor + nreceipt + nprojection] + codes = payload[cursor + nreceipt + nprojection : -32] + receipt = source._parse_json(receipt_bytes, source.ACS_HU_RECEIPT_MAX_BYTES) + projection = source._parse_json(projection_bytes, ACS_HU_PAYLOAD_MAX_BYTES) + prepared = source._parse_json(prepared_receipt, source.ACS_HU_RECEIPT_MAX_BYTES) + household, person, _typed, _lines, _pw, derived = source._tables(projection) + _require( + source._sha(prepared_receipt) == header["prepared_receipt_sha256"] + and source._sha(frame_context) == header["frame_context_sha256"], + "CONTEXT_BINDING", + ) + _verify_preparation(prepared, receipt, household, frame_context) + _require( + source._sha(receipt_bytes) + == header["source_receipt_sha256"] + == prepared["source_receipt_sha256"] + and source._sha(projection_bytes) + == header["projection_sha256"] + == prepared["projection_sha256"] + == receipt["projection_sha256"], + "SOURCE_BINDING", + ) + _require( + header["definition_sha256"] + == receipt["definition_sha256"] + == source._definition()[1] + and header["implementation_sha256"] + == prepared["implementation_sha256"] + == receipt["implementation_sha256"] + == source._implementation(), + "IMPLEMENTATION_BINDING", + ) + _require(header["frame_sha256"] == prepared["frame_sha256"], "FRAME_BINDING") + # Coverage of the pins is otherwise only transitive, through the + # implementation hash over the archive manifest. Comparing them here + # makes the evidence say what it rests on. + _require( + receipt["format"] == "microcosm.acs_housing_universe_source.v1" + and receipt["release_eligible"] is False + and receipt["vintage"] == 2024 + and receipt["archives"] + == [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in source._pins() + ], + "SOURCE_PINS", + ) + _require( + len(household) == nhouse == receipt["selected_counts"]["households"] + and len(person) + == header["person_rows"] + == receipt["selected_counts"]["persons"], + "ROW_COUNTS", + ) + _require( + codes + == b"".join( + derived[name].to_numpy(dtype="uint8").tobytes() for name in _CODES + ), + "NATIVE_CODES", + ) + return BoundACSHousingEvidence( + raw_header, receipt_bytes, projection_bytes, codes, _token=_TOKEN + ) + except source.ACSHousingSourceError: + raise + except ( + ValueError, + TypeError, + KeyError, + OverflowError, + struct.error, + AttributeError, + ): + raise source.ACSHousingSourceError("GRAPH_ACS_TRANSPORT_CONTRACT") from None + + +def _artifact(context, name, expected_type): + value = context.artifacts.get(name) + _require(value is not None and value.type == expected_type, "ARTIFACT_TYPE") + declarations = [item for item in context.node.artifact_inputs if item.name == name] + _require( + len(declarations) == 1 + and value.key + == opaque_artifact_key(value.producer_key, declarations[0].artifact), + "ARTIFACT_KEY", + ) + return value + + +def _keys(value): + if value is None: + return None + _require( + isinstance(value, (tuple, list)) + and bool(value) + and all(type(v) is str for v in value) + and len(set(value)) == len(value), + "SELECTION_KEYS", + ) + return tuple(value) + + +def _slices(columns): + # The compiler does not treat implicit entity IDs as owned data columns. + # Structural-only groups are reconstructed from declared person membership + # and checked against the parent's typed ID digest, never read as fake cells. + return tuple( + Slice(entity, tuple(c.column for c in columns if c.entity == entity)) + for entity in US_SCHEMA.entities + if any(c.entity == entity for c in columns) + ) + + +def _complete_structural_tables(context, frame_context): + document = json.loads(frame_context) + tables = dict(context.tables) + _require("person" in tables, "PERSON_VIEW") + for group in US_SCHEMA.group_entities: + if group in tables: + continue + id_column = US_SCHEMA.entity_id_column(group) + _require( + document["entities"][group]["columns"] == [id_column], + "UNDECLARED_GROUP_CELLS", + ) + membership = tables["person"][US_SCHEMA.membership_column(group)] + _require(membership.dtype == np.dtype("int64"), "MEMBERSHIP_DTYPE") + tables[group] = pd.DataFrame({id_column: np.unique(membership.to_numpy())}) + # us_frame_from_context checks every reconstructed group's actual IDs + # against the original producer's ordered-ID digest before Frame creation. + return replace(context, tables=tables) + + +class _Kernel(KernelBase): + def implementation_hash(self): + return implementation_hash(source.ACS_HU_STAGE) + + +class ACSHousingCreateKernel(_Kernel): + ref = "us.acs_housing.create@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def __init__(self, snapshot_root): + self.snapshot_root = Path(snapshot_root) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.CREATE + and tuple(node.sources) == (SOURCE_NAME,) + and not node.inputs + and not node.artifact_inputs + and node.artifact_outputs == _ARTIFACT_OUTPUTS, + "CREATE_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + prepared = source.prepare_acs_housing_population( + context.sources[SOURCE_NAME], + snapshot_root=self.snapshot_root, + serialnos=_keys(context.params["serialnos"]), + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "COLUMN_INVENTORY", + ) + payload = encode_acs_housing_evidence(prepared) + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": encode_us_frame_context(prepared.frame), + "prepared_receipt": prepared.receipt_json, + "housing_universe": payload, + }, + receipt={ + "phase": PHASE, + "prepared": prepared.receipt, + "housing_universe_sha256": source._sha(payload), + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "release_eligible": False, + }, + ) + + +class ACSHousingSelectKernel(_Kernel): + ref = "us.acs_housing.select@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.FILTER + and node.mass == "declared" + and not node.outputs + and not node.sources, + "SELECT_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + _require( + set(context.artifacts) + == {"frame_context", "prepared_receipt", "housing_universe"} + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + "SELECT_ARTIFACTS", + ) + values = [ + _artifact(context, n, t) + for n, t in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), + ) + ] + _require( + len({v.producer_key for v in values}) == 1 + and len({a.producer for a in node.artifact_inputs}) == 1 + and next(iter(node.artifact_inputs)).producer == node.base, + "SAME_PRODUCER", + ) + bound = bind_acs_housing_evidence( + values[2].payload, + prepared_receipt=values[1].payload, + frame_context=values[0].payload, + ) + frame = us_frame_from_context( + _complete_structural_tables(context, values[0].payload) + ) + _require( + tuple(node.inputs) == _slices(frame_column_declarations(frame)), + "COMPLETE_FRAME_DECLARATION", + ) + header = json.loads(bound.header_json) + _require( + source.frame_content_sha256(frame) == header["frame_sha256"], "FRAME_CELLS" + ) + projection = json.loads(bound.projection_json) + source.verify_acs_frame_projection(frame, projection) + selected_projection, chosen, _counts = source._select( + projection, _keys(context.params["serialnos"]) + ) + household = frame.table("household") + household_keep = household.SERIALNO.isin(chosen).to_numpy() + ids = household.household_id.to_numpy()[household_keep] + keep = frame.person.person_household_id.isin(ids) + _require(bool(keep.any()), "EMPTY_SELECTION") + selected = frame.select(keep) + source.verify_acs_frame_projection(selected, selected_projection) + positions = { + "person": np.flatnonzero(keep).astype("int64"), + "household": np.flatnonzero(household_keep).astype("int64"), + } + _require( + selected.weights_for("household").values.tobytes() + == frame.weights_for("household").values[positions["household"]].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + before_mass, after_mass = frame.stratum_mass(), selected.stratum_mass() + native_households = dict( + zip( + selected.table("household").household_id, + selected.table("household").SERIALNO, + strict=True, + ) + ) + selection = { + "format": "microcosm.acs_housing_selection.v2", + "release_eligible": False, + "representative": False, + "source_authority": "original_CREATE_only", + "producer_key": values[0].producer_key, + "source_payload_sha256": source._sha(values[2].payload), + "source_prepared_receipt_sha256": source._sha(values[1].payload), + "source_frame_context_sha256": source._sha(values[0].payload), + "selected_frame_sha256": source.frame_content_sha256(selected), + "selected_frame_context_sha256": source._sha( + encode_us_frame_context(selected) + ), + "selected_projection_sha256": source._sha( + canonical_json(selected_projection) + ), + "selected_serialnos_sha256": source._sha(canonical_json(chosen)), + "positions": {e: a.tolist() for e, a in positions.items()}, + "frame_identity_scope": "actual_CREATE_frame", + "native_keys": { + "household": selected.table("household").SERIALNO.tolist(), + "person": [ + [native_households[household_id], int(line)] + for household_id, line in selected.person[ + ["person_household_id", "SPORDER"] + ].itertuples(index=False, name=None) + ], + }, + "ordered_ids": { + e: selected.table(e)[US_SCHEMA.entity_id_column(e)].tolist() + for e in US_SCHEMA.entities + }, + "weight_kind": "design", + "mass_policy": "declared_no_normalization", + } + return KernelResult( + keep=pd.Series( + keep.to_numpy(dtype=bool), + index=pd.Index(frame.person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + artifacts={ + "frame_context": encode_us_frame_context(selected), + "selection": canonical_json(selection), + }, + receipt={ + "phase": PHASE, + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "selection_sha256": source._sha(canonical_json(selection)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(k): float(v) for k, v in before_mass.items() + }, + "stratum_after": {str(k): float(v) for k, v in after_mass.items()}, + }, + }, + ) + + +def acs_housing_graph(columns, *, serialnos=None, selected_serialnos=None): + """Explicit seed source selection followed by exact further household pruning.""" + columns = tuple(columns) + _keys(serialnos) + _keys(selected_serialnos) + create = Node( + id=CREATE_NODE, + kernel=ACSHousingCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={"phase": PHASE, "serialnos": serialnos}, + artifact_outputs=_ARTIFACT_OUTPUTS, + ) + select = Node( + id=SELECT_NODE, + kernel=ACSHousingSelectKernel.ref, + structural=StructuralDelta.FILTER, + mass="declared", + base=CREATE_NODE, + inputs=_slices(columns), + params={"phase": PHASE, "serialnos": selected_serialnos}, + artifact_inputs=tuple( + ArtifactInput(a.name, CREATE_NODE, a.name, a.type) + for a in _ARTIFACT_OUTPUTS + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + ) + return Graph( + country="us", + sources=(SourceRef(SOURCE_NAME, codec=source.ACS_HU_CODEC),), + nodes=(create, select), + ) + + +def acs_housing_registry(*, snapshot_root): + registry = KernelRegistry() + registry.register(ACSHousingCreateKernel(snapshot_root)) + registry.register(ACSHousingSelectKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json new file mode 100644 index 000000000..7232f1347 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json @@ -0,0 +1,89 @@ +{ + "base_stages": [ + "composed_asec_binding_v1", + "acs_housing_universe_2024" + ], + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_native_household_origin.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_native_origin_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.canonical", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "6420544565b4a2b11dc7238e06eef099a429982afa3dfbcec3dbdfbba7a27bd1" + }, + "microcosm.build/us_runtime/graph_native_origin_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_implementation" + ], + "resource_accesses_sha256": "ecc7b6bd345950216623b7de94c9aeb392504af36e398094481f04ad203914f9", + "unbound_uses_sha256": "51778c9d32f80dfeba9741570ff5fa19ef83b464d7b207bf2f438cb008ae3ae5" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + } + }, + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "extra_modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py" + ], + "import_classifications": { + "microcosm.build.cd_benchmark.canonical": "Frozen v4 canonical origin encoding helpers; extra module bytes attested.", + "microcosm.build.cd_benchmark.origin": "Frozen v4 typed original-household keys; extra module bytes attested, protocol unchanged.", + "microcosm.build.us_runtime": "Relative native source/ACS reader imports; exact imported-symbol use fences plus declared module closures.", + "microcosm.build.us_runtime.asec_student_controls": "Existing bounded immutable member snapshot helper, attested by the composition closure.", + "microcosm.build.us_runtime.education_assistance_source": "Existing original ASEC archive and cohort pins, attested by the composition closure.", + "microcosm.build.us_runtime.graph_implementation": "Unchanged upstream identity service and AST/resource dependency classifier.", + "microcosm.build.us_runtime.graph_native_origin_implementation": "This isolated extension identity; its own module bytes and inventory are attested.", + "microcosm.build.us_runtime.support_provenance": "Existing descendant source-ID, channel and role column names; no population treatment.", + "microcosm.frame": "Frame schema, linkage, effective typed weights and population content; existing complete Frame closure.", + "microcosm.graph": "Graph declarations, typed artifact outputs and kernel contracts; existing complete graph closure.", + "microcosm.graph.artifact_edges": "Actual graph typed-descriptor validation at full materialization boundary; existing graph closure.", + "microcosm.graph.canonical": "Graph canonical JSON encoding; existing complete graph closure.", + "microcosm.graph.keys": "Actual opaque producer/output artifact-key derivation; existing complete graph closure.", + "numpy": "Installed NumPy distribution version plus platform numeric scope; strict native integer handling." + }, + "schema": "microcosm.us.native-origin-extension-inventory.v1", + "stage": "native_household_origin_v1" +} From aa9dd948784f3b75b7082d8f1b18ac2686d7e508 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 013/255] Stage integration layer 11: PLACEMENT-ADDITIONS.patch --- PROGRESS.md | 2 + .../us_runtime/graph_full_puf_enrichment.py | 805 +++++++++++++ .../test_us_graph_full_puf_enrichment.py | 1022 +++++++++++++++++ 3 files changed, 1829 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py create mode 100644 packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py diff --git a/PROGRESS.md b/PROGRESS.md index d56c3a1b2..4f1125af5 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -757,3 +757,5 @@ Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and pr Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py new file mode 100644 index 000000000..182825a83 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py @@ -0,0 +1,805 @@ +"""Bounded full65 attachment to an independently retained upstream Population. + +This is a placement owner, not source admission or a source-successor issuer. +The caller must already have qualified the complete survey predictor stages and +canonical donor. The current diagnostic survey host has only five diagnostic +features; neither its matrix nor a standalone Frame qualifies for this boundary. +No missing predictor is supplied here, and prior wages are outside this chain. + +The upstream Population must already be an inherited version in which the output +incumbents can be rewritten. These two ordinary nodes add no structural version, +weight transition, or mass record. Retain independent expected state before +execution and call the materialized verifier after cold AND required warm runs. +KernelContext alone cannot expose metadata, owners or design anchors. +""" + +from __future__ import annotations + +import hashlib +import sys +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from itertools import chain +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import ( + graph_legacy_apply_matrix, + graph_legacy_qrf, + model_input, + qrf, + qrf_target, +) +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + artifact_edges, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph import store as store_ops +from microcosm.graph.canonical import canonical_json +from microcosm.graph.kernel import ArtifactValue +from microcosm.graph.keys import node_key, opaque_artifact_key, platform_fingerprint +from microcosm.graph.population import Population + +from . import full_puf_enrichment as full +from . import survey_population_replay as replay + +PERSON_MASK = "full_puf_person_mask" +TAX_UNIT_MASK = "full_puf_tax_unit_mask" +MASKS = MappingProxyType({"person": PERSON_MASK, "tax_unit": TAX_UNIT_MASK}) +PLACEMENT_TYPE = ArtifactType("microcosm.us.full_puf_placement", 1) +SCOPE = "full65_attachment_complete_upstream_required" +_ROSTERS = (("person", full.PERSON_OUTPUTS), ("tax_unit", full.TAX_UNIT_OUTPUTS)) +_OUTPUT_COORDINATES = frozenset((e, c) for e, cols in _ROSTERS for c in cols) +require = full._require + + +def _digest(parts): + digest = hashlib.sha256() + for part in parts: + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _series_parts(series): + yield repr(series.dtype).encode() + if pd.api.types.is_object_dtype(series.dtype): + # Portable typed scalar bytes, never Python object pointers. + yield from (store_ops._encode_object_scalar(v) for v in series) + else: + yield from population_ops._storage_parts( + series, np.ones(len(series), dtype=np.bool_) + ) + + +def _axis_parts(axis): + require(not isinstance(axis, pd.MultiIndex), "FULL_PUF_AXIS") + yield type(axis).__name__.encode() + yield canonical_json(store_ops._axis_name_payload(axis.name)) + yield from _series_parts(pd.Series(axis.array, copy=False)) + + +def _table_stamp(table): + return _digest( + chain( + _axis_parts(table.index), + _axis_parts(table.columns), + (str(table.flags.allows_duplicate_labels).encode(),), + (part for column in table for part in _series_parts(table[column])), + ) + ) + + +def _population_stamp(population): + """In-process mutation seal only; never source or successor qualification.""" + require(type(population) is Population, "FULL_PUF_POPULATION_REQUIRED") + frame = population.frame + return _digest( + ( + canonical_json( + { + "schema": asdict(frame.schema), + "entities": frame.entities, + "links": frame.links, + "metadata": store_ops._encode_frame_metadata(frame.metadata), + "mass_log": [asdict(r) for r in frame.mass_log], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": tuple(population.weight_kind.items()), + "mass_ledger": [asdict(r) for r in population.mass_ledger], + } + ), + *(_table_stamp(frame.table(e)).encode() for e in frame.entities), + *_axis_parts(frame.strata.index), + canonical_json(store_ops._axis_name_payload(frame.strata.name)), + *_series_parts(frame.strata), + *( + canonical_json( + ( + e, + frame.weights_for(e).kind, + str(frame.weights_for(e).values.dtype), + frame.weights_for(e).values.shape, + ) + ) + + frame.weights_for(e).values.tobytes() + for e in frame.weighted_entities + ), + *( + canonical_json((e, str(a.dtype), a.shape)) + a.tobytes() + for e, a in population.design_weights.items() + ), + ) + ) + + +def _structural(frame, entity): + schema = frame.schema + return ( + schema.entity_id_column(entity), + *( + tuple(schema.membership_column(e) for e in schema.group_entities) + if entity == schema.person_entity + else () + ), + ) + + +def _inputs(frame, *, masks=False): + result = [] + for entity in frame.entities: + columns = tuple( + c + for c in frame.table(entity) + if c not in _structural(frame, entity) + and (not masks or (entity, c) not in _OUTPUT_COORDINATES) + ) + # Full attachment must project every table, including its structural IDs. + # The graph cannot express a Slice with zero nonstructural columns. + require(columns or masks, "FULL_PUF_EMPTY_ENTITY_PROJECTION:" + entity) + if columns: + result.append(Slice(entity, columns)) + return tuple(result) + + +def _outputs(frame): + result = [] + for entity, columns in _ROSTERS: + table = frame.table(entity) + for column in columns: + boolean = column in full.support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + dtype = "boolean" if boolean else "float64" + if column in table: + token = population_ops.token_for_dtype(table[column].dtype) + require( + token in ({"bool", "boolean"} if boolean else {"float64"}), + "FULL_PUF_INCUMBENT_DTYPE:" + column, + ) + dtype = token + result.append( + Owned( + entity, + column, + dtype, + rows=MASKS[entity], + rewrite=column in table, + ) + ) + return tuple(result) + + +def _masks(frame): + masks = { + entity: full.support.puf_tax_detail_clone_mask( + frame.table(entity), entity=entity + ) + for entity in MASKS + } + person, tax_unit = frame.table("person"), frame.table("tax_unit") + ids = tax_unit.loc[masks["tax_unit"], "tax_unit_id"] + require( + bool(masks["person"].any()) + and bool(masks["tax_unit"].any()) + and np.array_equal(masks["person"], person.person_tax_unit_id.isin(ids)) + and set(person.loc[masks["person"], "person_tax_unit_id"]) == set(ids), + "FULL_PUF_MASK_MEMBERSHIP", + ) + return masks + + +@dataclass(frozen=True) +class FullPufAttachmentBinding: + """Retained comparison inputs; possession confers no source authority.""" + + population: Population + expected_population: Population + population_node: Node + input_owners: Mapping + donor: pd.DataFrame + predictor_known: pd.DataFrame + matrix: ArtifactValue + matrix_edge: ArtifactInput + fit_nodes: tuple[Node, ...] + apply_nodes: tuple[Node, ...] + prefix: str + population_stamp: str + expected_stamp: str + donor_stamp: str + known_stamp: str + + def check(self): + require( + _population_stamp(self.population) == self.population_stamp + and _population_stamp(self.expected_population) == self.expected_stamp, + "FULL_PUF_RETAINED_POPULATION_CHANGED", + ) + require(_table_stamp(self.donor) == self.donor_stamp, "FULL_PUF_DONOR_CHANGED") + require( + _table_stamp(self.predictor_known) == self.known_stamp, + "FULL_PUF_KNOWNNESS_CHANGED", + ) + require( + dict(self.input_owners) == dict(self.expected_population.owners), + "FULL_PUF_INPUT_OWNERS", + ) + + +def _value(edge, value, *, producer_key=None): + require( + type(value) is ArtifactValue + and value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact) + and (producer_key is None or value.producer_key == producer_key) + and value.numerics.numeric in (Numeric.BITWISE, Numeric.PLATFORM_BITWISE) + and ( + value.numerics.numeric is not Numeric.PLATFORM_BITWISE + or value.numerics.platform == platform_fingerprint() + ), + "FULL_PUF_ARTIFACT_IDENTITY:" + edge.name, + ) + return value + + +def retain_full_puf_attachment( + *, + population, + expected_population, + population_node, + input_owners, + donor, + predictor_known, + matrix, + fit_nodes, + apply_nodes, + prefix="puf_full_attachment", +): + """Pin already prepared upstream values and an actual matrix producer edge. + + ``population`` is executor-observed; ``expected_population`` is a separately + retained upstream comparison value, not derived from the attachment result. + ``input_owners`` and ``population_node`` come from that upstream graph. The + integrating source owner remains responsible for qualification and for opening + a version with inherited incumbents. This function admits neither a donor nor + a survey source and does not construct an upstream Population from a Frame. + """ + require( + type(population) is type(expected_population) is Population + and population is not expected_population + and population.frame is not expected_population.frame, + "FULL_PUF_INDEPENDENT_POPULATION_REQUIRED", + ) + replay.same_replayed_population(expected_population, population) + require( + population.frame.schema == full.support.US_SCHEMA + and not population.frame.links, + "FULL_PUF_US_POPULATION_REQUIRED", + ) + require( + type(population_node) is Node + and population_node.id == population.version + and population_node.base is not None + and population_node.structural + not in (StructuralDelta.CREATE, StructuralDelta.NONE) + and dict(input_owners) == dict(population.owners), + "FULL_PUF_UPSTREAM_GRAPH_BINDING", + ) + require( + len(full.PERSON_OUTPUTS) == 56 + and len(full.TAX_UNIT_OUTPUTS) == 9 + and not {"prior_year_wages", "employment_income_last_year"} & set(full.TARGETS) + and type(fit_nodes) is type(apply_nodes) is tuple + and len(fit_nodes) == len(apply_nodes) == 65, + "FULL_PUF_CHAIN_ROSTER", + ) + first = fit_nodes[0] + matrix_edges = tuple( + e for e in apply_nodes[0].artifact_inputs if e.name == "matrix" + ) + require(len(matrix_edges) == 1, "FULL_PUF_MATRIX_EDGE") + edge = matrix_edges[0] + require( + edge.artifact == "matrix" and edge.type == model_input.RECIPIENT_MATRIX_TYPE, + "FULL_PUF_MATRIX_EDGE", + ) + expected_fits, expected_applies = full.full_puf_train_apply_nodes( + donor_population=first.population, + recipient_population=population.version, + matrix_producer=edge.producer, + seed=first.params["seed"], + n_estimators=first.params["n_estimators"], + zero_atol=first.params["zero_atol"], + prefix=first.id.removesuffix(".fit.000"), + ) + require( + fit_nodes == expected_fits and apply_nodes == expected_applies, + "FULL_PUF_CHAIN_DECLARATION", + ) + # Structural patching labels all carried columns with the version owner; + # that runtime label is not a same-version declaration conflict. The real + # compiler checks inherited rewrite claims against the version's base. + require( + not _OUTPUT_COORDINATES + & {(o.entity, o.column) for o in population_node.outputs}, + "FULL_PUF_UPSTREAM_OUTPUT_CONFLICT", + ) + for entity, name in MASKS.items(): + require( + name not in population.frame.table(entity), "FULL_PUF_MASK_ALREADY_PRESENT" + ) + _outputs(population.frame) + _inputs(population.frame) + _masks(population.frame) + prepared = full.prepare_full_puf_inputs( + population.frame, donor, predictor_known=predictor_known + ) + _value(edge, matrix) + require(matrix.payload == prepared.matrix, "FULL_PUF_PREPARED_MATRIX") + binding = FullPufAttachmentBinding( + population, + expected_population, + population_node, + MappingProxyType(dict(input_owners)), + donor, + predictor_known, + matrix, + edge, + fit_nodes, + apply_nodes, + prefix, + _population_stamp(population), + _population_stamp(expected_population), + _table_stamp(donor), + _table_stamp(predictor_known), + ) + binding.check() + return binding + + +def _params(binding): + return { + "scope": SCOPE, + "upstream_state": binding.expected_stamp, + "donor_values": binding.donor_stamp, + "predictor_knownness": binding.known_stamp, + "matrix_producer_key": binding.matrix.producer_key, + "matrix_sha256": codec.sha(binding.matrix.payload), + "source_admission_issued": False, + } + + +def _attach_edges(binding): + return ( + binding.matrix_edge, + ArtifactInput( + "placement", binding.prefix + ".mask", "placement", PLACEMENT_TYPE + ), + *( + ArtifactInput(f"raw_{i:03d}", node.id, "raw_draw", codec.RAW_TARGET_TYPE) + for i, node in enumerate(binding.apply_nodes) + ), + ArtifactInput( + "apply_state", + binding.apply_nodes[-1].id, + "apply_state", + graph_legacy_apply_matrix.MATRIX_APPLY_STATE_TYPE, + ), + ArtifactInput( + "training_state", + binding.fit_nodes[-1].id, + "training_state", + codec.TRAINING_STATE_TYPE, + ), + ArtifactInput( + "last_model", + binding.fit_nodes[-1].id, + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + ), + ) + + +def full_puf_attachment_nodes(binding): + binding.check() + frame = binding.expected_population.frame + masks = Node( + binding.prefix + ".mask", + FullPufMaskKernel.ref, + population=binding.population.version, + # Reading fields rewritten by attach would create a graph dependency cycle. + inputs=_inputs(frame, masks=True), + outputs=tuple(Owned(e, c, "bool") for e, c in MASKS.items()), + params=_params(binding), + artifact_inputs=(binding.matrix_edge,), + artifact_outputs=(ArtifactOutput("placement", PLACEMENT_TYPE),), + ) + inputs = list(_inputs(frame)) + for entity, mask in MASKS.items(): + for i, item in enumerate(inputs): + if item.entity == entity: + inputs[i] = Slice(entity, (*item.columns, mask)) + break + else: + inputs.append(Slice(entity, (mask,))) + attach = Node( + binding.prefix + ".attach", + FullPufAttachKernel.ref, + population=binding.population.version, + inputs=tuple(inputs), + outputs=_outputs(frame), + params=_params(binding), + artifact_inputs=_attach_edges(binding), + ) + return masks, attach + + +def _context_projection(context, node, frame): + """Check the expected declaration's complete projection, never actual-chosen columns.""" + require( + context.node == node and dict(context.params) == dict(node.params), + "FULL_PUF_DECLARATION", + ) + require(not context.sources, "FULL_PUF_UNDECLARED_SOURCE") + entities = {s.entity for s in node.inputs} | {o.entity for o in node.outputs} + require(set(context.tables) == entities, "FULL_PUF_CONTEXT_TABLES") + weights = {} + for entity in sorted(entities): + columns = list(_structural(frame, entity)) + columns.extend(c for s in node.inputs if s.entity == entity for c in s.columns) + columns.extend( + o.column for o in node.outputs if o.entity == entity and o.rewrite + ) + expected = frame.table(entity).loc[ + np.ones(frame.n(entity), dtype=np.bool_), list(dict.fromkeys(columns)) + ] + actual = context.tables[entity] + replay._axis(expected.index, actual.index) + replay._axis(expected.columns, actual.columns) + require(expected.flags == actual.flags, "FULL_PUF_CONTEXT_FLAGS") + for column in expected: + replay._series(expected[column], actual[column]) + try: + weights[entity] = frame.resolve_weights(entity) + except ValueError: + if entity in frame.weighted_entities: + raise + require(set(context.weights) == set(weights), "FULL_PUF_CONTEXT_WEIGHTS") + for entity, expected in weights.items(): + actual = context.weights[entity] + require( + actual.kind is expected.kind + and replay._array_bytes_equal(expected.values, actual.values), + "FULL_PUF_CONTEXT_WEIGHT_VALUES", + ) + strata = frame.strata.loc[np.ones(len(frame.strata), dtype=np.bool_)] + replay._axis(strata.index, context.strata.index) + require( + replay._name_bytes(strata.name) == replay._name_bytes(context.strata.name), + "FULL_PUF_CONTEXT_STRATA_NAME", + ) + replay._series(strata, context.strata) + + +def _mask_result(binding): + frame = binding.expected_population.frame + return KernelResult( + columns={ + (entity, MASKS[entity]): pd.Series( + mask, + index=frame.table(entity)[frame.schema.entity_id_column(entity)], + dtype=bool, + ) + for entity, mask in _masks(frame).items() + } + ) + + +def _placement(binding): + frame = binding.expected_population.frame + return codec.encode_json( + { + **_params(binding), + "target_order": list(full.TARGETS), + "selected_ids": { + entity: frame.table(entity) + .loc[mask, frame.schema.entity_id_column(entity)] + .tolist() + for entity, mask in _masks(frame).items() + }, + "release_eligible": False, + } + ) + + +def _checked_artifacts(binding, edges, artifacts, producer_keys=None): + require(set(artifacts) == {e.name for e in edges}, "FULL_PUF_ARTIFACT_ROSTER") + values = {} + for edge in edges: + if producer_keys is not None: + require(edge.producer in producer_keys, "FULL_PUF_PRODUCER_ROSTER") + values[edge.name] = _value( + edge, + artifacts[edge.name], + producer_key=None + if producer_keys is None + else producer_keys[edge.producer], + ) + require(values["matrix"] == binding.matrix, "FULL_PUF_MATRIX_PIN") + if "placement" in values: + require( + values["placement"].payload == _placement(binding), + "FULL_PUF_PLACEMENT_BINDING", + ) + for names in (("raw_064", "apply_state"), ("last_model", "training_state")): + require( + len({values[name].producer_key for name in names}) == 1, + "FULL_PUF_SIBLING_PRODUCER", + ) + return values + + +def _finalized_columns(binding, values): + frame = binding.expected_population.frame + finalized, evidence = full.finalize_full_puf( + frame, + binding.donor, + predictor_known=binding.predictor_known, + matrix=values["matrix"].payload, + matrix_producer_key=values["matrix"].producer_key, + raw_draws={ + target: values[f"raw_{i:03d}"].payload + for i, target in enumerate(full.TARGETS) + }, + apply_state=values["apply_state"].payload, + training_state=values["training_state"].payload, + last_model=values["last_model"].payload, + seed=binding.fit_nodes[0].params["seed"], + ) + masks, columns = _masks(frame), {} + for owned in _outputs(frame): + table, mask = finalized.table(owned.entity), masks[owned.entity] + selected = table.loc[mask, owned.column] + require(selected.notna().all(), "FULL_PUF_FINALIZED_UNKNOWN:" + owned.column) + require( + np.isfinite(selected.to_numpy(dtype=np.float64)).all(), + "FULL_PUF_FINALIZED_NONFINITE:" + owned.column, + ) + # Cast only owned values: an incumbent physical bool must remain bool. + columns[(owned.entity, owned.column)] = pd.Series( + selected.astype(owned.dtype).array, + index=table.loc[mask, frame.schema.entity_id_column(owned.entity)], + dtype=owned.dtype, + ) + binding.check() + return KernelResult(columns=columns, receipt=evidence) + + +class _FullPufKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, binding): + self.binding = binding + + def implementation_hash(self): + # Independent placement closure; never refresh a source owner's inventory. + return source_hash( + sys.modules[__name__], + full, + full.support, + replay, + population_ops, + store_ops, + artifact_edges, + codec, + model_input, + qrf, + qrf_target, + graph_legacy_qrf, + graph_legacy_apply_matrix, + dependencies=self.capabilities.dependencies, + ) + + +class FullPufMaskKernel(_FullPufKernel): + ref = "us.full_puf.mask@1" + + def run(self, context): + binding = self.binding + node, _ = full_puf_attachment_nodes(binding) + _context_projection(context, node, binding.expected_population.frame) + _checked_artifacts(binding, node.artifact_inputs, context.artifacts) + return KernelResult( + columns=_mask_result(binding).columns, + artifacts={"placement": _placement(binding)}, + receipt={"scope": SCOPE, "source_admission_issued": False}, + ) + + +class FullPufAttachKernel(_FullPufKernel): + ref = "us.full_puf.attach@1" + + def run(self, context): + binding = self.binding + mask, attach = full_puf_attachment_nodes(binding) + expected_masked = population_ops.patch( + binding.expected_population, mask, _mask_result(binding) + ) + _context_projection(context, attach, expected_masked.frame) + values = _checked_artifacts(binding, attach.artifact_inputs, context.artifacts) + return _finalized_columns(binding, values) + + +def verify_materialized_full_puf_attachment( + binding, *, upstream_population, population, artifacts, producer_keys +): + """Mandatory independent complete Population comparison, cold and replay. + + Pass actual executor-observed Populations and actual loaded typed artifacts; + ``producer_keys`` must come from the same run manifest, not payload labels. + ``load_full_puf_attachment_artifacts`` performs the manifest/store descriptor + checks. These comparisons issue no source, successor, or release evidence. + """ + binding.check() + replay.same_replayed_population(binding.expected_population, upstream_population) + mask, attach = full_puf_attachment_nodes(binding) + values = _checked_artifacts( + binding, attach.artifact_inputs, artifacts, producer_keys + ) + result = _finalized_columns(binding, values) + expected = population_ops.patch( + binding.expected_population, mask, _mask_result(binding) + ) + expected = population_ops.patch(expected, attach, result) + replay.same_replayed_population(expected, population) + binding.check() + return {**result.receipt, "scope": SCOPE, "complete_population_compared": True} + + +def _manifest_contract(binding, compiled, manifest): + """Bind declarations, real kernel implementations and all 130 chain nodes.""" + mask, attach = full_puf_attachment_nodes(binding) + require( + compiled.graph.node(binding.population_node.id) == binding.population_node, + "FULL_PUF_MANIFEST_UPSTREAM_DECLARATION", + ) + kernels = ( + LegacyQRFTrainKernel(), + graph_legacy_apply_matrix.LegacyQRFApplyMatrixKernel(), + FullPufMaskKernel(binding), + FullPufAttachKernel(binding), + ) + contracts = {k.ref: (k.capabilities, k.implementation_hash()) for k in kernels} + keys = {name: record.key for name, record in manifest.nodes.items()} + for node in (*binding.fit_nodes, *binding.apply_nodes, mask, attach): + require(compiled.graph.node(node.id) == node, "FULL_PUF_MANIFEST_DECLARATION") + record = manifest.node(node.id) + capabilities, implementation = contracts[node.kernel] + require( + record.kernel_ref == node.kernel + and record.capabilities == capabilities + and record.kernel_impl_hash == implementation + and record.key + == node_key( + compiled, + node.id, + keys, + implementation, + {}, + kernel_capabilities=capabilities, + ), + "FULL_PUF_MANIFEST_KERNEL:" + node.id, + ) + expected_outputs = { + output.name: artifact_edges.descriptor( + producer=node.id, + artifact=output.name, + type_=output.type, + producer_key=record.key, + capabilities=capabilities, + ) + for output in node.artifact_outputs + } + require( + record.typed_artifacts["outputs"] == expected_outputs + and dict(record.opaque_artifacts) + == {name: d["key"] for name, d in expected_outputs.items()}, + "FULL_PUF_MANIFEST_OUTPUTS:" + node.id, + ) + expected_inputs = {} + for edge in node.artifact_inputs: + producer = manifest.node(edge.producer) + descriptor = artifact_edges.descriptor( + producer=edge.producer, + artifact=edge.artifact, + type_=edge.type, + producer_key=producer.key, + capabilities=producer.capabilities, + ) + require( + producer.typed_artifacts["outputs"][edge.artifact] == descriptor, + "FULL_PUF_MANIFEST_ANCESTRY:" + edge.name, + ) + expected_inputs[edge.name] = descriptor + require( + record.typed_artifacts["inputs"] == expected_inputs, + "FULL_PUF_MANIFEST_INPUTS:" + node.id, + ) + + +def load_full_puf_attachment_artifacts(binding, *, compiled, manifest, store): + """Load real producer bytes after verifying the graph and both ends of edges. + + ``compiled`` is the actual graph used in this run. Recompute full65 node keys + from its exact declarations and maintained kernel identities before loading + trusted model bytes. Upstream source qualification remains the caller's duty. + """ + _manifest_contract(binding, compiled, manifest) + _, attach = full_puf_attachment_nodes(binding) + consumer = manifest.node(attach.id) + require(consumer.kernel_ref == attach.kernel, "FULL_PUF_MANIFEST_CONSUMER") + descriptors = consumer.typed_artifacts["inputs"] + require( + set(descriptors) == {e.name for e in attach.artifact_inputs}, + "FULL_PUF_MANIFEST_ROSTER", + ) + artifacts, keys = {}, {} + for edge in attach.artifact_inputs: + record = manifest.node(edge.producer) + descriptor = record.typed_artifacts["outputs"][edge.artifact] + require( + descriptor == descriptors[edge.name] + and descriptor["producer"] == edge.producer + and descriptor["artifact"] == edge.artifact + and descriptor["producer_key"] == record.key + and descriptor["key"] + == record.opaque_artifacts[edge.artifact] + == opaque_artifact_key(record.key, edge.artifact), + "FULL_PUF_MANIFEST_EDGE:" + edge.name, + ) + value = artifact_edges.value_from_descriptor( + store.load_bytes(descriptor["key"]), descriptor + ) + artifacts[edge.name] = _value(edge, value, producer_key=record.key) + keys[edge.producer] = record.key + _checked_artifacts(binding, attach.artifact_inputs, artifacts, keys) + return artifacts, keys diff --git a/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py b/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py new file mode 100644 index 000000000..3f0e3f9b8 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py @@ -0,0 +1,1022 @@ +"""Invented complete upstream control, real QRF/store/attachment, no admission. + +These ordinary fixtures import maintained helpers normally. They do not extract +functions, impersonate source issuers, use genuine data, or load a tax engine. +The current diagnostic survey host does not supply this complete predictor +surface; source-qualified production integration remains an upstream duty. +""" + +from copy import deepcopy +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import ( + InventedMatrix, + InventedSource, + _donor, + _known, + _recipient, +) + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.artifact_edges import descriptor, value_from_descriptor +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.errors import NodeRejectedError +from microcosm.graph.keys import opaque_artifact_key +from microcosm.graph.population import dtype_for_token + + +def _copy_population(population, *, metadata=None, weights=None, mass_log=None): + """Retain a separate comparison object, including all receiving context.""" + frame = population.frame + return replace( + population, + frame=Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + weights + if weights is not None + else { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + metadata=deepcopy(dict(frame.metadata)) if metadata is None else metadata, + mass_log=tuple(frame.mass_log) if mass_log is None else mass_log, + ), + owners=dict(population.owners), + weight_kind=dict(population.weight_kind), + design_weights={ + key: value.copy() for key, value in population.design_weights.items() + }, + ) + + +class InventedCompleteBoundary(KernelBase): + """Real identity FILTER opening a version in which incumbents may rewrite.""" + + ref = "test.full_puf.complete_upstream_boundary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.PLATFORM_BITWISE, + ) + + def run(self, context): + return KernelResult( + keep=pd.Series( + True, + index=pd.Index(context.tables["person"].person_id, name="person_id"), + dtype=bool, + ) + ) + + +def _complete_frame(): + frame = _recipient() + people = frame.table("person") + people["self_employment_income_would_be_qualified"] = np.resize( + np.array([False, True], dtype=bool), len(people) + ) + people["rental_income_would_be_qualified"] = pd.array( + np.resize(np.array([None, True], dtype=object), len(people)), + dtype="boolean", + ) + # Exercise preservation of unrelated columns, nullable masks, signed zero, + # complete nested metadata, and the fixture's native ACS structural blanks. + for entity in frame.entities: + table = frame.table(entity) + # The invented source declares graph-native string storage explicitly. + for name in table.select_dtypes(include="object"): + assert table[name].dropna().map(lambda value: isinstance(value, str)).all() + table[name] = table[name].astype(pd.StringDtype(storage="python")) + prefix = "" if entity == "person" else entity + "_" + table[prefix + "unrelated_signed"] = np.resize( + np.array([-0.0, 13.0], dtype="float64"), len(table) + ) + table[prefix + "unrelated_nullable"] = pd.array( + np.resize(np.array([True, None], dtype=object), len(table)), dtype="boolean" + ) + return Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + # CREATE retains this separate invented design-weight vector as + # a real Population anchor while household importance weights keep + # the helper's assembly/clone mass contract unchanged. + "person": Weights( + frame.resolve_weights("person").values.copy(), WeightKind.DESIGN + ), + **{ + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + }, + frame.strata.copy(deep=True), + metadata={ + **deepcopy(dict(frame.metadata)), + "complete_upstream_control": { + "nested": [None, {"kept": True}], + "zero": -0.0, + }, + "source_admission": False, + }, + mass_log=frame.mass_log, + ) + + +def _load_artifact(manifest, store, edge): + """Read the actual typed producer descriptor and its stored payload.""" + record = manifest.node(edge.producer) + actual_descriptor = record.typed_artifacts["outputs"][edge.artifact] + assert actual_descriptor == descriptor( + producer=edge.producer, + artifact=edge.artifact, + type_=edge.type, + producer_key=record.key, + capabilities=record.capabilities, + ) + key = opaque_artifact_key(record.key, edge.artifact) + assert record.opaque_artifacts[edge.artifact] == key + return value_from_descriptor(store.load_bytes(key), actual_descriptor) + + +def _independent_expected(upstream, donor, known, nodes, artifacts): + """Call the actual finalizer, without reusing the attachment's result builder.""" + candidate, receipt = full.finalize_full_puf( + upstream.frame, + donor, + predictor_known=known, + matrix=artifacts["matrix"].payload, + matrix_producer_key=artifacts["matrix"].producer_key, + raw_draws={ + target: artifacts[f"raw_{index:03d}"].payload + for index, target in enumerate(full.TARGETS) + }, + apply_state=artifacts["apply_state"].payload, + training_state=artifacts["training_state"].payload, + last_model=artifacts["last_model"].payload, + seed=578, + ) + mask_node, attach_node = nodes + masks = { + entity: support.puf_tax_detail_clone_mask( + upstream.frame.table(entity), entity=entity + ) + for entity in ("person", "tax_unit") + } + expected = population_ops.patch( + upstream, + mask_node, + KernelResult( + columns={ + (entity, placement.MASKS[entity]): pd.Series( + masks[entity], + index=pd.Index( + upstream.frame.table(entity)[ + upstream.frame.schema.entity_id_column(entity) + ], + name=upstream.frame.schema.entity_id_column(entity), + ), + dtype=bool, + ) + for entity in masks + } + ), + ) + columns = {} + for owned in attach_node.outputs: + table = candidate.table(owned.entity) + mask = masks[owned.entity] + id_column = candidate.schema.entity_id_column(owned.entity) + columns[(owned.entity, owned.column)] = pd.Series( + table.loc[mask, owned.column].array, + index=pd.Index(table.loc[mask, id_column], name=id_column), + dtype=dtype_for_token(owned.dtype), + ) + return population_ops.patch( + expected, attach_node, KernelResult(columns=columns) + ), receipt + + +@pytest.fixture(scope="module") +def attached_full65(tmp_path_factory): + root = tmp_path_factory.mktemp("full65_attachment") + frame, donor = _complete_frame(), _donor() + known = _known(frame) + prepared = full.prepare_full_puf_inputs(frame, donor, predictor_known=known) + decoded = model_input.decode_recipient_matrix(prepared.matrix) + table = decoded.features.copy() + table.insert(0, "tax_unit_id", decoded.entity_ids) + matrix_frame = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": decoded.entity_ids, + "person_tax_unit_id": decoded.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.ones(len(table), dtype="float64"), WeightKind.DESIGN)}, + ) + retained_store = ContentStore( + root / "retained", codecs={"frame-store": load_frame_store} + ) + sources, source_nodes = {}, [] + for name, source_frame in ( + ("survey", frame), + ("donor", prepared.donor_frame), + ("matrix_input", matrix_frame), + ): + source_name = "invented." + name + sources[source_name] = retained_store.put_frame( + full.codec.sha(("ordinary-full65-attachment:" + name).encode()), + source_frame, + ) + outputs = ( + frame_column_declarations(source_frame) + if name == "survey" + else tuple( + Owned("tax_unit", column, "float64") + for column in source_frame.table("tax_unit") + if column != "tax_unit_id" + ) + ) + source_nodes.append( + Node( + name, + InventedSource.ref, + sources=(source_name,), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + ) + boundary = Node( + "complete_upstream", + InventedCompleteBoundary.ref, + base="survey", + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + ) + matrix_node = Node( + "matrix", + InventedMatrix.ref, + population="matrix_input", + inputs=(Slice("tax_unit", full.PREDICTORS),), + artifact_outputs=(ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),), + ) + base_graph = Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*source_nodes, boundary, matrix_node), + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + observed = {} + retained_manifest = run_graph( + compile_graph(base_graph), + sources=sources, + store=retained_store, + kernels=kernels, + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + upstream = observed[boundary.id] + independent_upstream = _copy_population(upstream) + matrix = _load_artifact( + retained_manifest, + retained_store, + ArtifactInput( + "matrix", matrix_node.id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + assert matrix.payload == prepared.matrix + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population=boundary.id, + matrix_producer=matrix_node.id, + seed=578, + n_estimators=2, + zero_atol=0, + ) + retention_arguments = dict( + population=upstream, + expected_population=independent_upstream, + population_node=boundary, + input_owners=dict(upstream.owners), + donor=donor, + predictor_known=known, + matrix=matrix, + fit_nodes=fits, + apply_nodes=applies, + ) + binding = placement.retain_full_puf_attachment(**retention_arguments) + attachment_nodes = placement.full_puf_attachment_nodes(binding) + kernels.register(placement.FullPufMaskKernel(binding)) + kernels.register(placement.FullPufAttachKernel(binding)) + compiled = compile_graph( + replace( + base_graph, nodes=(*base_graph.nodes, *fits, *applies, *attachment_nodes) + ) + ) + store = ContentStore(root / "application", codecs={"frame-store": load_frame_store}) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=lambda name, value, target=observed: ( + target.__setitem__(name, value) + ), + ) + assert all( + record.hit is (resume == "require") for record in manifest.nodes.values() + ) + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + binding, compiled=compiled, manifest=manifest, store=store + ) + assert artifacts == { + edge.name: _load_artifact(manifest, store, edge) + for edge in attachment_nodes[-1].artifact_inputs + } + actual_upstream = observed[boundary.id] + actual = observed[attachment_nodes[-1].id] + evidence = placement.verify_materialized_full_puf_attachment( + binding, + upstream_population=actual_upstream, + population=actual, + artifacts=artifacts, + producer_keys=producer_keys, + ) + expected, finalizer_receipt = _independent_expected( + independent_upstream, donor, known, attachment_nodes, artifacts + ) + replay.same_replayed_population(expected, actual) + # Also exercise actual Frame storage, beyond warm column-artifact replay. + frame_key = full.codec.sha( + ("materialized:" + manifest.node(attachment_nodes[-1].id).key).encode() + ) + store.put_frame(frame_key, actual.frame) + replay.same_replayed_frame(expected.frame, store.load_frame(frame_key)) + results.append( + SimpleNamespace( + manifest=manifest, + upstream=_copy_population(actual_upstream), + population=_copy_population(actual), + artifacts=artifacts, + producer_keys=producer_keys, + evidence=evidence, + finalizer_receipt=finalizer_receipt, + ) + ) + replay.same_replayed_population(results[0].population, results[1].population) + assert results[0].artifacts == results[1].artifacts + assert results[0].manifest.key == results[1].manifest.key + return SimpleNamespace( + binding=binding, + retention_arguments=retention_arguments, + nodes=attachment_nodes, + fits=fits, + applies=applies, + results=results, + compiled=compiled, + sources=sources, + store=store, + kernels=kernels, + ) + + +def _verify(case, result, **changes): + return placement.verify_materialized_full_puf_attachment( + changes.pop("binding", case.binding), + **{ + "upstream_population": result.upstream, + "population": result.population, + "artifacts": result.artifacts, + "producer_keys": result.producer_keys, + **changes, + }, + ) + + +def test_full65_cold_and_required_replay_preserve_complete_population(attached_full65): + case = attached_full65 + mask_node, attach_node = case.nodes + assert len(case.fits) == len(case.applies) == 65 + assert len(full.PERSON_OUTPUTS) == 56 and len(full.TAX_UNIT_OUTPUTS) == 9 + assert "prior_year_wages" not in full.TARGETS + assert "employment_income_last_year" not in full.TARGETS + assert tuple(owned.column for owned in attach_node.outputs) == full.TARGETS + assert tuple( + (owned.entity, owned.column, owned.dtype) for owned in mask_node.outputs + ) == ( + ("person", placement.PERSON_MASK, "bool"), + ("tax_unit", placement.TAX_UNIT_MASK, "bool"), + ) + for result in case.results: + assert result.evidence["source_admission_issued"] is False + assert result.evidence["release_eligible"] is False + assert result.finalizer_receipt["target_order"] == list(full.TARGETS) + assert result.finalizer_receipt["tail_bounds"] + frame, upstream = result.population.frame, result.upstream.frame + assert result.population.mass_ledger == result.upstream.mass_ledger + assert frame.mass_log == upstream.mass_log + expected_new = { + ("person", placement.PERSON_MASK), + ("tax_unit", placement.TAX_UNIT_MASK), + *(("person", name) for name in full.PERSON_OUTPUTS), + *(("tax_unit", name) for name in full.TAX_UNIT_OUTPUTS), + } + for cell, owner in result.upstream.owners.items(): + if cell not in expected_new: + assert result.population.owners[cell] == owner + for owned in attach_node.outputs: + table, before = frame.table(owned.entity), upstream.table(owned.entity) + mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) + assert owned.rows == placement.MASKS[owned.entity] + assert table[owned.column].dtype == dtype_for_token(owned.dtype) + assert table.loc[mask, owned.column].notna().all() + assert ( + result.population.owners[(owned.entity, owned.column)] == attach_node.id + ) + if owned.column in before: + assert owned.rewrite is True + assert population_ops.storage_equal( + before[owned.column], table[owned.column], ~mask + ) + else: + assert owned.rewrite is False + assert table.loc[~mask, owned.column].isna().all() + np.testing.assert_array_equal(table[owned.rows], mask) + assert frame.table("person").is_full_time_college_student.dtype == np.dtype( + "bool" + ) + assert frame.table( + "person" + ).self_employment_income_would_be_qualified.dtype == np.dtype("bool") + assert ( + frame.table("person").rental_income_would_be_qualified.dtype + == pd.BooleanDtype() + ) + assert frame.table("person").business_is_sstb.dtype == pd.BooleanDtype() + people = frame.table("person") + children = people[placement.PERSON_MASK] & people.age.lt(15) + assert people.loc[children, "employment_income_before_lsr"].eq(0).all() + tax_mask = frame.table("tax_unit")[placement.TAX_UNIT_MASK].to_numpy() + assert np.any(upstream.resolve_weights("tax_unit").values[tax_mask] == 0) + decoded = model_input.decode_recipient_matrix( + result.artifacts["matrix"].payload + ) + assert len(decoded.features) == int(tax_mask.sum()) + + +@pytest.mark.parametrize( + "mutation", + ( + "raw_values", + "matrix_values", + "model_bytes", + "training_history", + "apply_history", + "raw_roster", + "raw_key", + "wrong_type", + "apply_sibling", + "model_sibling", + "producer_key", + ), +) +def test_actual_artifact_value_identity_and_sibling_mutations_are_refused( + attached_full65, mutation +): + case, result = attached_full65, attached_full65.results[-1] + artifacts = dict(result.artifacts) + producer_keys = dict(result.producer_keys) + matrix = model_input.decode_recipient_matrix(artifacts["matrix"].payload) + if mutation == "raw_values": + raw = full.codec.read_raw_target( + artifacts["raw_000"].payload, + target=full.TARGETS[0], + index=matrix.features.index, + ).copy() + raw[0] += 1.0 + artifacts["raw_000"] = replace( + artifacts["raw_000"], + payload=full.codec.encode_raw_target( + raw, target=full.TARGETS[0], index=matrix.features.index + ), + ) + elif mutation == "matrix_values": + features = matrix.features.copy(deep=True) + features.iloc[0, 2] += 1.0 + artifacts["matrix"] = replace( + artifacts["matrix"], + payload=model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=matrix.entity_ids + ), + ) + elif mutation == "model_bytes": + artifacts["last_model"] = replace( + artifacts["last_model"], + payload=artifacts["last_model"].payload + b"changed", + ) + elif mutation == "training_history": + changed = full.codec.decode_json(artifacts["training_state"].payload) + changed["models"][0]["sha256"] = "e" * 64 + artifacts["training_state"] = replace( + artifacts["training_state"], payload=full.codec.encode_json(changed) + ) + elif mutation == "apply_history": + changed = full.codec.decode_json(artifacts["apply_state"].payload) + changed["application"]["raw_targets"][0]["sha256"] = "e" * 64 + artifacts["apply_state"] = replace( + artifacts["apply_state"], payload=full.codec.encode_json(changed) + ) + elif mutation == "raw_roster": + artifacts.pop("raw_064") + elif mutation == "raw_key": + artifacts["raw_000"] = replace( + artifacts["raw_000"], key=artifacts["raw_001"].key + ) + elif mutation == "wrong_type": + artifacts["last_model"] = replace( + artifacts["last_model"], type=ArtifactType("test.unrelated", 1) + ) + elif mutation in ("apply_sibling", "model_sibling"): + alias = "apply_state" if mutation == "apply_sibling" else "training_state" + artifacts[alias] = replace( + artifacts[alias], + producer_key="f" * 64, + key=opaque_artifact_key("f" * 64, alias), + ) + # Even a matching altered node-key map cannot pair this with the actual + # last raw draw/model sibling, whose actual producer key still differs. + node_id = ( + case.applies[-1].id if mutation == "apply_sibling" else case.fits[-1].id + ) + producer_keys[node_id] = "f" * 64 + elif mutation == "producer_key": + producer_keys[case.applies[0].id] = "e" * 64 + with pytest.raises(ValueError): + _verify(case, result, artifacts=artifacts, producer_keys=producer_keys) + + +@pytest.mark.parametrize( + "mutation", + ( + "metadata", + "weights", + "design_anchor", + "output_owner", + "unrelated_owner", + "strata", + "membership", + "ids", + "unrelated_value", + "native_output", + "clone_output", + "person_mask", + "tax_unit_mask", + "mass_ledger", + "frame_mass_log", + ), +) +def test_complete_materialized_population_mutations_are_refused( + attached_full65, mutation +): + case, result = attached_full65, attached_full65.results[-1] + changed = _copy_population(result.population) + people = changed.frame.table("person") + if mutation == "metadata": + changed = _copy_population( + changed, + metadata={ + **dict(changed.frame.metadata), + "unexpected": {"nested": [False]}, + }, + ) + elif mutation == "weights": + weights = { + entity: Weights( + changed.frame.weights_for(entity).values.copy(), + changed.frame.weights_for(entity).kind, + ) + for entity in changed.frame.weighted_entities + } + entity = changed.frame.weighted_entities[0] + values = weights[entity].values.copy() + values[0] += 0.25 + weights[entity] = Weights(values, weights[entity].kind) + changed = _copy_population(changed, weights=weights) + elif mutation == "design_anchor": + design = { + entity: values.copy() for entity, values in changed.design_weights.items() + } + assert design + design[next(iter(design))][0] += 0.25 + changed = replace(changed, design_weights=design) + elif mutation in ("output_owner", "unrelated_owner"): + column = ( + full.PERSON_OUTPUTS[0] if mutation == "output_owner" else "unrelated_signed" + ) + changed = replace( + changed, owners={**changed.owners, ("person", column): "different.owner"} + ) + elif mutation == "strata": + changed.frame.strata.name = "changed_strata_name" + elif mutation == "membership": + people.loc[people.index[0], "person_tax_unit_id"] = ( + people.person_tax_unit_id.iloc[-1] + ) + elif mutation == "ids": + people.loc[people.index[0], "person_id"] += 1_000_000 + elif mutation == "unrelated_value": + people.loc[people.index[0], "unrelated_signed"] = ( + 0.0 # Signed-zero bits changed. + ) + elif mutation in ("native_output", "clone_output"): + mask = people[placement.PERSON_MASK].to_numpy() + selected = ~mask if mutation == "native_output" else mask + people.loc[people.index[selected][0], "employment_income_before_lsr"] += 1.0 + elif mutation in ("person_mask", "tax_unit_mask"): + entity = "person" if mutation == "person_mask" else "tax_unit" + table = changed.frame.table(entity) + table.loc[table.index[0], placement.MASKS[entity]] = ~table.loc[ + table.index[0], placement.MASKS[entity] + ] + elif mutation == "mass_ledger": + assert changed.mass_ledger + changed = replace(changed, mass_ledger=changed.mass_ledger[:-1]) + elif mutation == "frame_mass_log": + assert changed.frame.mass_log + changed = _copy_population(changed, mass_log=changed.frame.mass_log[:-1]) + with pytest.raises(ValueError): + _verify(case, result, population=changed) + + +@pytest.mark.parametrize( + "mutation", + ("unrelated_value", "column_missing", "column_order", "owner", "metadata"), +) +def test_upstream_replay_checks_full_state_beyond_matrix_values( + attached_full65, mutation +): + case, result = attached_full65, attached_full65.results[-1] + changed = _copy_population(result.upstream) + people = changed.frame.table("person") + if mutation == "unrelated_value": + people.loc[people.index[0], "unrelated_signed"] = 9.0 + elif mutation == "column_missing": + people.drop(columns="unrelated_signed", inplace=True) + elif mutation == "column_order": + values = people.pop("unrelated_signed") + people.insert(0, "unrelated_signed", values) + elif mutation == "owner": + changed = replace( + changed, + owners={**changed.owners, ("person", "age"): "different.upstream.owner"}, + ) + elif mutation == "metadata": + changed = _copy_population(changed, metadata={"matrix_values_unchanged": True}) + # The actual model matrix is unaffected by every mutation in this group. + assert ( + full.prepare_full_puf_inputs( + changed.frame, + case.retention_arguments["donor"], + predictor_known=case.retention_arguments["predictor_known"], + ).matrix + == result.artifacts["matrix"].payload + ) + with pytest.raises(ValueError): + _verify(case, result, upstream_population=changed) + + +@pytest.mark.parametrize( + "mutation", + ( + "donor", + "knownness", + "population", + "expected_population", + "expected_null_backing", + "owners", + ), +) +def test_retained_live_inputs_cannot_change_after_binding(attached_full65, mutation): + case, result = attached_full65, attached_full65.results[-1] + arguments = { + **case.retention_arguments, + "population": _copy_population(case.retention_arguments["population"]), + "expected_population": _copy_population( + case.retention_arguments["expected_population"] + ), + "donor": case.retention_arguments["donor"].copy(deep=True), + "predictor_known": case.retention_arguments["predictor_known"].copy(deep=True), + "input_owners": dict(case.retention_arguments["input_owners"]), + } + binding = placement.retain_full_puf_attachment(**arguments) + if mutation == "donor": + arguments["donor"].loc[arguments["donor"].index[0], "casualty_loss"] += 1.0 + elif mutation == "knownness": + arguments["predictor_known"].iloc[0, 0] = False + elif mutation == "population": + table = arguments["population"].frame.table("person") + table.loc[table.index[0], "unrelated_signed"] += 1.0 + elif mutation == "expected_population": + table = arguments["expected_population"].frame.table("person") + table.loc[table.index[0], "unrelated_signed"] += 1.0 + elif mutation == "expected_null_backing": + values = ( + arguments["expected_population"] + .frame.table("person")["unrelated_nullable"] + .array + ) + position = np.flatnonzero(values._mask)[0] + values._data[position] = ~values._data[position] + elif mutation == "owners": + binding = replace( + binding, + input_owners={**binding.input_owners, ("person", "age"): "different.owner"}, + ) + with pytest.raises(ValueError): + _verify(case, result, binding=binding) + + +@pytest.mark.parametrize( + "mutation", + ( + "short_chain", + "fit_seed", + "apply_matrix", + "create_population", + "incomplete_owner_map", + ), +) +def test_boundary_requires_complete_chain_and_explicit_population_owner_binding( + attached_full65, mutation +): + arguments = dict(attached_full65.retention_arguments) + if mutation == "short_chain": + arguments["fit_nodes"] = arguments["fit_nodes"][:-1] + arguments["apply_nodes"] = arguments["apply_nodes"][:-1] + elif mutation == "fit_seed": + fits = arguments["fit_nodes"] + arguments["fit_nodes"] = ( + replace(fits[0], params={**fits[0].params, "seed": 579}), + *fits[1:], + ) + elif mutation == "apply_matrix": + applies = arguments["apply_nodes"] + edges = tuple( + replace(edge, producer="different.matrix") + if edge.name == "matrix" + else edge + for edge in applies[0].artifact_inputs + ) + arguments["apply_nodes"] = ( + replace(applies[0], artifact_inputs=edges), + *applies[1:], + ) + elif mutation == "create_population": + arguments["population_node"] = Node( + "complete_upstream", + InventedSource.ref, + structural=StructuralDelta.CREATE, + sources=("invented.survey",), + outputs=(Owned("person", "age", "float64"),), + ) + elif mutation == "incomplete_owner_map": + arguments["input_owners"] = dict(arguments["input_owners"]) + arguments["input_owners"].pop(("person", "age")) + with pytest.raises(ValueError): + placement.retain_full_puf_attachment(**arguments) + + +def test_actual_graph_refuses_omitted_unrelated_attachment_input(attached_full65): + case = attached_full65 + attach = case.nodes[-1] + changed = replace( + attach, + inputs=tuple( + replace( + slice_, + columns=tuple( + name for name in slice_.columns if name != "unrelated_signed" + ), + ) + if slice_.entity == "person" + else slice_ + for slice_ in attach.inputs + ), + ) + assert changed.inputs != attach.inputs + compiled = compile_graph( + replace( + case.compiled.graph, + nodes=tuple( + changed if node.id == attach.id else node + for node in case.compiled.graph.nodes + ), + ) + ) + with pytest.raises(NodeRejectedError, match="FULL_PUF_DECLARATION"): + run_graph( + compiled, sources=case.sources, store=case.store, kernels=case.kernels + ) + + +@pytest.mark.parametrize( + "mutation", + ("unknown_predictor", "monetary_float32", "numeric_boolean", "incomplete_matrix"), +) +def test_complete_input_knownness_and_incumbent_physical_dtypes_are_required( + attached_full65, mutation +): + case = attached_full65 + arguments = { + **case.retention_arguments, + "population": _copy_population(case.retention_arguments["population"]), + "predictor_known": case.retention_arguments["predictor_known"].copy(deep=True), + } + frame = arguments["population"].frame + if mutation == "unknown_predictor": + arguments["predictor_known"].iloc[0, 0] = False + elif mutation == "monetary_float32": + table = frame.table("person") + table["employment_income_before_lsr"] = ( + table.employment_income_before_lsr.astype("float32") + ) + elif mutation == "numeric_boolean": + table = frame.table("person") + table["self_employment_income_would_be_qualified"] = ( + table.self_employment_income_would_be_qualified.astype("float64") + ) + elif mutation == "incomplete_matrix": + value = arguments["matrix"] + decoded = model_input.decode_recipient_matrix(value.payload) + arguments["matrix"] = replace( + value, + payload=model_input.encode_recipient_matrix( + decoded.features.iloc[:, :-1], + entity="tax_unit", + entity_ids=decoded.entity_ids, + ), + ) + arguments["expected_population"] = _copy_population(arguments["population"]) + with pytest.raises(ValueError): + placement.retain_full_puf_attachment(**arguments) + + +@pytest.mark.parametrize( + "mutation", + ( + "kernel_ref", + "kernel_impl", + "producer_key", + "producer_descriptor_type", + "consumer_descriptor_type", + "compiled_fit_params", + "compiled_apply_edge", + ), +) +def test_actual_manifest_and_compiled_producer_contract_mutations_are_refused( + attached_full65, mutation +): + case, result = attached_full65, attached_full65.results[-1] + manifest, compiled = result.manifest, case.compiled + records = dict(manifest.nodes) + node_id = case.fits[-1].id + record = records[node_id] + # Constructors may themselves reject inconsistent typed ancestry. In the + # cases which remain structurally valid, the actual loader must reject the + # changed implementation/declaration against the retained complete chain. + with pytest.raises(ValueError): + if mutation == "kernel_ref": + records[node_id] = replace(record, kernel_ref="test.different.fit@1") + elif mutation == "kernel_impl": + records[node_id] = replace(record, kernel_impl_hash="e" * 64) + elif mutation == "producer_key": + records[node_id] = replace(record, key="e" * 64) + elif mutation in ("producer_descriptor_type", "consumer_descriptor_type"): + if mutation == "consumer_descriptor_type": + node_id = case.nodes[-1].id + record = records[node_id] + typed = { + direction: { + name: {**value, "type": dict(value["type"])} + for name, value in entries.items() + } + for direction, entries in record.typed_artifacts.items() + } + direction, name = ( + ("outputs", "model") + if mutation == "producer_descriptor_type" + else ("inputs", "last_model") + ) + typed[direction][name]["type"]["name"] = "test.different.model" + records[node_id] = replace(record, typed_artifacts=typed) + elif mutation == "compiled_fit_params": + fit = case.fits[-1] + changed = replace(fit, params={**fit.params, "n_estimators": 3}) + compiled = compile_graph( + replace( + compiled.graph, + nodes=tuple( + changed if node.id == fit.id else node + for node in compiled.graph.nodes + ), + ) + ) + elif mutation == "compiled_apply_edge": + apply = case.applies[-1] + changed = replace( + apply, + artifact_inputs=tuple( + replace(edge, producer=case.applies[1].id) + if edge.name == "prior_000" + else edge + for edge in apply.artifact_inputs + ), + ) + compiled = compile_graph( + replace( + compiled.graph, + nodes=tuple( + changed if node.id == apply.id else node + for node in compiled.graph.nodes + ), + ) + ) + manifest = replace(manifest, nodes=records) + placement.load_full_puf_attachment_artifacts( + case.binding, compiled=compiled, manifest=manifest, store=case.store + ) + + +def test_person_and_tax_unit_masks_must_describe_the_same_memberships(attached_full65): + arguments = dict(attached_full65.retention_arguments) + population = _copy_population(arguments["population"]) + people, units = population.frame.table("person"), population.frame.table("tax_unit") + person_mask = support.puf_tax_detail_clone_mask(people, entity="person") + unit_mask = support.puf_tax_detail_clone_mask(units, entity="tax_unit") + people.loc[people.index[person_mask][0], "person_tax_unit_id"] = units.loc[ + ~unit_mask, "tax_unit_id" + ].iloc[0] + arguments.update( + population=population, + expected_population=_copy_population(population), + ) + # Both retained inputs agree on this malformed candidate; the independent + # mask-membership invariant, rather than a changed-state seal, must reject it. + with pytest.raises(ValueError, match="FULL_PUF_MASK_MEMBERSHIP"): + placement.retain_full_puf_attachment(**arguments) From bd236b79cf34b25ba9692b0dccd9f98f07dec0f1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 014/255] Stage exact ordinary solver and fit test fixture closure --- PROGRESS.md | 2 + .../group_bounds/legacy_prepatch.json | 217 +++++++ .../tests/test_group_bounds.py | 508 ++++++++++++++++ .../tests/test_grouped_fixed_support.py | 292 +++++++++ .../tests/golden/legacy-slice-apply-v1.json | 36 ++ .../tests/test_graph_legacy_matrix.py | 543 +++++++++++++++++ .../tests/test_graph_legacy_qrf.py | 555 ++++++++++++++++++ .../microcosm-fit/tests/test_qrf_stateless.py | 159 +++++ .../microcosm-fit/tests/test_qrf_target.py | 373 ++++++++++++ 9 files changed, 2685 insertions(+) create mode 100644 packages/microcosm-calibrate/tests/fixtures/group_bounds/legacy_prepatch.json create mode 100644 packages/microcosm-calibrate/tests/test_group_bounds.py create mode 100644 packages/microcosm-calibrate/tests/test_grouped_fixed_support.py create mode 100644 packages/microcosm-fit/tests/golden/legacy-slice-apply-v1.json create mode 100644 packages/microcosm-fit/tests/test_graph_legacy_matrix.py create mode 100644 packages/microcosm-fit/tests/test_graph_legacy_qrf.py create mode 100644 packages/microcosm-fit/tests/test_qrf_stateless.py create mode 100644 packages/microcosm-fit/tests/test_qrf_target.py diff --git a/PROGRESS.md b/PROGRESS.md index 4f1125af5..78184f6e6 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -759,3 +759,5 @@ Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and prese Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/packages/microcosm-calibrate/tests/fixtures/group_bounds/legacy_prepatch.json b/packages/microcosm-calibrate/tests/fixtures/group_bounds/legacy_prepatch.json new file mode 100644 index 000000000..10e3c8385 --- /dev/null +++ b/packages/microcosm-calibrate/tests/fixtures/group_bounds/legacy_prepatch.json @@ -0,0 +1,217 @@ +{ + "fixture_source": "packages/microcosm-calibrate/tests/test_solve.py::_exact_k_design_fixture", + "inputs": { + "measure": [ + 1.0, + 2.0, + 3.0, + 0.5 + ], + "target": 800.0, + "weights": [ + 100.0, + 120.0, + 80.0, + 40.0 + ] + }, + "launch": { + "dont_write_bytecode": 1, + "isolated": 1, + "pycache_prefix": "/private/tmp/origin-cap-prepatch-FBB56CDA-23F6-45AC-9C67-0370A90813D0" + }, + "loaded_source_before_after": { + "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py": { + "git_blob": "5aa9db0832e58c44479eeef11b08a7a5ea2710f4", + "sha256": "c15615552128dbaae6f71cc3156105cf04a29a8cf5885ed8c6b318782d61fd19" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/_target_loss_attribution.py": { + "git_blob": "1bdeb3c3ed65debb6549fcbb2d94956a43c5617a", + "sha256": "5addce7c3c708117bba12639b2949c18c99a810533ae94d89851d8fe7b15e2fb" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/diagnostics.py": { + "git_blob": "f63913d683dfdf20ba5df6bcff312af7bbf62a6a", + "sha256": "87ff674609f428ec49d64edbcb5ee8062cf6efb8058542a4add8e3ef50d2f156" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py": { + "git_blob": "82a745b114e1567f72547d88f58d350ace688f91", + "sha256": "c5084a0dd45617922f8efca08560f243195eb4456430c1d87258e6240f379ab4" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py": { + "git_blob": "306d426156f38c0c1f5c01b5268e0b8ef52b714f", + "sha256": "379e4adbb3ed7cdd13f7c7c548da45163f2a50330c874ee51095d38243d392e6" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/matrix.py": { + "git_blob": "11aeb35c47e0210d1baeb987aadb7ef68e17c615", + "sha256": "ca609f47ba7f5bf55ab8a0c85c07d2dac9b7a52fffcf0816f8289f143764c55a" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/monetary_binding.py": { + "git_blob": "ff41bd38e240030482718fb8e1ab527a93acbbf1", + "sha256": "f68027f38aa90738870addbe3da55ccfaa0f13a48c8d79a3f54d41abdc723867" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/registry.py": { + "git_blob": "f2cf74646d1c63c4aee0892d46ad2e113a1f48e2", + "sha256": "94952d466a21c6913019d5f14f50d322b90cb51205672758b76696d5f2ba5520" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/score.py": { + "git_blob": "00290a3a5b9d2c0e133343d39e3c2425d15f2c88", + "sha256": "aabf8f2fc44ac062a6af74463ccba225a6dacb8855db5ca5d50c0bd461a23662" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py": { + "git_blob": "746ed9e6a6d4ddab9262c9cbd411242af5a6c8f7", + "sha256": "f4b32c40a8675c37b0a4f3c1e48ca7b2078e803dac4413221aeb98c23b9e2066" + }, + "packages/microcosm-calibrate/src/microcosm/calibrate/target.py": { + "git_blob": "c3c8c39a6810ab20e6693218e7aa8d7052c15287", + "sha256": "764e87f4895930e1df725bc3c38c2d119afe10f7c715956cbcdb8d55ccc91b7a" + }, + "packages/microcosm-frame/src/microcosm/frame/__init__.py": { + "git_blob": "c0a97a493cddba1a75014dce33cd061ae08b18a0", + "sha256": "3a43c2b2812720d49b70db66cdfaaf2cfb045cf4718c033bcca2d92f87af0245" + }, + "packages/microcosm-frame/src/microcosm/frame/accounting.py": { + "git_blob": "f739f8de3893c559d9f470b08773d5de8ddfc563", + "sha256": "8c37f1394db523a8e0bcc98350456e4d9c8c268ca5a4432c07fc72ab218eaea3" + }, + "packages/microcosm-frame/src/microcosm/frame/bundle.py": { + "git_blob": "e16c3a45a7d940f979d384f17043df08834dd3bd", + "sha256": "2acfda18180003898f0453fbf7244540820cdc9ba755f0a917b768055e5b6ff1" + }, + "packages/microcosm-frame/src/microcosm/frame/materialize.py": { + "git_blob": "be49c5ebe7e807150e90d6491225e4f6e732ff21", + "sha256": "86c404724319099001132897a7ad82a5fb6b1e808cca29804ae3ef6d4310d0a3" + }, + "packages/microcosm-frame/src/microcosm/frame/rules.py": { + "git_blob": "34a3e1ca5f19c1f71bcf1f60d3665629b20ef759", + "sha256": "ea754746d47f7cb7311edfd7eeed7c8e84cccfebbf990df1e8fa4ce1965baaff" + }, + "packages/microcosm-frame/src/microcosm/frame/schema.py": { + "git_blob": "72db5c81b168afb184c5e23a4ef19eb5e32d9f9a", + "sha256": "75ba6c47b95ff826384e23f11e94178bdf89696f651df3cc6c93cee75e768946" + }, + "packages/microcosm-frame/src/microcosm/frame/units.py": { + "git_blob": "bdc4b6f48c58ee23633e4b5d64093ddad88e2817", + "sha256": "7ff5d7aea68dfc96e8528e3d604220b5cfc4728dd7436627bac904e7bb0a1b86" + }, + "packages/microcosm-frame/src/microcosm/frame/weights.py": { + "git_blob": "d4419a3faf7619b2a7b8da4e00aa8d651b745df0", + "sha256": "76ac3d735976e807d6df003bb75b901834f0bd1cdc91dfcac9cd903cec518e9d" + }, + "packages/microcosm-graph/src/microcosm/graph/__init__.py": { + "git_blob": "bbcbc5ea7a61cbab1466db25a3bd4707be66f52a", + "sha256": "c2e7710e96bb7ace608f8381c1a94767cff0c58a30fde62636ac05de2d1dc031" + }, + "packages/microcosm-graph/src/microcosm/graph/artifact_edges.py": { + "git_blob": "0b84849e57744448a1e627133a4550d12b0dad1e", + "sha256": "0e0fc6d9fe7e0d4790621be3f9fc7834d90ff78cc28ceb6373ae9cec9acec226" + }, + "packages/microcosm-graph/src/microcosm/graph/attachments.py": { + "git_blob": "885e2360722238b7398432f5807b089220900ae7", + "sha256": "e2f139086ca73d44a771f6266a7971324129161b92db58f45f49b52cb6eb3cf5" + }, + "packages/microcosm-graph/src/microcosm/graph/availability.py": { + "git_blob": "e89b1d90f0715d8536b41036cb388dcbda6fc504", + "sha256": "35bda816bd1b9a1324847c1b1389bd312edd889898876f8cb1e22a98f439df42" + }, + "packages/microcosm-graph/src/microcosm/graph/canonical.py": { + "git_blob": "50e6dd65c8b0541399370fbfc9091785ddf87fe9", + "sha256": "001b4a58439a1703df69c42cb1c355cd51f409f301984b70b141c5671217e79d" + }, + "packages/microcosm-graph/src/microcosm/graph/codecs.py": { + "git_blob": "3ebc9d8e9399af1a8020b4e0533aee8b287316b0", + "sha256": "9ef98d1ea83c8f52152b57e09eff6cfb472fd675f100d352e90b8470bf002b6e" + }, + "packages/microcosm-graph/src/microcosm/graph/decl.py": { + "git_blob": "b6134c210b913e552a0a204a2de0f4691c50fe3b", + "sha256": "8229270f3328f537c8f8e83d6c81af39aa75d4ca5aa6a7bd3d37ecfc25aee2fe" + }, + "packages/microcosm-graph/src/microcosm/graph/errors.py": { + "git_blob": "297f1b50705de9f8da2b736f15e2e13aa2b33beb", + "sha256": "917e3adb200f15e984b418529fb05eb8e74c4ea7517af6f3cdf4e75a29253296" + }, + "packages/microcosm-graph/src/microcosm/graph/executor.py": { + "git_blob": "ec54c905f4dad7aebd71497b5d124908a9fd887a", + "sha256": "1ffa7c6958ce67c406c2be5ae3c0faef306661a2d21de5c032cf6ff4437862d7" + }, + "packages/microcosm-graph/src/microcosm/graph/explain.py": { + "git_blob": "26abfdbd4d93495360ad0317eb059d431e52d2d2", + "sha256": "5e81b2162a9172fedd980ad76a576796cc07e6ce3ae28691f53b0475f83b0e05" + }, + "packages/microcosm-graph/src/microcosm/graph/kernel.py": { + "git_blob": "0846a3eaac1faeb1425103c85affc740048b2ddc", + "sha256": "07691fb5cf45ae700a258713ebdc9aa845891a432ec9107e223b8c943b5f3cb6" + }, + "packages/microcosm-graph/src/microcosm/graph/keys.py": { + "git_blob": "3844ec56e44cbfc8252efaaaca4ef0e5cbb67929", + "sha256": "d1bf4e5298dd04ef8445f42702d85e349262031191a24fe034d75ac47efafa3d" + }, + "packages/microcosm-graph/src/microcosm/graph/manifest.py": { + "git_blob": "f7b1dd066cab17aeebd3e5ffeab841595a497641", + "sha256": "2e459faab7b008c3636501638ca4757b9a5b0d5c97d72fb349437152eabf8674" + }, + "packages/microcosm-graph/src/microcosm/graph/population.py": { + "git_blob": "1a52baf3f62ca2e1da10d134f0538ddeb4f7f5a7", + "sha256": "4c272bcff4f5f24f02ca110a7018886adf725ddabc191b8777fb0bd78a0792b8" + }, + "packages/microcosm-graph/src/microcosm/graph/randomness.py": { + "git_blob": "5a5510bc276b5d80222e803c5b6e03db7eeecb1d", + "sha256": "797fa7e4c7cc2fc81312f7886521e9b0f3e02be577e601a9cabc24362485847a" + }, + "packages/microcosm-graph/src/microcosm/graph/serialize.py": { + "git_blob": "85c71b4de6b41e29810672af90edb56306d43ec1", + "sha256": "1137264eaa5faedb04f06284c5b0737dd3b13c12edbf2e78255827618540a8cc" + }, + "packages/microcosm-graph/src/microcosm/graph/store.py": { + "git_blob": "6f93b8b650515b1da2a4d5968ba95df84d21aa39", + "sha256": "f16dfca2bf77843594fc3848bceab52379438ad722f800063cde445f63f196f3" + }, + "packages/microcosm-graph/src/microcosm/graph/view.py": { + "git_blob": "446a788385e407889d6880c2be921cd0a623d8c9", + "sha256": "e3b155b2eb004a121f8a129b92efb2506d17310f41ddff4fe1a0a7e820296b59" + } + }, + "options": { + "epochs": 7, + "learning_rate": 0.02, + "mass": "free", + "max_weight_ratio": 2.0, + "seed": 0 + }, + "settings": { + "backend": "Adam", + "device": "cpu", + "matrix_dtype": "float32", + "numpy": "2.4.6", + "output_dtype": "float64", + "parameter_dtype": "float32", + "platform_fingerprint": "arm64/darwin/py3.14", + "python": "3.14.4 (main, Apr 14 2026, 14:46:33) [Clang 22.1.3 ]", + "python_binary_sha256": "bf4ea26231212e330b8e9b6f24969a30f894fec88665bf42754070d114ac7fc6", + "thread_environment": { + "MKL_NUM_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "torch": "2.12.0", + "torch_interop_threads": 18, + "torch_threads": 1 + }, + "source_head": "a4e69c0a1e7b5e953922d6d0a0d0283890c3c329", + "source_tree": "/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-national-age-root-20260906", + "trajectory": { + "bytes_hex": "000000000000d03f000000a0870fce3f00000040ca14cc3f000000c0650fca3f00000080f4fec73f000000200ce3c53f000000803dbbc33f", + "sha256": "01943d29ea7d5e727cf88b02fb37416640471509212cd59181e6b1af6b9b4459", + "shape": [ + 7 + ] + }, + "weights": { + "bytes_hex": "00000000d2c45c4000000060e442614000000040db035740000000c0d9034740", + "sha256": "260f8beb3154aab82e1484e41f2b8d0cf33e45478b2b106deb66f97b64cbe11a", + "shape": [ + 4 + ] + } +} diff --git a/packages/microcosm-calibrate/tests/test_group_bounds.py b/packages/microcosm-calibrate/tests/test_group_bounds.py new file mode 100644 index 000000000..0ad218d1d --- /dev/null +++ b/packages/microcosm-calibrate/tests/test_group_bounds.py @@ -0,0 +1,508 @@ +"""Invented strict grouped-cap numerical and prepatch byte contracts.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import torch + +from microcosm.calibrate import GroupedUpperBounds, Target, TargetSet, calibrate, solve +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _frame(weights=(40.0, 40.0, 40.0, 40.0), ids=(0, 1, 2, 3)): + n = len(weights) + return Frame( + { + "person": pd.DataFrame({"person_id": range(n), "person_household_id": ids}), + "household": pd.DataFrame( + { + "household_id": ids, + "native": np.array([1.0, 0.0] * (n // 2)), + "detail": np.array([0.0, 1.0] * (n // 2)), + } + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(values=np.asarray(weights), kind=WeightKind.IMPORTANCE)}, + ) + + +def _targets(): + return TargetSet( + ( + Target(name="native", entity="household", value=380.0, measure="native"), + Target(name="detail", entity="household", value=20.0, measure="detail"), + ) + ) + + +def _groups(bounds=(200.0, 200.0), ids=(0, 1, 2, 3)): + return GroupedUpperBounds(ids, np.array([0, 0, 1, 1]), np.array(bounds)) + + +def _internal(initial, groups, **overrides): + options = dict( + epochs=2, + learning_rate=0.02, + conserve_mass=False, + max_weight_ratio=None, + l0_lambda=0.0, + l2_lambda=0.0, + target_records=None, + init_mean=0.999, + temperature=0.25, + grouped_upper_bounds=groups, + ) + options.update(overrides) + return solve._optimize( + torch.ones((1, len(initial)), dtype=torch.float32), + torch.tensor([200.0], dtype=torch.float32), + None, + torch.tensor([200.0], dtype=torch.float32), + 10.0, + np.asarray(initial, dtype=np.float64), + **options, + ) + + +def test_two_reviewed_examples_and_strict_binding_counts(): + design = GroupedUpperBounds( + (0, 1), np.array([0, 0]), np.array([min(2 * 100.0, 3 * 80.0)]) + ) + design.check(np.array([190.0, 10.0])) + with pytest.raises(ValueError, match="exceed"): + design.check(np.array([120.0, 120.0])) + incoming = GroupedUpperBounds( + (0, 1), np.array([0, 0]), np.array([min(4 * 100.0, 2 * 80.0)]) + ) + incoming.check(np.array([150.0, 10.0])) + with pytest.raises(ValueError, match="exceed"): + incoming.check(np.array([170.0, 0.0])) + assert incoming.diagnostics(np.array([150.0, 10.0]))["binding_group_count"] == 1 + assert ( + incoming.diagnostics(np.array([np.nextafter(150.0, 0.0), 10.0]))[ + "binding_group_count" + ] + == 0 + ) + + +def test_groups_are_immutable_and_complete_even_with_zero_rows(): + ids = [2, 1] + indices = np.array([0, 0]) + bounds = np.array([0.0]) + groups = GroupedUpperBounds(ids, indices, bounds) + indices[0] = 9 + bounds[0] = 9 + ids[0] = 9 + assert groups.household_ids == (2, 1) + groups.check(np.zeros(2)) + for array in (groups.group_indices, groups.absolute_bounds): + with pytest.raises(ValueError): + array.setflags(write=True) + with pytest.raises(ValueError, match="exceed"): + groups.check(np.array([0.0, np.nextafter(0.0, 1.0)])) + with pytest.raises(ValueError, match="align"): + groups.check(np.zeros(1)) + + +@pytest.mark.parametrize( + "count", [-1, 3, True, False, np.bool_(True), 1.0, 0.5, np.nan, np.inf, "1", None] +) +def test_diagnostic_corrected_count_refuses_invalid_domain(count): + with pytest.raises(ValueError, match="last corrected group count"): + _groups().diagnostics(np.full(4, 40.0), count) + + +@pytest.mark.parametrize("count", [0, 1, 2, np.int64(2)]) +def test_diagnostic_corrected_count_retains_integer_domain(count): + recorded = _groups().diagnostics(np.full(4, 40.0), count) + assert recorded["last_corrected_group_count"] == int(count) + assert type(recorded["last_corrected_group_count"]) is int + + +@pytest.mark.parametrize( + "ids,indices,bounds", + [ + ((1, 1), [0, 0], [1.0]), + ((1, 2), [0], [1.0]), + ((1, 2), [0.0, 0.0], [1.0]), + ((1, 2), [0, 2], [1.0, 1.0]), + ((1, 2), [0, 0], [1.0, 1.0]), + ((1, 2), [0, 0], [np.nan]), + ((1, 2), [0, 0], [np.inf]), + ((1, 2), [0, 0], [-1.0]), + ], +) +def test_invalid_maps_refuse(ids, indices, bounds): + with pytest.raises(ValueError): + GroupedUpperBounds(ids, np.array(indices), np.array(bounds)) + + +@pytest.mark.parametrize( + "values", [[np.inf, 1.0], [np.nan, 1.0], [-1.0, 1.0], [1e308, 1e308]] +) +def test_nonfinite_negative_or_overflowed_sums_refuse(values): + groups = GroupedUpperBounds((0, 1), np.array([0, 0]), np.array([1e308])) + with pytest.raises(ValueError): + groups.check(np.array(values)) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"max_weight_ratio": 2.0}, + {"method": "prox"}, + {"mass": "conserve"}, + {"l0_lambda": 0.001}, + {"target_records": 2}, + {"l1_lambda": 0.1}, + ], +) +def test_public_unsupported_modes_refuse_before_optimizer(monkeypatch, kwargs): + calls = [] + + def forbidden(*args, **kw): + calls.append(1) + raise AssertionError("optimizer constructed") + + monkeypatch.setattr(torch.optim, "Adam", forbidden) + with pytest.raises(ValueError, match="grouped upper bounds"): + calibrate(_frame(), _targets(), grouped_upper_bounds=_groups(), **kwargs) + assert not calls + + +@pytest.mark.parametrize( + "kwargs", + [ + {"max_weight_ratio": 2.0}, + {"conserve_mass": True}, + {"l0_lambda": 0.001}, + {"target_records": 2}, + ], +) +def test_internal_unsupported_modes_refuse_before_optimizer(monkeypatch, kwargs): + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match="grouped upper bounds"): + _internal([40.0, 40.0, 40.0, 40.0], _groups(), **kwargs) + assert not calls + + +@pytest.mark.parametrize( + "initial,bounds,warm", + [ + ([120.0, 120.0, 40.0, 40.0], (200.0, 200.0), None), + ([40.0, 40.0, 40.0, 40.0], (0.0, 200.0), None), + ([0.0, 40.0, 40.0, 40.0], (200.0, 200.0), None), + ([40.0, 40.0, 40.0, 40.0], (200.0, 200.0), [120.0, 120.0, 40.0, 40.0]), + ([40.0, 40.0, 40.0, 40.0], (200.0, 200.0), [0.0, 40.0, 40.0, 40.0]), + ([40.0, 40.0, 40.0, 40.0], (200.0, 200.0), [np.nan, 40.0, 40.0, 40.0]), + ], +) +def test_initial_and_effective_warm_refuse_before_optimizer( + monkeypatch, initial, bounds, warm +): + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError): + _internal( + initial, + _groups(bounds), + warm_start_weights=None if warm is None else np.array(warm), + ) + assert calls == [] + + +def test_public_overbudget_warm_checks_effective_prepared_vector(monkeypatch): + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match="exceed"): + calibrate( + _frame(), + _targets(), + epochs=2, + grouped_upper_bounds=_groups(), + warm_start_weights=np.array([150.0, 100.0, 40.0, 40.0]), + ) + assert calls == [] + # Test the actual preparation boundary too, rather than only raw warm input. + monkeypatch.setattr( + solve, + "_prepare_warm_start_weights", + lambda *a, **k: np.array([150.0, 100.0, 40.0, 40.0]), + ) + with pytest.raises(ValueError, match="exceed"): + calibrate(_frame(), _targets(), epochs=2, grouped_upper_bounds=_groups()) + assert calls == [] + + +@pytest.mark.parametrize("sparse", [False, True]) +def test_real_adam_redistributes_and_every_accepted_update_is_observed( + monkeypatch, sparse +): + groups = _groups() + observed = [] + layouts = [] + apply = solve._apply_constraint + + def checked_apply(matrix, weights): + assert matrix.dtype == weights.dtype == torch.float32 + assert weights.grad_fn is not None + layouts.append(matrix.layout) + return apply(matrix, weights) + + monkeypatch.setattr(solve, "_apply_constraint", checked_apply) + if sparse: + monkeypatch.setattr(solve, "_SPARSE_MIN_CELLS", 1) + monkeypatch.setattr(solve, "_SPARSE_DENSITY_CUTOFF", 1.0) + + def observer(payload): + weights = payload["weights"] + assert weights.dtype == np.float64 + assert np.array_equal( + groups.check(weights, positive=True), payload["group_totals"] + ) + assert payload["household_ids"] == groups.household_ids + observed.append(payload) + + result = calibrate( + _frame(), + _targets(), + epochs=60, + learning_rate=0.08, + grouped_upper_bounds=groups, + _post_projection_observer=observer, + ) + assert len(observed) == 61 + assert [x["epoch"] for x in observed] == list(range(61)) + assert result.weights[0] > 120.0 # exceeds removed role-wise rI * 40 cap + assert result.weights[1] < 40.0 + assert result.weights.tobytes() == observed[-1]["weights"].tobytes() + assert ( + result.frame.resolve_weights("household").values.tobytes() + == result.weights.tobytes() + ) + assert all(x == (torch.sparse_csr if sparse else torch.strided) for x in layouts) + assert result.options["grouped_upper_bounds"] == groups.diagnostics( + result.weights, observed[-1]["corrected_group_count"] + ) + + +@pytest.mark.parametrize("sparse", [False, True]) +def test_float32_matmul_remains_differentiable_to_float64_log_weights(sparse): + matrix = torch.tensor([[1.0, 0.0], [0.0, 2.0]], dtype=torch.float32) + if sparse: + matrix = matrix.to_sparse_csr() + log_w = torch.tensor(np.log([40.0, 40.0]), dtype=torch.float64, requires_grad=True) + estimate = solve._apply_constraint(matrix, torch.exp(log_w).to(torch.float32)) + assert estimate.dtype == torch.float32 + estimate.sum().backward() + assert log_w.grad.dtype == torch.float64 + assert torch.isfinite(log_w.grad).all() and torch.all(log_w.grad != 0) + assert matrix.dtype == torch.float32 + + +def test_no_update_final_is_effective_initial_bytes_without_reexp(): + initial = np.array([100.0, 120.0, 80.0, 40.0]) + assert np.exp(np.log(initial)).tobytes() != initial.tobytes() + observed = [] + final, trajectory = _internal( + initial, + _groups((220.0, 120.0)), + epochs=0, + _post_projection_observer=observed.append, + ) + assert not len(trajectory) + assert final.tobytes() == initial.tobytes() == observed[-1]["weights"].tobytes() + + +def test_projection_is_common_factor_from_original_candidate_and_strict(): + rng = np.random.default_rng(11) + found_roundoff = False + for _ in range(1000): + candidate = rng.uniform(1.0, 200.0, 3) + bound = float(rng.uniform(1.0, 100.0)) + groups = GroupedUpperBounds( + (30, 10, 20), np.zeros(3, dtype=int), np.array([bound]) + ) + total = math.fsum(float(candidate[i]) for i in (1, 2, 0)) + if total <= bound: + continue + factor = np.float64(bound / total) + naive = candidate * factor + naive_total = math.fsum(float(naive[i]) for i in (1, 2, 0)) + if naive_total <= bound: + continue + found_roundoff = True + for _ in range(64): + factor = np.nextafter(factor, 0.0) + expected = candidate * factor + if math.fsum(float(expected[i]) for i in (1, 2, 0)) <= bound: + break + accepted, count = groups.project(candidate) + assert count == 1 + assert accepted.tobytes() == expected.tobytes() + assert groups.project(accepted)[0].tobytes() == accepted.tobytes() + assert groups.check(accepted)[0] <= bound + break + assert found_roundoff, "invented roundoff boundary must be exercised" + + +def test_reorder_requires_aligned_ids_and_preserves_by_key_solution(): + groups = _groups() + changed = GroupedUpperBounds( + (2, 3, 0, 1), np.array([1, 1, 0, 0]), np.array([200.0, 200.0]) + ) + with pytest.raises(ValueError, match="ordered IDs"): + calibrate(_frame(), _targets(), epochs=2, grouped_upper_bounds=changed) + # Frame requires sorted IDs. Its lower numerical seam accepts an explicit + # order, where simultaneous vector/map alignment must preserve keyed values. + baseline, _ = _internal([10.0, 20.0, 30.0, 40.0], groups, epochs=3) + reordered, _ = _internal([30.0, 40.0, 10.0, 20.0], changed, epochs=3) + assert dict(zip(groups.household_ids, baseline, strict=True)) == dict( + zip(changed.household_ids, reordered, strict=True) + ) + assert changed.constraint_digest != groups.constraint_digest + + +def test_singleton_admission_is_exactly_direct_inequalities(): + design = np.array([100.0, 20.0]) + incoming = np.array([80.0, 10.0]) + bound = np.minimum(2 * design, 3 * incoming) + groups = GroupedUpperBounds((0, 1), np.array([0, 1]), bound) + for candidate in ( + np.array([190.0, 10.0]), + np.array([210.0, 10.0]), + np.array([100.0, 31.0]), + np.array([0.0, 0.0]), + ): + direct = bool( + np.all(candidate <= 2 * design) and np.all(candidate <= 3 * incoming) + ) + if direct: + groups.check(candidate) + else: + with pytest.raises(ValueError): + groups.check(candidate) + + +def test_saved_prepatch_legacy_exact_bytes(): + from microcosm.graph import platform_fingerprint + + path = Path(__file__).parent / "fixtures/group_bounds/legacy_prepatch.json" + frozen = json.loads(path.read_text()) + running = { + "platform_fingerprint": platform_fingerprint(), + "python": sys.version, + "numpy": np.__version__, + "torch": torch.__version__, + "torch_threads": torch.get_num_threads(), + "torch_interop_threads": torch.get_num_interop_threads(), + "device": "cpu", + "backend": "Adam", + "parameter_dtype": "float32", + "matrix_dtype": "float32", + "output_dtype": "float64", + "thread_environment": { + k: os.environ.get(k) + for k in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS", + ) + }, + "python_binary_sha256": hashlib.sha256( + Path(sys.executable).read_bytes() + ).hexdigest(), + } + if running != frozen["settings"]: + pytest.skip( + "Matching configuration required; recorded=" + + json.dumps(frozen["settings"], sort_keys=True) + + " running=" + + json.dumps(running, sort_keys=True) + ) + inputs = frozen["inputs"] + frame = Frame( + { + "person": pd.DataFrame( + {"person_id": range(4), "person_household_id": range(4)} + ), + "household": pd.DataFrame( + {"household_id": range(4), "adjudicated_measure": inputs["measure"]} + ), + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + values=np.array(inputs["weights"]), kind=WeightKind.DESIGN + ) + }, + ) + targets = TargetSet( + ( + Target( + name="adjudicated", + entity="household", + value=inputs["target"], + measure="adjudicated_measure", + ), + ) + ) + result = calibrate(frame, targets, **frozen["options"]) + for label, values in ( + ("weights", result.weights), + ("trajectory", result.loss_trajectory), + ): + data = np.asarray(values, dtype=" 40.0 and result.weights[3] < 40.0 + assert result.weights.tobytes() == observations[-1]["weights"].tobytes() + assert ( + result.frame.resolve_weights("household").values.tobytes() + == result.weights.tobytes() + ) + assert ( + result.initial_weights.tobytes() + == original.resolve_weights("household").values.tobytes() + ) + assert result.options["grouped_preserve_zeros"] == { + "enabled": True, + "fixed_zero_count": 3, + "ordered_zero_mask_sha256": hashlib.sha256( + bytes([1, 1, 0, 0, 1, 0]) + ).hexdigest(), + } + + +@pytest.mark.parametrize( + "warm", + [ + [1.0, 0.0, 40.0, 40.0, 0.0, 20.0], + [0.0, 0.0, 0.0, 40.0, 0.0, 20.0], + [0.0, 0.0, 60.0, 60.0, 0.0, 20.0], + [0.0, 0.0, 40.0, 40.0, -1.0, 20.0], + [0.0, 0.0, float("nan"), 40.0, 0.0, 20.0], + [0.0, 0.0, 40.0, 40.0, 0.0], + ], +) +def test_invalid_warm_support_refuses_before_optimizer(monkeypatch, warm): + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError): + _fit(warm_start_weights=np.array(warm)) + assert calls == [] + + +def test_valid_warm_starts_keep_original_support_and_initial_diagnostics(): + first = _fit() + seen = [] + second = _fit( + epochs=2, + warm_start_weights=first.weights, + _post_projection_observer=seen.append, + ) + assert seen[0]["weights"].tobytes() == first.weights.tobytes() + assert ( + second.initial_weights.tobytes() + == _frame().resolve_weights("household").values.tobytes() + ) + assert second.weights[[0, 1, 4]].tobytes() == np.zeros(3).tobytes() + + +@pytest.mark.parametrize("value", [1, "yes", None, np.bool_(True)]) +def test_nonboolean_zero_option_refuses(value): + with pytest.raises(ValueError, match="boolean"): + _fit(grouped_preserve_zeros=value) + + +def test_zero_option_requires_grouping_and_default_keeps_positive_contract(): + with pytest.raises(ValueError, match="requires grouped"): + _fit(grouped_upper_bounds=None) + with pytest.raises(ValueError, match="strictly positive"): + _fit(grouped_preserve_zeros=False) + + +@pytest.mark.parametrize("anchor", ["initial", "uniform", "explicit"]) +def test_l2_penalty_is_finite_with_fixed_zero_support(anchor): + options = {"l2_lambda": 0.01, "l2_anchor": anchor} + if anchor == "explicit": + options["l2_anchor_weights"] = np.array([1.0, 1.0, 40.0, 40.0, 1.0, 20.0]) + result = _fit(**options) + assert ( + np.isfinite(result.loss_trajectory).all() and np.isfinite(result.weights).all() + ) + assert result.weights[[0, 1, 4]].tobytes() == np.zeros(3).tobytes() + + +def test_all_zero_numeric_support_refuses_before_optimizer(monkeypatch): + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match="positive initial"): + solve._optimize( + torch.ones((1, 2), dtype=torch.float32), + torch.ones(1, dtype=torch.float32), + None, + torch.ones(1, dtype=torch.float32), + 10.0, + np.zeros(2), + epochs=2, + learning_rate=0.02, + conserve_mass=False, + max_weight_ratio=None, + l0_lambda=0.0, + l2_lambda=0.0, + target_records=None, + init_mean=0.999, + temperature=0.25, + grouped_upper_bounds=GroupedUpperBounds( + (0, 1), np.array([0, 0]), np.array([0.0]) + ), + grouped_preserve_zeros=True, + ) + assert calls == [] + + +def test_observer_cannot_change_fixed_mask_or_accepted_weights(): + baseline = _fit() + + def corrupt(payload): + payload["weights"][:] = 1.0 + payload["group_indices"][:] = 0 + payload["absolute_bounds"][:] = 99.0 + payload["group_totals"][:] = 99.0 + + changed = _fit(_post_projection_observer=corrupt) + assert changed.weights.tobytes() == baseline.weights.tobytes() + + +def test_materialized_zero_activation_refuses(monkeypatch): + apply = solve._apply_weights + + def corrupt(*args, **kwargs): + result = apply(*args, **kwargs) + current = result.resolve_weights("household") + values = current.values.copy() + # Within a positive group bound, so group totals alone cannot catch this. + values[4] = np.nextafter(0.0, 1.0) + return result.with_weights( + "household", current.with_values(values, kind=current.kind), mass="conserve" + ) + + monkeypatch.setattr(solve, "_apply_weights", corrupt) + with pytest.raises(ValueError, match="zero support"): + _fit() + + +def test_aligned_reordering_required_with_zero_rows(): + reordered = GroupedUpperBounds( + (1, 0, 2, 3, 4, 5), np.array([0, 0, 1, 1, 2, 2]), np.array([0.0, 100.0, 30.0]) + ) + with pytest.raises(ValueError, match="ordered IDs"): + _fit(grouped_upper_bounds=reordered) + + +def test_retained_zero_ratio_is_finite_and_activation_is_infinite(): + result = _fit() + active = result.initial_weights > 0 + expected = float((result.weights[active] / result.initial_weights[active]).max()) + assert result.realized_max_weight_ratio == expected + # A diagnostic must not hide invalid positive mass on an initial zero. + from dataclasses import replace + + changed = result.weights.copy() + changed[0] = 1.0 + assert replace(result, weights=changed).realized_max_weight_ratio == float("inf") + + +def test_returned_id_swap_refuses_even_when_weight_bytes_match(monkeypatch): + apply = solve._apply_weights + + def corrupt(*args, **kwargs): + result = apply(*args, **kwargs) + result.table("household").loc[[0, 2], "household_id"] = [2, 0] + return result + + monkeypatch.setattr(solve, "_apply_weights", corrupt) + with pytest.raises(ValueError, match="ordered IDs"): + _fit() + + +def test_late_options_identity_mutation_refuses(monkeypatch): + apply = solve._apply_weights + diagnostics = GroupedUpperBounds.diagnostics + frames = [] + + def capture(*args, **kwargs): + result = apply(*args, **kwargs) + frames.append(result) + return result + + def corrupt(self, weights, last_corrected_count=0): + result = diagnostics(self, weights, last_corrected_count) + frames[-1].table("household").loc[[0, 2], "household_id"] = [2, 0] + return result + + monkeypatch.setattr(solve, "_apply_weights", capture) + monkeypatch.setattr(GroupedUpperBounds, "diagnostics", corrupt) + with pytest.raises(ValueError, match="ordered IDs"): + _fit() diff --git a/packages/microcosm-fit/tests/golden/legacy-slice-apply-v1.json b/packages/microcosm-fit/tests/golden/legacy-slice-apply-v1.json new file mode 100644 index 000000000..914ec3b95 --- /dev/null +++ b/packages/microcosm-fit/tests/golden/legacy-slice-apply-v1.json @@ -0,0 +1,36 @@ +{ + "base": "c71e31654faf0e3f874e68e8c100b09c7eb9955f", + "platform": "arm64/darwin/py3.14", + "cases": { + "positive": [ + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a2279227d0a00000000000008408e75aa87bb4c1b4000000000000008400000000000001c40", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2239633536356632386533313730646437393464376332336139303034626335333731666136633035386430373862383165313930636534363833366333333032222c22746172676574223a2279222c22747261696e696e675f6964223a2262663632616139626439353930376637613165376331343061373138363633656162383234333935626539303132346339393031666264393833636138393738227d5d2c227261775f74617267657473223a5b7b22736861323536223a2266623739383562333563396139386630346232313234643561613563393431316166653335363438366364376335623436376232636132333163613762363161222c22746172676574223a2279227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a3233333234303435363237333137383639363231333336323532363536373331353537373437347d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a312c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a3133333938303936313439343338343135353036313135363839353032383538383831373035377d2c2275696e7465676572223a313432323336313731377d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + }, + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a227a227d0a000000000000144000000000000034400000000000001c400000000000804440", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2239633536356632386533313730646437393464376332336139303034626335333731666136633035386430373862383165313930636534363833366333333032222c22746172676574223a2279222c22747261696e696e675f6964223a2262663632616139626439353930376637613165376331343061373138363633656162383234333935626539303132346339393031666264393833636138393738227d2c7b22736861323536223a2234646436646463363864333435303635393537656239343834633435323363616434316463643632396436376365303333353365396238333238343535653965222c22746172676574223a227a222c22747261696e696e675f6964223a2231353163663738613838306365616463653339316633353930356261373538656366303831316561623136653736633062323132643835393739666435323738227d5d2c227261775f74617267657473223a5b7b22736861323536223a2266623739383562333563396139386630346232313234643561613563393431316166653335363438366364376335623436376232636132333163613762363161222c22746172676574223a2279227d2c7b22736861323536223a2232623363313434636364363630386332396266653766633031303265306264316536623635316637353336653335353630393439366439656232626161303136222c22746172676574223a227a227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279222c227a225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a3236343333313134303938343430363136383536333836353536303439343432343537383636327d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a3332333531303338383036313931333538383435333433383039333838333635323331343737397d2c2275696e7465676572223a313432323336313731377d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + } + ], + "hurdle": [ + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a2279227d0a829752b72a240540000000000000184000000000000000000000000000002040", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2266343938396536656137313030366563326639636235333834326466373762396532613939636230353339303738386564363235363236323031623564396438222c22746172676574223a2279222c22747261696e696e675f6964223a2261626363303232393362373461353239663461383366643737643336343761643961623337343134326264383439643430396530383265393437383863643933227d5d2c227261775f74617267657473223a5b7b22736861323536223a2235333935336531343730376531633562373161666262303638666539326430393430346238316363353466346636373932666135343531396665666636353334222c22746172676574223a2279227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a3236343333313134303938343430363136383536333836353536303439343432343537383636327d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a3137313834313631383630373538303934343431393033353835353131323335363535363832327d2c2275696e7465676572223a313432323336313731377d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + }, + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a227a227d0a0000000000001840e10b30bbbcfa3d4000000000000014400000000000804440", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2266343938396536656137313030366563326639636235333834326466373762396532613939636230353339303738386564363235363236323031623564396438222c22746172676574223a2279222c22747261696e696e675f6964223a2261626363303232393362373461353239663461383366643737643336343761643961623337343134326264383439643430396530383265393437383863643933227d2c7b22736861323536223a2235616465616335313938363032316564346537333139313037356635333562393531366265623062383234303332396633633036383935333065313362376565222c22746172676574223a227a222c22747261696e696e675f6964223a2231636663383765323266373361616237303265326163643634316365656536303963633233343038363435343236303532346333383031343433396433616663227d5d2c227261775f74617267657473223a5b7b22736861323536223a2235333935336531343730376531633562373161666262303638666539326430393430346238316363353466346636373932666135343531396665666636353334222c22746172676574223a2279227d2c7b22736861323536223a2233623237663062323362313838626663373037633139613961373263383261303131393333386339373161363833326431643930333133613933363965306463222c22746172676574223a227a227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279222c227a225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a393139393837363435313632383037343833373437333130303539373930383331303533387d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a312c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a38363938373530373735353638313133343432393437383337353232323434343132383135317d2c2275696e7465676572223a333739333137363238377d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + } + ], + "zero_constant": [ + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a2279227d0a0000000000000000000000000000000000000000000000000000000000000000", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2238316163313134356661343434383931376265643833346435653639656531383265363131616137303338333439623633663562353730663430623562316565222c22746172676574223a2279222c22747261696e696e675f6964223a2265656436613536383139383535316535336262363763376134313237353161643764666665613464363464366330316434316161373663303661376631636133227d5d2c227261775f74617267657473223a5b7b22736861323536223a2265323765656666303362643832306334316238376166613930363630316162616464353434316239373237376635363165636262336139386132323432613030222c22746172676574223a2279227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a3331353432353934303731333632383330353930323939313538353831383134363636313938327d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a3230383438303536343233343633383630393734363631383636393238393238313431323930347d2c2275696e7465676572223a307d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + }, + { + "raw_draw": "6d6963726f636f736d2e6669742e7172662e6c65676163795f7261775f7461726765742f310a7b22696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22746172676574223a227a227d0a0000000000000440000000000000044000000000000004400000000000000440", + "apply_state": "7b226d6f64656c73223a5b7b22736861323536223a2238316163313134356661343434383931376265643833346435653639656531383265363131616137303338333439623633663562353730663430623562316565222c22746172676574223a2279222c22747261696e696e675f6964223a2265656436613536383139383535316535336262363763376134313237353161643764666665613464363464366330316434316161373663303661376631636133227d2c7b22736861323536223a2238393562313934626530393636613661343536376662303430393638386563343833303738396262386130316133393061383330666539363539323439623139222c22746172676574223a227a222c22747261696e696e675f6964223a2264316439363865616430343466663332373732363036353039366663653038336634333636353834633430356164373938643432363931343363666339316266227d5d2c227261775f74617267657473223a5b7b22736861323536223a2265323765656666303362643832306334316238376166613930363630316162616464353434316239373237376635363165636262336139386132323432613030222c22746172676574223a2279227d2c7b22736861323536223a2239343733393031323763666636396565326364663165353362396432346133376538353731616630313638666161663061326165333335626530343532363063222c22746172676574223a227a227d5d2c22736368656d615f76657273696f6e223a312c2273656564223a33312c227374617465223a7b22636f6d706c657465645f74617267657473223a5b2279222c227a225d2c22646f6e6f725f696e646578223a7b22636c6173735f6e616d65223a2252616e6765496e646578222c226474797065223a22696e743634222c226c656e677468223a34322c226e616d65735f72657072223a22284e6f6e652c29222c22736861323536223a2232643264613732333831663462623134373162396637383564376535363738303038333566396233366465353939353666663261666231393535306338306235227d2c22647261775f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a302c227374617465223a7b22696e63223a38313337333934343436393231343637303133323435343135323931363837373137383631352c227374617465223a3233333234303435363237333137383639363231333336323532363536373331353537373437347d2c2275696e7465676572223a307d2c22656e74697479223a227461785f756e6974222c226669745f726e675f7374617465223a7b226269745f67656e657261746f72223a225043473634222c226861735f75696e743332223a312c227374617465223a7b22696e63223a3239383335373133323530363933333430393335383635383139373432393932303538323739392c227374617465223a3133333938303936313439343338343135353036313135363839353032383538383831373035377d2c2275696e7465676572223a313432323336313731377d2c226d6f64656c5f636f6e666967223a7b226669745f6e5f6a6f6273223a312c226d61785f73616d706c65735f6c656166223a6e756c6c2c226d61785f73616d706c65735f6c6561665f6b696e64223a226e6f6e65222c226e5f657374696d61746f7273223a322c2273656564223a33312c227a65726f5f61746f6c223a31652d30367d2c22707265646963746f7273223a5b2278225d2c22726563697069656e745f696e646578223a7b22636c6173735f6e616d65223a22496e646578222c226474797065223a22696e743634222c226c656e677468223a342c226e616d65735f72657072223a222827726f77272c29222c22736861323536223a2236653361373365653564643164353562666264613035663334393965316632393038396263353566353039383564613330323436393335303331666330623662227d2c22736368656d615f76657273696f6e223a312c2274617267657473223a5b2279222c227a225d2c227765696768745f6b696e64223a2264657369676e222c227765696768745f736861323536223a2261323436353435363332333964336334396339313230623633303632653939643539356236646136333837343736616131666339343564653033613433363039227d7d" + } + ] + } +} diff --git a/packages/microcosm-fit/tests/test_graph_legacy_matrix.py b/packages/microcosm-fit/tests/test_graph_legacy_matrix.py new file mode 100644 index 000000000..dc93d54ee --- /dev/null +++ b/packages/microcosm-fit/tests/test_graph_legacy_matrix.py @@ -0,0 +1,543 @@ +"""Derived model rows cross typed edges without a new population or index edit.""" + +import hashlib +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.fit import QRF +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.graph_legacy_apply_matrix import ( + LegacyQRFApplyMatrixKernel, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + LegacyQRFTrainKernel, + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.fit.model_input import ( + RECIPIENT_MATRIX_TYPE, + decode_recipient_matrix, + encode_recipient_matrix, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + NumericScope, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + load_source, + platform_fingerprint, + run_graph, +) + + +@pytest.fixture(autouse=True) +def serial(monkeypatch): + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + + +def population(values, ids): + table = values.copy() + table.insert(0, "household_id", ids) + return Frame( + { + "household": table, + "person": pd.DataFrame({"person_id": ids, "person_household_id": ids}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(len(ids)), WeightKind.DESIGN)}, + ) + + +def frames(): + x = np.arange(42, dtype=float) % 7 + donor = population( + pd.DataFrame( + { + "x": x, + "indicator": np.arange(42, dtype=float) % 2, + "y": 1 + x, + "z": 2 + x * x, + } + ), + np.arange(42, dtype=np.int64) + 10, + ) + recipient = population( + pd.DataFrame({"x": [1.0, 5.0, 2.0], "indicator": [0.0, 1.0, 0.0]}), + np.array([11, 97, 301], dtype=np.int64), + ) + return donor, recipient + + +class Source(KernelBase): + ref = "test.matrix.source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult( + frame=load_source("frame-store", context.sources[context.node.sources[0]]) + ) + + +class Prepare(KernelBase): + ref = "test.matrix.prepare@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE + ) + + def run(self, context): + table = context.tables["household"] + ids = table.household_id.to_numpy() + features = table[["x", "indicator"]].copy() + features.index = pd.Index(ids) + return KernelResult( + artifacts={ + "matrix": encode_recipient_matrix( + features, entity="household", entity_ids=ids + ) + } + ) + + +def setup(tmp_path, donor, recipient): + store = ContentStore(tmp_path) + sources, source_nodes = {}, [] + for name, frame in (("donor", donor), ("recipient", recipient)): + table = frame.table("household") + key = hashlib.sha256( + table.to_json(orient="table").encode() + + frame.resolve_weights("household").values.tobytes() + ).hexdigest() + sources[name] = store.put_frame(key, frame) + source_nodes.append( + Node( + name, + Source.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("household", col, "float64") + for col in table + if col != "household_id" + ), + ) + ) + fits = legacy_qrf_train_nodes( + "fit", + population="donor", + entity="household", + predictors=("x", "indicator"), + targets=("y", "z"), + seed=31, + n_estimators=2, + phase="early_gap_fill", + ) + matrix = Node( + "matrix", + Prepare.ref, + population="recipient", + inputs=(Slice("household", ("x", "indicator")),), + artifact_outputs=(ArtifactOutput("matrix", RECIPIENT_MATRIX_TYPE),), + ) + applies = legacy_qrf_apply_matrix_nodes( + "apply", + population="recipient", + fit_nodes=fits, + matrix_producer="matrix", + matrix_artifact="matrix", + seed=31, + phase="early_gap_fill", + ) + graph = Graph( + "us", + (SourceRef("donor", "frame-store"), SourceRef("recipient", "frame-store")), + (*source_nodes, matrix, *fits, *applies), + ) + kernels = KernelRegistry() + for kernel in ( + Source(), + Prepare(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + return compile_graph(graph), store, kernels, sources, fits, applies + + +def context_for(manifest, store, node): + artifacts = {} + for edge in node.artifact_inputs: + producer = manifest.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + store.load_bytes(key), + edge.type, + key, + producer.key, + NumericScope(Numeric.PLATFORM_BITWISE, platform=platform_fingerprint()), + ) + return KernelContext( + node, + {}, + {}, + pd.Series([], dtype=str), + node.params, + np.random.default_rng(0), + artifacts=artifacts, + ) + + +def test_real_matrix_chain_matches_legacy_and_replays_without_population_index_change( + tmp_path, +): + donor, recipient = frames() + compiled, store, kernels, sources, fits, applies = setup(tmp_path, donor, recipient) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + matrix = decode_recipient_matrix( + store.load_bytes(manifest.node("matrix").opaque_artifacts["matrix"]) + ) + assert isinstance( + manifest.population("recipient").table("household").index, pd.RangeIndex + ) + assert type(matrix.features.index) is pd.Index + model = QRF(seed=31, n_estimators=2) + state = model.start_chain(donor, ["x", "indicator"], ["y", "z"], weights="design") + raw = pd.DataFrame(index=matrix.features.index) + for node in applies: + expected = model.fit_draw_next( + donor, matrix.features, raw, state=state, weights="design" + ) + payload = store.load_bytes(manifest.node(node.id).opaque_artifacts["raw_draw"]) + np.testing.assert_array_equal( + codec.read_raw_target(payload, target=expected.target, index=raw.index), + expected.raw_draw, + ) + with pytest.raises(ValueError): + codec.read_raw_target( + payload, + target=expected.target, + index=recipient.table("household").index, + ) + packet = decode_matrix_apply_state( + store.load_bytes(manifest.node(node.id).opaque_artifacts["apply_state"]) + ) + assert packet["application"]["state"] == expected.state.to_dict() + assert packet["matrix_producer_key"] == manifest.node("matrix").key + state = expected.state + raw[expected.target] = expected.raw_draw + warm = run_graph( + compiled, + sources=sources, + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert warm.key == manifest.key and all(node.hit for node in warm.nodes.values()) + for node in applies: + for name in ("raw_draw", "apply_state"): + assert store.load_bytes( + warm.node(node.id).opaque_artifacts[name] + ) == store.load_bytes(manifest.node(node.id).opaque_artifacts[name]) + + +@pytest.mark.parametrize( + "mutation", ("features", "membership", "weights", "source_pin") +) +def test_recipient_mutations_reuse_fits_but_rekey_matrix_chain( + tmp_path, monkeypatch, mutation +): + import microcosm.fit.graph_legacy_train as train + + donor, recipient = frames() + compiled, store, kernels, sources, fits, applies = setup(tmp_path, donor, recipient) + calls = [] + original = train.fit_target + + def spy(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(train, "fit_target", spy) + first = run_graph(compiled, sources=sources, store=store, kernels=kernels) + assert len(calls) == 2 + if mutation == "features": + recipient.table("household").loc[0, "x"] += 1 + elif mutation == "membership": + recipient = population( + pd.DataFrame({"x": [1.0, 2.0], "indicator": [0.0, 1.0]}), + np.array([20, 70], dtype=np.int64), + ) + elif mutation == "weights": + recipient = Frame( + {entity: recipient.table(entity) for entity in recipient.entities}, + recipient.schema, + {"household": Weights(np.full(3, 7.0), WeightKind.DESIGN)}, + ) + else: + recipient.table("household")["unconsumed_pin"] = 7.0 + second_args = setup(tmp_path, donor, recipient) + second = run_graph( + second_args[0], sources=second_args[3], store=store, kernels=kernels + ) + assert len(calls) == 2 + for node in fits: + assert second.node(node.id).hit + assert second.node(node.id).key == first.node(node.id).key + assert ( + second.node(node.id).opaque_artifacts + == first.node(node.id).opaque_artifacts + ) + assert second.node("matrix").key != first.node("matrix").key + for node in applies: + assert not second.node(node.id).hit + assert second.node(node.id).key != first.node(node.id).key + assert ( + second.node(node.id).opaque_artifacts["apply_state"] + != first.node(node.id).opaque_artifacts["apply_state"] + ) + # A weights/pin-only change keeps actual prepared values and raw draws equal, + # while the state still binds the new real producer ancestry. + if mutation in {"weights", "source_pin"}: + assert store.load_bytes( + second.node("matrix").opaque_artifacts["matrix"] + ) == store.load_bytes(first.node("matrix").opaque_artifacts["matrix"]) + assert all( + store.load_bytes(second.node(n.id).opaque_artifacts["raw_draw"]) + == store.load_bytes(first.node(n.id).opaque_artifacts["raw_draw"]) + for n in applies + ) + required = run_graph( + second_args[0], + sources=second_args[3], + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert all(node.hit for node in required.nodes.values()) + + +def test_declaration_pins_one_matrix_across_chain(tmp_path): + donor, recipient = frames() + _, _, _, _, fits, applies = setup(tmp_path, donor, recipient) + for node in applies: + assert node.population == "recipient" + assert not node.inputs and not node.outputs and not node.sources + assert node.params["phase"] == "early_gap_fill" + edge = [edge for edge in node.artifact_inputs if edge.name == "matrix"] + assert edge == [ + ArtifactInput("matrix", "matrix", "matrix", RECIPIENT_MATRIX_TYPE) + ] + with pytest.raises(ValueError, match="complete target chain"): + legacy_qrf_apply_matrix_nodes( + "bad", + population="recipient", + fit_nodes=fits[:1], + matrix_producer="matrix", + seed=31, + phase="early_gap_fill", + ) + + +@pytest.mark.parametrize( + "mutation", + ( + "values", + "rows", + "predictor_order", + "predictor_name", + "entity", + "producer", + "state_producer", + "state_sha", + "raw_prior", + "model_history", + "draw_seed", + "missing_prior", + "state_type", + "truncated_matrix", + "matrix_platform", + "phase", + ), +) +def test_matrix_route_refuses_changed_chain_bindings(tmp_path, mutation): + donor, recipient = frames() + compiled, store, kernels, sources, _, applies = setup(tmp_path, donor, recipient) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + context = context_for(manifest, store, applies[1]) + artifacts = dict(context.artifacts) + value = artifacts["matrix"] + decoded = decode_recipient_matrix(value.payload) + if mutation in {"values", "rows", "predictor_order", "predictor_name", "entity"}: + features = decoded.features.copy() + entity = decoded.entity + if mutation == "values": + features.iloc[0, 0] += 1 + elif mutation == "rows": + features = features.iloc[::-1] + elif mutation == "predictor_order": + features = features[["indicator", "x"]] + elif mutation == "predictor_name": + features = features.rename(columns={"x": "other"}) + else: + entity = "tax_unit" + payload = encode_recipient_matrix( + features, entity=entity, entity_ids=features.index.to_numpy() + ) + artifacts["matrix"] = replace(value, payload=payload) + elif mutation == "producer": + artifacts["matrix"] = replace(value, producer_key="f" * 64) + elif mutation in {"state_producer", "state_sha", "model_history"}: + packet = decode_matrix_apply_state(artifacts["apply_state"].payload) + if mutation == "model_history": + packet["application"]["models"][0]["sha256"] = "f" * 64 + else: + packet[ + "matrix_producer_key" + if mutation == "state_producer" + else "matrix_sha256" + ] = "f" * 64 + artifacts["apply_state"] = replace( + artifacts["apply_state"], payload=codec.encode_json(packet) + ) + elif mutation == "raw_prior": + raw = codec.read_raw_target( + artifacts["prior_000"].payload, target="y", index=decoded.features.index + ).copy() + raw[0] += 1 + artifacts["prior_000"] = replace( + artifacts["prior_000"], + payload=codec.encode_raw_target( + raw, target="y", index=decoded.features.index + ), + ) + elif mutation == "draw_seed": + context = replace(context, params={**context.params, "seed": 67}) + elif mutation == "missing_prior": + del artifacts["prior_000"] + elif mutation == "state_type": + artifacts["apply_state"] = replace( + artifacts["apply_state"], type=codec.APPLY_STATE_TYPE + ) + elif mutation == "truncated_matrix": + artifacts["matrix"] = replace(value, payload=value.payload[:-1]) + elif mutation == "matrix_platform": + artifacts["matrix"] = replace( + value, numerics=NumericScope(Numeric.PLATFORM_BITWISE, platform="foreign") + ) + else: + context = replace(context, params={**context.params, "phase": ""}) + with pytest.raises(ValueError): + LegacyQRFApplyMatrixKernel().run(replace(context, artifacts=artifacts)) + + +@pytest.mark.parametrize( + "mutation", ("model_bytes", "model_producer", "envelope_platform") +) +def test_matrix_model_refusal_precedes_unpickle(tmp_path, monkeypatch, mutation): + import json + + from microcosm.fit import qrf_target + + donor, recipient = frames() + compiled, store, kernels, sources, _, applies = setup(tmp_path, donor, recipient) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + context = context_for(manifest, store, applies[0]) + artifacts = dict(context.artifacts) + value = artifacts["model"] + if mutation == "model_bytes": + artifacts["model"] = replace(value, payload=value.payload + b"foreign") + elif mutation == "model_producer": + artifacts["model"] = replace(value, producer_key="f" * 64) + else: + body = value.payload[len(qrf_target._MAGIC) :] + length = int.from_bytes(body[:8], "big") + header = json.loads(body[8 : 8 + length]) + header["implementation"]["platform"] = "foreign" + encoded = codec.encode_json(header) + payload = ( + qrf_target._MAGIC + + len(encoded).to_bytes(8, "big") + + encoded + + body[8 + length :] + ) + artifacts["model"] = replace(value, payload=payload) + packet = codec.decode_json(artifacts["training_state"].payload) + packet["models"][0]["sha256"] = codec.sha(payload) + artifacts["training_state"] = replace( + artifacts["training_state"], payload=codec.encode_json(packet) + ) + + def forbidden(_): + pytest.fail("unpickle ran before model envelope validation") + + monkeypatch.setattr(qrf_target.pickle, "loads", forbidden) + with pytest.raises(ValueError): + LegacyQRFApplyMatrixKernel().run(replace(context, artifacts=artifacts)) + + +@pytest.mark.parametrize( + "field,value", + ( + ("schema_version", True), + ("matrix_sha256", "bad"), + ("matrix_producer_key", "bad"), + ("application", {}), + ), +) +def test_public_matrix_state_parser_refuses_malformed_packets(tmp_path, field, value): + donor, recipient = frames() + compiled, store, kernels, sources, _, applies = setup(tmp_path, donor, recipient) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + packet = decode_matrix_apply_state( + store.load_bytes(manifest.node(applies[0].id).opaque_artifacts["apply_state"]) + ) + packet[field] = value + with pytest.raises(ValueError): + decode_matrix_apply_state(codec.encode_json(packet)) + + +def test_both_kernel_keys_change_when_shared_implementation_changes( + tmp_path, monkeypatch +): + from microcosm.fit import _graph_legacy_apply as shared + from microcosm.fit import model_input + from microcosm.fit.graph_legacy_apply import LegacyQRFApplyKernel + + original = shared.__file__ + source = tmp_path / "shared.py" + source.write_text(open(original).read() + "\n# implementation identity probe\n") + kernels = (LegacyQRFApplyKernel(), LegacyQRFApplyMatrixKernel()) + before = [kernel.implementation_hash() for kernel in kernels] + monkeypatch.setattr(shared, "__file__", str(source)) + after = [kernel.implementation_hash() for kernel in kernels] + assert all(left != right for left, right in zip(before, after, strict=True)) + monkeypatch.setattr(shared, "__file__", original) + matrix_source = tmp_path / "matrix.py" + matrix_source.write_text( + open(model_input.__file__).read() + "\n# codec identity probe\n" + ) + monkeypatch.setattr(model_input, "__file__", str(matrix_source)) + assert kernels[0].implementation_hash() == before[0] + assert kernels[1].implementation_hash() != before[1] diff --git a/packages/microcosm-fit/tests/test_graph_legacy_qrf.py b/packages/microcosm-fit/tests/test_graph_legacy_qrf.py new file mode 100644 index 000000000..9766646fc --- /dev/null +++ b/packages/microcosm-fit/tests/test_graph_legacy_qrf.py @@ -0,0 +1,555 @@ +"""Graph-native target chains preserve the existing weighted QRF protocol.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.fit import QRF +from microcosm.fit.graph_legacy_qrf import ( + LegacyQRFApplyKernel, + LegacyQRFTrainKernel, + legacy_qrf_apply_nodes, + legacy_qrf_train_nodes, + read_raw_target, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + NumericScope, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + load_source, + platform_fingerprint, + run_graph, +) + + +@pytest.fixture(autouse=True) +def serial_workers(monkeypatch): + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + + +def model_frame(values, *, entity="tax_unit", weights=None): + values = values.copy() + ids = np.arange(len(values), dtype=np.int64) + 10 + values.insert(0, f"{entity}_id", ids) + if entity == "person": + values.insert(1, "person_model_unit_id", ids) + schema = EntitySchema(group_entities=("model_unit",)) + tables = {"person": values, "model_unit": pd.DataFrame({"model_unit_id": ids})} + else: + schema = EntitySchema(group_entities=(entity,)) + tables = { + entity: values, + "person": pd.DataFrame({"person_id": ids, f"person_{entity}_id": ids}), + } + return Frame( + tables, + schema, + { + entity: Weights( + np.ones(len(values)) if weights is None else weights, WeightKind.DESIGN + ) + }, + ) + + +def fixture_frames(*, index=None, entity="tax_unit"): + x = np.arange(42, dtype=float) % 7 + donor = pd.DataFrame( + {"x": x, "y": 1.0 + x + np.arange(42) % 3, "z": 3.0 + x * x + np.arange(42) % 5} + ) + recipient = pd.DataFrame({"x": [1.0, 5.0, 2.0, 6.0]}, index=index) + return model_frame( + donor, entity=entity, weights=1.0 + np.arange(42) % 4 + ), model_frame(recipient, entity=entity) + + +class SourceKernel(KernelBase): + ref = "test.legacy.source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult( + frame=load_source("frame-store", context.sources[context.node.sources[0]]) + ) + + +def source_node(name, frame): + return Node( + name, + SourceKernel.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(entity, column, str(table[column].dtype)) + for entity in frame.entities + for table in (frame.table(entity),) + for column in table + if not column.endswith("_id") + ), + ) + + +def source_path(store, frame): + digest = hashlib.sha256() + for entity in frame.entities: + digest.update(entity.encode()) + digest.update(frame.table(entity).to_json(orient="table").encode()) + for entity in frame.weighted_entities: + digest.update(frame.weights_for(entity).values.tobytes()) + return store.put_frame(digest.hexdigest(), frame) + + +def setup_graph( + tmp_path, donor, recipient, *, seed=31, draw_seed=None, predictors=("x",) +): + store = ContentStore(tmp_path) + entity = next(entity for entity in donor.weighted_entities) + fits = legacy_qrf_train_nodes( + "fit", + population="donor", + entity=entity, + predictors=predictors, + targets=("y", "z"), + seed=seed, + n_estimators=2, + phase="early_gap_fill", + ) + applies = legacy_qrf_apply_nodes( + "apply", + population="recipient", + fit_nodes=fits, + seed=seed if draw_seed is None else draw_seed, + phase="early_gap_fill", + ) + graph = Graph( + country="us", + sources=( + SourceRef("donor", "frame-store"), + SourceRef("recipient", "frame-store"), + ), + nodes=( + source_node("donor", donor), + source_node("recipient", recipient), + *fits, + *applies, + ), + ) + kernels = KernelRegistry() + for kernel in (SourceKernel(), LegacyQRFTrainKernel(), LegacyQRFApplyKernel()): + kernels.register(kernel) + sources = { + "donor": source_path(store, donor), + "recipient": source_path(store, recipient), + } + return compile_graph(graph), store, kernels, sources, fits, applies + + +@pytest.mark.parametrize("entity", ("person", "tax_unit")) +@pytest.mark.parametrize( + "index", + ( + pd.Index([11, 19, 31, 97], name="original_row"), + pd.Index(["alpha", "beta", "gamma", "delta"], name="original_row"), + ), +) +def test_exact_legacy_parity_and_fresh_store_restart(tmp_path, entity, index): + donor, recipient = fixture_frames(index=index, entity=entity) + compiled, store, kernels, sources, fits, applies = setup_graph( + tmp_path, donor, recipient + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + model = QRF(seed=31, n_estimators=2) + state = model.start_chain(donor, ["x"], ["y", "z"], weights="design") + raw = pd.DataFrame(index=recipient.table(entity).index) + for fit, apply in zip(fits, applies, strict=True): + expected = model.fit_draw_next( + donor, recipient, raw, state=state, weights="design" + ) + actual = read_raw_target( + store.load_bytes(manifest.node(apply.id).opaque_artifacts["raw_draw"]), + target=expected.target, + index=raw.index, + ) + np.testing.assert_array_equal(actual, expected.raw_draw) + state = expected.state + raw[expected.target] = expected.raw_draw + checkpoint = json.loads( + store.load_bytes(manifest.node(apply.id).opaque_artifacts["apply_state"]) + ) + assert checkpoint["state"] == state.to_dict() + training = json.loads( + store.load_bytes(manifest.node(fit.id).opaque_artifacts["training_state"]) + ) + assert "recipient_index" not in training["state"] + assert "draw_rng_state" not in training["state"] + # Reopen the store rather than reusing any in-memory populations/models. + again = run_graph( + compiled, + sources=sources, + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert again.key == manifest.key + assert all(node.hit for node in again.nodes.values()) + + +@pytest.mark.parametrize("entity", ("person", "tax_unit")) +def test_unsupported_multiindex_refuses_without_silent_index_conversion( + tmp_path, entity +): + index = pd.MultiIndex.from_tuples( + [("a", 1), ("a", 2), ("b", 1), ("b", 2)], names=("arm", "row") + ) + donor, recipient = fixture_frames(index=index, entity=entity) + with pytest.raises(TypeError, match="ContentStore does not support MultiIndex"): + setup_graph(tmp_path, donor, recipient) + assert recipient.table(entity).index.equals(index) + + +@pytest.mark.parametrize( + "mutation", ("membership", "weights", "source_pin", "draw_seed") +) +def test_recipient_changes_reuse_every_fitted_target(tmp_path, monkeypatch, mutation): + import microcosm.fit.graph_legacy_train as train_module + + donor, recipient = fixture_frames() + args = setup_graph(tmp_path, donor, recipient) + compiled, store, kernels, sources, fits, applies = args + calls = [] + original = train_module.fit_target + + def spy(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(train_module, "fit_target", spy) + first = run_graph(compiled, sources=sources, store=store, kernels=kernels) + assert len(calls) == 2 + training_only = Graph( + country="us", + sources=(SourceRef("donor", "frame-store"),), + nodes=(source_node("donor", donor), *fits), + ) + pinned_fits = run_graph( + compile_graph(training_only), + sources={"donor": sources["donor"]}, + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert all(node.hit for node in pinned_fits.nodes.values()) + assert all( + pinned_fits.node(node.id).key == first.node(node.id).key for node in fits + ) + if mutation == "membership": + recipient = model_frame( + pd.DataFrame({"x": [1.0, 3.0, 6.0]}, index=[101, 102, 103]) + ) + elif mutation == "weights": + recipient = model_frame( + recipient.table("tax_unit")[["x"]], weights=np.full(4, 9.0) + ) + elif mutation == "source_pin": + recipient.table("tax_unit")["unconsumed_source_receipt"] = 7 + second_args = setup_graph( + tmp_path, donor, recipient, draw_seed=67 if mutation == "draw_seed" else None + ) + second = run_graph( + second_args[0], sources=second_args[3], store=store, kernels=kernels + ) + assert len(calls) == 2 + for node in fits: + assert second.node(node.id).key == first.node(node.id).key + assert ( + second.node(node.id).opaque_artifacts["model"] + == first.node(node.id).opaque_artifacts["model"] + ) + assert second.node(node.id).hit + assert all(not second.node(node.id).hit for node in applies) + required = run_graph( + second_args[0], + sources=second_args[3], + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert all(node.hit for node in required.nodes.values()) + + +@pytest.mark.parametrize( + "mutation", + ("predictor", "observed_prior", "weights", "pattern_seed", "optional_pattern"), +) +def test_real_donor_or_fit_policy_change_invalidates_training(tmp_path, mutation): + donor, recipient = fixture_frames() + compiled, store, kernels, sources, fits, _ = setup_graph(tmp_path, donor, recipient) + first = run_graph(compiled, sources=sources, store=store, kernels=kernels) + seed = 31 + predictors = ("x",) + if mutation == "predictor": + donor.table("tax_unit").loc[0, "x"] += 20 + elif mutation == "observed_prior": + donor.table("tax_unit").loc[0, "y"] += 20 + elif mutation == "weights": + donor = model_frame( + donor.table("tax_unit")[["x", "y", "z"]], weights=np.full(42, 2.0) + ) + elif mutation == "pattern_seed": + seed = 71 + elif mutation == "optional_pattern": + donor.table("tax_unit")["optional"] = np.arange(42) % 2 + recipient.table("tax_unit")["optional"] = np.arange(4) % 2 + predictors = ("x", "optional") + second_args = setup_graph( + tmp_path, donor, recipient, seed=seed, predictors=predictors + ) + second = run_graph( + second_args[0], sources=second_args[3], store=store, kernels=kernels + ) + for node in fits: + assert second.node(node.id).key != first.node(node.id).key + assert not second.node(node.id).hit + + +def apply_context(manifest, store, node, frame): + entity = node.inputs[0].entity + scope = NumericScope(Numeric.PLATFORM_BITWISE, platform=platform_fingerprint()) + artifacts = {} + for item in node.artifact_inputs: + producer = manifest.node(item.producer) + key = producer.opaque_artifacts[item.artifact] + artifacts[item.name] = ArtifactValue( + store.load_bytes(key), item.type, key, producer.key, scope + ) + return KernelContext( + node=node, + tables={entity: frame.table(entity)}, + weights={entity: frame.resolve_weights(entity)}, + strata=frame.strata, + params=node.params, + rng=np.random.default_rng(0), + artifacts=artifacts, + ) + + +@pytest.mark.parametrize( + "mutation", + ( + "raw_bytes", + "raw_target", + "row_order", + "draw_seed", + "model_history", + "missing_prior", + ), +) +def test_apply_refuses_changed_raw_prefix_or_chain_identity(tmp_path, mutation): + from microcosm.fit import _graph_legacy_qrf as codec + + donor, recipient = fixture_frames(index=pd.Index([11, 19, 31, 97], name="row")) + compiled, store, kernels, sources, _, applies = setup_graph( + tmp_path, donor, recipient + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + context = apply_context(manifest, store, applies[1], recipient) + artifacts = dict(context.artifacts) + if mutation in {"raw_bytes", "raw_target"}: + raw = read_raw_target( + artifacts["prior_000"].payload, + target="y", + index=recipient.table("tax_unit").index, + ).copy() + raw[0] += 1 + payload = codec.encode_raw_target( + raw, + target="z" if mutation == "raw_target" else "y", + index=recipient.table("tax_unit").index, + ) + artifacts["prior_000"] = replace(artifacts["prior_000"], payload=payload) + elif mutation == "row_order": + context = replace( + context, tables={"tax_unit": recipient.table("tax_unit").iloc[::-1]} + ) + elif mutation == "draw_seed": + context = replace(context, params={**context.params, "seed": 67}) + elif mutation == "model_history": + state = codec.decode_json(artifacts["apply_state"].payload) + state["models"][0]["sha256"] = "f" * 64 + artifacts["apply_state"] = replace( + artifacts["apply_state"], payload=codec.encode_json(state) + ) + elif mutation == "missing_prior": + del artifacts["prior_000"] + with pytest.raises(ValueError): + LegacyQRFApplyKernel().run(replace(context, artifacts=artifacts)) + + +@pytest.mark.parametrize( + "mutation", + ("model_bytes", "model_platform", "model_producer", "envelope_implementation"), +) +def test_graph_model_validation_precedes_unpickle(tmp_path, monkeypatch, mutation): + from microcosm.fit import _graph_legacy_qrf as codec + from microcosm.fit import qrf_target + + donor, recipient = fixture_frames() + compiled, store, kernels, sources, _, applies = setup_graph( + tmp_path, donor, recipient + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + context = apply_context(manifest, store, applies[0], recipient) + artifacts = dict(context.artifacts) + value = artifacts["model"] + if mutation == "model_bytes": + artifacts["model"] = replace(value, payload=value.payload + b"bad") + elif mutation == "model_platform": + artifacts["model"] = replace( + value, + numerics=NumericScope(Numeric.PLATFORM_BITWISE, platform="other-platform"), + ) + elif mutation == "model_producer": + artifacts["model"] = replace(value, producer_key="f" * 64) + else: + # Update the trusted sibling digest to reach the inner envelope check. + body = value.payload[len(qrf_target._MAGIC) :] + length = int.from_bytes(body[:8], "big") + metadata = json.loads(body[8 : 8 + length]) + metadata["implementation"]["platform"] = "other-platform" + encoded = json.dumps(metadata, sort_keys=True, separators=(",", ":")).encode() + payload = ( + qrf_target._MAGIC + + len(encoded).to_bytes(8, "big") + + encoded + + body[8 + length :] + ) + artifacts["model"] = replace(value, payload=payload) + packet = codec.decode_json(artifacts["training_state"].payload) + packet["models"][0]["sha256"] = codec.sha(payload) + artifacts["training_state"] = replace( + artifacts["training_state"], payload=codec.encode_json(packet) + ) + + def forbidden(_): + pytest.fail("unpickle ran before envelope validation") + + monkeypatch.setattr(qrf_target.pickle, "loads", forbidden) + with pytest.raises(ValueError): + LegacyQRFApplyKernel().run(replace(context, artifacts=artifacts)) + + +def test_zero_target_keeps_legacy_no_draw_rng_transition(tmp_path): + donor, recipient = fixture_frames() + donor.table("tax_unit")["y"] = 0.0 + donor.table("tax_unit")["z"] = 2.5 + compiled, store, kernels, sources, _, applies = setup_graph( + tmp_path, donor, recipient + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + first = json.loads( + store.load_bytes(manifest.node(applies[0].id).opaque_artifacts["apply_state"]) + ) + state = QRF(seed=31, n_estimators=2).start_chain( + donor, ["x"], ["y", "z"], weights="design" + ) + assert first["state"]["draw_rng_state"] == state.to_dict()["draw_rng_state"] + raw = read_raw_target( + store.load_bytes(manifest.node(applies[0].id).opaque_artifacts["raw_draw"]), + target="y", + index=recipient.table("tax_unit").index, + ) + np.testing.assert_array_equal(raw, np.zeros(4)) + + +def test_application_seed_controls_the_actual_second_seedsequence_stream(tmp_path): + from microcosm.fit import qrf + + donor, recipient = fixture_frames() + compiled, store, kernels, sources, _, applies = setup_graph( + tmp_path, donor, recipient, draw_seed=67 + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + model = QRF(seed=31, n_estimators=2) + initial = model.start_chain(donor, ["x"], ["y", "z"], weights="design") + _, draw_seed = np.random.SeedSequence(67).spawn(2) + state = replace( + initial, + draw_rng_state_json=qrf._rng_state_json(np.random.default_rng(draw_seed)), + ) + raw = pd.DataFrame(index=recipient.table("tax_unit").index) + for apply in applies: + expected = model.fit_draw_next( + donor, recipient, raw, state=state, weights="design" + ) + actual = read_raw_target( + store.load_bytes(manifest.node(apply.id).opaque_artifacts["raw_draw"]), + target=expected.target, + index=raw.index, + ) + np.testing.assert_array_equal(actual, expected.raw_draw) + packet = json.loads( + store.load_bytes(manifest.node(apply.id).opaque_artifacts["apply_state"]) + ) + assert packet["state"] == expected.state.to_dict() + state = expected.state + raw[expected.target] = expected.raw_draw + + +@pytest.mark.parametrize("case", ("positive", "hurdle", "zero_constant")) +def test_slice_adapter_retains_pre_refactor_raw_and_state_bytes(tmp_path, case): + from microcosm.fit import _graph_legacy_qrf as codec + + golden = json.loads( + (Path(__file__).parent / "golden/legacy-slice-apply-v1.json").read_text() + ) + donor, recipient = fixture_frames(index=pd.Index([11, 19, 31, 97], name="row")) + if case == "hurdle": + donor.table("tax_unit").loc[:20, "y"] = 0.0 + elif case == "zero_constant": + donor.table("tax_unit")["y"] = 0.0 + donor.table("tax_unit")["z"] = 2.5 + compiled, store, kernels, sources, fits, applies = setup_graph( + tmp_path, donor, recipient + ) + manifest = run_graph(compiled, sources=sources, store=store, kernels=kernels) + for fit, apply, captured in zip(fits, applies, golden["cases"][case], strict=True): + assert store.load_bytes( + manifest.node(apply.id).opaque_artifacts["raw_draw"] + ) == bytes.fromhex(captured["raw_draw"]) + # Forest identities are platform/version-bound. Only those independently + # supplied input records vary; every state/RNG/raw-history byte is frozen. + expected = codec.decode_json(bytes.fromhex(captured["apply_state"])) + inputs = codec.decode_json( + store.load_bytes(manifest.node(fit.id).opaque_artifacts["training_state"]) + ) + assert [item["target"] for item in expected["models"]] == [ + item["target"] for item in inputs["models"] + ] + expected["models"] = inputs["models"] + assert store.load_bytes( + manifest.node(apply.id).opaque_artifacts["apply_state"] + ) == codec.encode_json(expected) diff --git a/packages/microcosm-fit/tests/test_qrf_stateless.py b/packages/microcosm-fit/tests/test_qrf_stateless.py new file mode 100644 index 000000000..3d099a70d --- /dev/null +++ b/packages/microcosm-fit/tests/test_qrf_stateless.py @@ -0,0 +1,159 @@ +"""Caller-owned uniforms make repeated QRF draws stable by identity.""" + +import copy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.fit import fit +from microcosm.fit.qrf import Regime + + +@pytest.fixture(scope="module") +def model(): + x = np.tile(np.arange(30, dtype=float), 6) + donor = pd.DataFrame( + { + "x": x, + "positive": x + 1, + "negative": -x - 1, + "zero": np.zeros(len(x)), + "mixed": np.tile([-2.0, 0.0, 3.0], len(x) // 3), + "inflated": np.tile([0.0, 4.0], len(x) // 2), + "negative_inflated": np.tile([0.0, -4.0], len(x) // 2), + "two_sign": np.tile([-3.0, 4.0], len(x) // 2), + } + ) + return fit( + donor, + ["x"], + list(donor.columns[1:]), + weights="none", + n_estimators=4, + seed=7, + ) + + +def uniforms(model, n): + return { + "quantiles": {t: np.linspace(0, 0.99, n) for t in model.targets}, + "sign_uniforms": {t: np.linspace(0.99, 0, n) for t in model.targets}, + } + + +def test_stateless_draws_preserve_rng_and_forests(model): + recipient = pd.DataFrame({"x": np.arange(20, dtype=float)}) + before = copy.deepcopy(model._rng.bit_generator.state) + draws = uniforms(model, len(recipient)) + first = model.predict_from_uniforms(recipient, **draws) + pd.testing.assert_frame_equal( + first, model.predict_from_uniforms(recipient, **draws) + ) + assert model._rng.bit_generator.state == before + assert (first.positive > 0).all() + assert (first.negative < 0).all() + assert (first.zero == 0).all() + assert set(first.mixed) == {-2.0, 0.0, 3.0} + assert set(first.inflated) == {0.0, 4.0} + assert set(first.negative_inflated) == {0.0, -4.0} + assert set(first.two_sign) == {-3.0, 4.0} + + +def test_permutation_and_chunking_preserve_chained_draws(model): + recipient = pd.DataFrame({"x": np.arange(20, dtype=float)}) + draws = uniforms(model, len(recipient)) + expected = model.predict_from_uniforms(recipient, **draws) + order = np.random.default_rng(4).permutation(len(recipient)) + reordered = model.predict_from_uniforms( + recipient.iloc[order], + **{k: {t: v[order] for t, v in d.items()} for k, d in draws.items()}, + ) + pd.testing.assert_frame_equal(expected, reordered.sort_index()) + chunks = [] + for rows in (slice(0, 7), slice(7, 20)): + chunks.append( + model.predict_from_uniforms( + recipient.iloc[rows], + **{k: {t: v[rows] for t, v in d.items()} for k, d in draws.items()}, + ) + ) + pd.testing.assert_frame_equal(expected, pd.concat(chunks)) + + +@pytest.mark.parametrize("bad", [[-0.1, 0.5], [0.1, 1.0], [np.nan, 0.1], [0.1]]) +@pytest.mark.parametrize("field", ["quantiles", "sign_uniforms"]) +def test_invalid_uniforms_rejected_before_drawing(model, bad, field): + recipient = pd.DataFrame({"x": [1.0, 2.0]}) + draws = uniforms(model, 2) + draws[field][model.targets[-1]] = np.array(bad) + with pytest.raises(ValueError, match="uniform|shape"): + model.predict_from_uniforms(recipient, **draws) + + +def test_uniform_target_names_must_match(model): + draws = uniforms(model, 2) + del draws["quantiles"][model.targets[-1]] + with pytest.raises(ValueError, match="targets"): + model.predict_from_uniforms(pd.DataFrame({"x": [1.0, 2.0]}), **draws) + + +def test_empty_recipient_batch(model): + actual = model.predict_from_uniforms( + pd.DataFrame({"x": pd.Series(dtype=float)}), **uniforms(model, 0) + ) + assert list(actual.columns) == model.targets + assert actual.empty + assert all(dtype == np.dtype("float64") for dtype in actual.dtypes) + + +def test_zero_uniform_skips_a_zero_probability_sign(model, monkeypatch): + # Exercise the inverse-CDF boundary that ordinary RNG draws almost never hit. + gate = model._target_models["mixed"].gate + monkeypatch.setattr( + gate, "predict_proba", lambda x: np.tile([0.0, 0.0, 1.0], (len(x), 1)) + ) + draws = uniforms(model, 2) + draws["sign_uniforms"]["mixed"] = np.zeros(2) + actual = model.predict_from_uniforms(pd.DataFrame({"x": [1.0, 2.0]}), **draws) + assert (actual.mixed == 3.0).all() + + +def test_stateless_replays_legacy_uniforms_across_all_regimes(model): + """Ordinary uniforms preserve legacy draws; exact CDF ties differ deliberately. + + The zero-uniform test above pins the intentional strict-CDF boundary. + This independent RNG replay detects swapped or inverted quantile streams. + """ + legacy = copy.deepcopy(model) + recipient = pd.DataFrame({"x": np.arange(20, dtype=float)}) + assert set(legacy.regimes().values()) == { + Regime.POSITIVE_ONLY, + Regime.NEGATIVE_ONLY, + Regime.DEGENERATE_ZERO, + Regime.THREE_SIGN, + Regime.ZERO_INFLATED_POSITIVE, + Regime.ZERO_INFLATED_NEGATIVE, + Regime.SIGN_ONLY, + } + for _ in range(4): + replay = np.random.default_rng() + replay.bit_generator.state = copy.deepcopy(legacy._rng.bit_generator.state) + quantiles, signs = {}, {} + for target in legacy.targets: + target_model = legacy._target_models[target] + quantiles[target] = ( + replay.random(len(recipient)) + if target_model.regime != Regime.DEGENERATE_ZERO + else np.zeros(len(recipient)) + ) + signs[target] = ( + replay.random(len(recipient)) + if target_model.gate is not None + else np.zeros(len(recipient)) + ) + expected = legacy.predict(recipient) + actual = legacy.predict_from_uniforms( + recipient, quantiles=quantiles, sign_uniforms=signs + ) + pd.testing.assert_frame_equal(expected, actual, check_exact=True) diff --git a/packages/microcosm-fit/tests/test_qrf_target.py b/packages/microcosm-fit/tests/test_qrf_target.py new file mode 100644 index 000000000..8bec7a4b2 --- /dev/null +++ b/packages/microcosm-fit/tests/test_qrf_target.py @@ -0,0 +1,373 @@ +"""Real fitted target reuse without changing the legacy QRF estimator/draws.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.fit import RegimeGatedQRF +from microcosm.fit import qrf as legacy +from microcosm.fit import qrf_target as split +from microcosm.graph import Numeric, platform_fingerprint + + +@pytest.fixture(autouse=True) +def serial_workers(monkeypatch): + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + + +@pytest.fixture +def donors(): + rng = np.random.default_rng(7) + n = 90 + positive = np.exp(rng.normal(2, 0.4, n)) + sign = np.resize(np.array([-1.0, 0.0, 1.0]), n) + return pd.DataFrame( + { + "x": rng.normal(size=n), + "zero": np.zeros(n), + "constant": np.full(n, 2.5), + "positive": positive, + "negative": -positive, + "zero_positive": np.where(sign > 0, positive, 0.0), + "zero_negative": np.where(sign < 0, -positive, 0.0), + "signed": np.where(sign > 0, positive, -positive), + "mixed": sign * positive, + "last": positive * positive + rng.uniform(0, 0.2, n), + "weight": np.where(sign > 0, 0.03, 12.0), + }, + index=pd.Index(np.arange(n) + 100, name="donor"), + ) + + +def _model(): + return RegimeGatedQRF(n_estimators=3, seed=29) + + +def _fit_chain(model, donors, initial, *, weights="weight"): + state = split.LegacyQRFTrainingState.from_chain(initial) + artifacts = [] + while not state.is_complete: + artifact = split.fit_target(model, donors, state=state, weights=weights) + artifacts.append(artifact) + # Every fit completes before the first recipient is seen; checkpoints + # have no draw stream or recipient identity, even between targets. + assert "recipient_index" not in state.to_dict() + assert "draw_rng_state" not in state.to_dict() + state = split.LegacyQRFTrainingState.from_dict( + json.loads(json.dumps(artifact.next_training_state.to_dict())) + ) + return artifacts + + +def _apply_chain(artifacts, recipients, initial): + state = initial + raw = pd.DataFrame(index=recipients.index) + for artifact in artifacts: + step = split.apply_target(artifact, recipients, raw, state=state) + assert not step.raw_draw.flags.writeable + raw[step.target] = step.raw_draw + state = step.state + return raw, state + + +@pytest.mark.parametrize("weights", ["weight", "none"]) +def test_all_regimes_and_raw_chain_are_bit_identical(donors, weights): + """Fit all targets first, then roundtrip models and match both old APIs.""" + model = _model() + targets = [column for column in donors if column not in {"x", "weight"}] + initial = model.start_chain(donors, ["x"], targets, weights=weights) + recipients = donors.iloc[::3][["x"]].copy() + artifacts = _fit_chain(model, donors, initial, weights=weights) + restored = [] + for artifact in artifacts: + payload = artifact.to_bytes() + restored.append( + split.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=hashlib.sha256(payload).hexdigest() + ) + ) + assert artifact.numeric is Numeric.PLATFORM_BITWISE + assert artifact.platform == platform_fingerprint() + actual, final = _apply_chain(restored, recipients, initial) + expected = model.fit(donors, ["x"], targets, weights=weights).predict(recipients) + np.testing.assert_array_equal(actual.to_numpy(), expected.to_numpy()) + old_state = initial + old_raw = pd.DataFrame(index=recipients.index) + for artifact in artifacts: + old = model.fit_draw_next( + donors, recipients, old_raw, state=old_state, weights=weights + ) + np.testing.assert_array_equal(actual[old.target], old.raw_draw) + old_raw[old.target] = old.raw_draw + old_state = old.state + if artifact.target == "zero": + assert old.state.fit_rng_state_json == initial.fit_rng_state_json + assert old.state.draw_rng_state_json == initial.draw_rng_state_json + assert final == old_state + + +def test_model_reuse_excludes_recipient_and_draw_identity(donors, monkeypatch): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive", "last"], weights="weight") + calls = [] + real_fit = RegimeGatedQRF._fit_target + + def count_fit(self, **kwargs): + calls.append(kwargs["columns"]) + return real_fit(self, **kwargs) + + monkeypatch.setattr(RegimeGatedQRF, "_fit_target", count_fit) + artifacts = _fit_chain(model, donors, initial) + assert calls == [("x",), ("x", "positive")] + identity = artifacts[0].training_id + alternate = replace( + initial, + draw_rng_state_json=legacy._rng_state_json(np.random.default_rng(802)), + ) + assert split.LegacyQRFTrainingState.from_chain(initial) == ( + split.LegacyQRFTrainingState.from_chain(alternate) + ) + for state, recipients in ( + (initial, donors.iloc[:21][["x"]]), + (initial, donors.iloc[20:60][["x"]].set_axis(range(500, 540))), + (alternate, donors.iloc[::2][["x"]]), + ): + raw, _ = _apply_chain(artifacts, recipients, state) + # Independent legacy reference can use a different draw checkpoint. + expected = pd.DataFrame(index=recipients.index) + for target in state.targets: + step = model.fit_draw_next( + donors, recipients, expected, state=state, weights="weight" + ) + state = step.state + expected[target] = step.raw_draw + np.testing.assert_array_equal(raw.to_numpy(), expected.to_numpy()) + assert artifacts[0].training_id == identity + assert len(calls) == 2 + 3 * 2 # all six extra fits are legacy references + + +def test_training_identity_binds_consumed_donor_values_and_weights(donors): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive"], weights="weight") + state = split.LegacyQRFTrainingState.from_chain(initial) + first = split.fit_target(model, donors, state=state, weights="weight") + changed = donors.copy() + changed.loc[changed.index[0], "positive"] += 1 + second = split.fit_target(model, changed, state=state, weights="weight") + assert first.training_id != second.training_id + ignored = donors.assign(unused=np.arange(len(donors))) + same = split.fit_target(model, ignored, state=state, weights="weight") + assert first.training_id == same.training_id + assert first.to_bytes() == same.to_bytes() + changed.loc[changed.index[0], "weight"] += 1 + with pytest.raises(ValueError, match="weight values/order changed"): + split.fit_target(model, changed, state=state, weights="weight") + with pytest.raises(ValueError, match="donor index/order changed"): + split.fit_target(model, donors.iloc[::-1], state=state, weights="weight") + + +def test_state_and_raw_prefix_drift_are_refused(donors): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive", "last"], weights="weight") + artifacts = _fit_chain(model, donors, initial) + recipients = donors.iloc[:20][["x"]] + empty = pd.DataFrame(index=recipients.index) + with pytest.raises(ValueError, match="training state"): + split.apply_target(artifacts[1], recipients, empty, state=initial) + changed = replace( + initial, fit_rng_state_json=legacy._rng_state_json(np.random.default_rng(99)) + ) + with pytest.raises(ValueError, match="training state"): + split.apply_target(artifacts[0], recipients, empty, state=changed) + first = split.apply_target(artifacts[0], recipients, empty, state=initial) + raw = pd.DataFrame({"positive": first.raw_draw}, index=recipients.index) + with pytest.raises(ValueError, match="recipient index/order changed"): + split.apply_target( + artifacts[1], recipients.iloc[::-1], raw.iloc[::-1], state=first.state + ) + with pytest.raises(ValueError, match="float64"): + split.apply_target( + artifacts[1], recipients, raw.astype("float32"), state=first.state + ) + with pytest.raises(ValueError, match="exact completed target prefix"): + split.apply_target(artifacts[1], recipients, empty, state=first.state) + last = split.apply_target(artifacts[1], recipients, raw, state=first.state) + with pytest.raises(ValueError, match="complete"): + split.apply_target(artifacts[1], recipients, raw, state=last.state) + with pytest.raises(ValueError, match="complete"): + split.fit_target( + model, donors, state=artifacts[1].next_training_state, weights="weight" + ) + + +def test_frame_front_door_uses_real_typed_weights(make_person_frame): + x = np.arange(60, dtype=np.float64) + frame = make_person_frame( + {"x": x, "target": x + 1}, weights=np.where(x < 30, 0.02, 12) + ) + model = _model() + initial = model.start_chain(frame, ["x"], ["target"]) + artifact = split.fit_target( + model, frame, state=split.LegacyQRFTrainingState.from_chain(initial) + ) + raw = pd.DataFrame(index=frame.table("person").index) + actual = split.apply_target(artifact, frame, raw, state=initial) + expected = model.fit_draw_next(frame, frame, raw, state=initial) + np.testing.assert_array_equal(actual.raw_draw, expected.raw_draw) + assert actual.weight_kind == "design" + assert actual.state == expected.state + + +def test_fit_uses_observed_priors_apply_uses_raw_draws(donors, monkeypatch): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive", "last"], weights="weight") + fit_features = [] + draw_features = [] + real_fit = RegimeGatedQRF._fit_target + real_draw = legacy._draw_target_with_rng + + def inspect_fit(self, **kwargs): + fit_features.append(kwargs["features"].copy()) + return real_fit(self, **kwargs) + + def inspect_draw(features, target_model, rng): + draw_features.append(features.copy()) + return real_draw(features, target_model, rng) + + monkeypatch.setattr(RegimeGatedQRF, "_fit_target", inspect_fit) + monkeypatch.setattr(legacy, "_draw_target_with_rng", inspect_draw) + artifacts = _fit_chain(model, donors, initial) + recipients = donors.iloc[:23][["x"]] + raw, _ = _apply_chain(artifacts, recipients, initial) + np.testing.assert_array_equal(fit_features[1][:, 1], donors["positive"].to_numpy()) + np.testing.assert_array_equal(draw_features[1]["positive"], raw["positive"]) + assert (raw["positive"] != np.round(raw["positive"])).any() + + +@pytest.mark.parametrize("drift", ["config", "fit_workers"]) +def test_changed_fit_configuration_is_rejected(donors, monkeypatch, drift): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive"], weights="weight") + state = split.LegacyQRFTrainingState.from_chain(initial) + if drift == "config": + model = RegimeGatedQRF(n_estimators=4, seed=29) + else: + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "2") + with pytest.raises(ValueError, match="model configuration changed"): + split.fit_target(model, donors, state=state, weights="weight") + + +@pytest.mark.parametrize("field", ["schema", "next_state", "draw_state", "versions"]) +def test_malformed_envelope_is_rejected_before_unpickle(donors, monkeypatch, field): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive"], weights="weight") + artifact = split.fit_target( + model, + donors, + state=split.LegacyQRFTrainingState.from_chain(initial), + weights="weight", + ) + payload = artifact.to_bytes() + offset = len(split._MAGIC) + size = int.from_bytes(payload[offset : offset + 8], "big") + offset += 8 + metadata = json.loads(payload[offset : offset + size]) + if field == "schema": + metadata["schema_version"] = 2 + elif field == "next_state": + metadata["next_training_state"]["completed_targets"] = [] + elif field == "draw_state": + metadata["training_state"]["draw_rng_state"] = np.random.PCG64(0).state + else: + metadata["implementation"]["versions"]["numpy"] = "different-version" + header = json.dumps(metadata).encode() + changed = ( + split._MAGIC + + len(header).to_bytes(8, "big") + + header + + payload[offset + size :] + ) + calls = [] + monkeypatch.setattr(split.pickle, "loads", lambda value: calls.append(value)) + with pytest.raises(ValueError): + split.LegacyQRFTargetArtifact.from_trusted_bytes( + changed, expected_sha256=hashlib.sha256(changed).hexdigest() + ) + assert not calls + + +def test_integrity_platform_and_implementation_checked_before_unpickle( + donors, monkeypatch +): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive"], weights="weight") + artifact = split.fit_target( + model, + donors, + state=split.LegacyQRFTrainingState.from_chain(initial), + weights="weight", + ) + payload = artifact.to_bytes() + expected = hashlib.sha256(payload).hexdigest() + called = [] + monkeypatch.setattr(split.pickle, "loads", lambda value: called.append(value)) + with pytest.raises(ValueError, match="digest"): + split.LegacyQRFTargetArtifact.from_trusted_bytes( + payload[:-1] + b"!", expected_sha256=expected + ) + with pytest.raises(ValueError, match="digest"): + split.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256="invalid" + ) + original = split._implementation_binding + for key, value in ( + ("platform", "other-architecture/linux/py3.13"), + ("qrf_sha256", "0" * 64), + ): + monkeypatch.setattr( + split, + "_implementation_binding", + lambda key=key, value=value: {**original(), key: value}, + ) + with pytest.raises(ValueError, match="implementation"): + split.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=expected + ) + with pytest.raises(ValueError, match="implementation"): + split.apply_target( + artifact, donors[["x"]], pd.DataFrame(index=donors.index), state=initial + ) + assert called == [] + + +def test_apply_retains_prediction_chunk_ceiling(donors, monkeypatch): + model = _model() + initial = model.start_chain(donors, ["x"], ["positive"], weights="weight") + artifact = split.fit_target( + model, + donors, + state=split.LegacyQRFTrainingState.from_chain(initial), + weights="weight", + ) + sizes = [] + original = legacy.RandomForestQuantileRegressor.predict + + def observe(self, x, *args, **kwargs): + sizes.append(len(x)) + return original(self, x, *args, **kwargs) + + monkeypatch.setattr(legacy.RandomForestQuantileRegressor, "predict", observe) + recipients = pd.DataFrame({"x": np.linspace(-2, 2, 40_001)}) + step = split.apply_target( + artifact, recipients, pd.DataFrame(index=recipients.index), state=initial + ) + assert len(step.raw_draw) == 40_001 + assert sum(sizes) == 40_001 + assert max(sizes) == legacy._PREDICT_CHUNK_ROWS == 10_000 From 93ff498a21d3c3e367b1c94d3fad256875f6c676 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 015/255] Add focused US launch integration regressions and changelog --- PROGRESS.md | 2 + changelog.d/us-launch-integration.fixed.md | 1 + .../tests/test_us_geography_integration.py | 276 ++++++++++++ .../tests/test_us_runtime_facade_union.py | 113 +++++ .../tests/test_solve_us_launch_integration.py | 163 +++++++ ...test_lazy_snapshot_metadata_integration.py | 417 ++++++++++++++++++ 6 files changed, 972 insertions(+) create mode 100644 changelog.d/us-launch-integration.fixed.md create mode 100644 packages/microcosm-build/tests/test_us_geography_integration.py create mode 100644 packages/microcosm-build/tests/test_us_runtime_facade_union.py create mode 100644 packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py create mode 100644 packages/microcosm-graph/tests/test_lazy_snapshot_metadata_integration.py diff --git a/PROGRESS.md b/PROGRESS.md index 78184f6e6..2912cb736 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -761,3 +761,5 @@ Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. diff --git a/changelog.d/us-launch-integration.fixed.md b/changelog.d/us-launch-integration.fixed.md new file mode 100644 index 000000000..170634a55 --- /dev/null +++ b/changelog.d/us-launch-integration.fixed.md @@ -0,0 +1 @@ +Preserve grouped calibration closing-state results alongside ordinary best-iterate selection, retain the combined US runtime facade and joint geography validation, and publish complete metadata for lazy graph snapshots. diff --git a/packages/microcosm-build/tests/test_us_geography_integration.py b/packages/microcosm-build/tests/test_us_geography_integration.py new file mode 100644 index 000000000..7e2521de9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_geography_integration.py @@ -0,0 +1,276 @@ +"""Ordinary invented geography controls for the combined graph contract. + +All NPZ/CSV bytes are created in the test. The national-sized fixture satisfies +the shape validator with invented equal cell masses and district allocations; +it is not a Census crosswalk or evidence of genuine source admission. +""" + +import hashlib +import json +from collections import Counter +from dataclasses import fields +from io import BytesIO + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_geography as geography +from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, +) +from microcosm.build.us_runtime.puma_ladder import ( + assign_us_puma_ladder, + decode_us_puma_ladder, + load_us_puma_ladder, +) +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ArtifactValue, KernelContext, Owned +from microcosm.graph.kernel import NumericScope + + +def _ordinary_lookup_bytes(**overrides): + # The production validator requires 436 districts across 51 jurisdictions. + # Allocate invented counts: 7 at-large + 33*10 + 11*9 = 436. + states = [i for i in range(1, 57) if i not in {3, 7, 14, 43, 52}] + at_large = {2, 10, 11, 38, 46, 50, 56} + multi = [i for i in states if i not in at_large] + rows, crosswalk = [], [] + for state in states: + districts = ( + [0] + if state in at_large + else range(1, 11 if multi.index(state) < 33 else 10) + ) + for district in districts: + county = max(1, district) + if state == 36: + county = 61 if district == 1 else (1 if district == 2 else district) + puma = state * 100_000 + 100 + tract = (state * 1000 + county) * 1_000_000 + 100 + cd = state * 100 + district + rows.append((puma, tract, cd)) + crosswalk.append( + { + "source_geography_id": f"{SOURCE_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "target_geography_id": f"{CURRENT_CONGRESSIONAL_DISTRICT_PREFIX}{cd:04d}", + "weight": 1.0, + "pair_population": 1, + } + ) + rows.sort() + pumas = Counter(row[0] for row in rows) + arrays = { + "puma": np.asarray(sorted(pumas), dtype=np.int64), + "puma_population": np.asarray([pumas[p] for p in sorted(pumas)]), + "joint_overlap_puma": np.asarray([r[0] for r in rows]), + "joint_overlap_tract": np.asarray([r[1] for r in rows]), + "joint_overlap_cd": np.asarray([r[2] for r in rows]), + "joint_overlap_population": np.ones(len(rows), dtype=np.int64), + "metadata_json": np.asarray( + json.dumps( + { + "schema_version": 2, + "kind": "us_puma_ladder", + "puma_vintage": "2020_puma", + "sampling_basis": "population", + "layers": { + name: { + "vintage": vintage, + "source": "invented ordinary fixture", + } + for name, vintage in ( + ("congressional_district", "119th_congress"), + ("county", "2020_census"), + ("tract", "2020_census"), + ) + }, + } + ) + ), + } + for layer, coordinate in (("cd", 2), ("county", 1), ("tract", 1)): + counts = Counter( + ( + row[0], + row[coordinate] // 1_000_000 if layer == "county" else row[coordinate], + ) + for row in rows + ) + keys = sorted(counts) + arrays[f"{layer}_overlap_puma"] = np.asarray([key[0] for key in keys]) + arrays[f"{layer}_overlap_{layer}"] = np.asarray([key[1] for key in keys]) + arrays[f"{layer}_overlap_population"] = np.asarray( + [counts[key] for key in keys] + ) + arrays.update(overrides) + stream = BytesIO() + np.savez_compressed(stream, **arrays) + return stream.getvalue(), pd.DataFrame(crosswalk).to_csv(index=False).encode() + + +def _nodes(): + return geography.us_geography_nodes( + (Owned("person", "age", "int64"), Owned("household", "state_fips", "int64")), + base="ordinary_base", + context_producer="ordinary_context", + ) + + +def _context(household, weights, ladder_bytes, crosswalk_bytes): + artifacts = {} + for alias, payload, type_ in ( + ("ladder", ladder_bytes, geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", crosswalk_bytes, geography.US_CD_CROSSWALK_TYPE), + ): + artifacts[alias] = ArtifactValue( + payload, + type_, + hashlib.sha256(payload).hexdigest(), + "a" * 64, + NumericScope(), + ) + node = _nodes()[-1] + return KernelContext( + node=node, + tables={"household": household}, + weights={"household": Weights(np.asarray(weights), WeightKind.DESIGN)}, + strata=pd.Series(dtype="int64"), + params=node.params, + rng=np.random.default_rng(0), + artifacts=artifacts, + ) + + +def _households(): + # Same ordinary mass control as test_us_puma_ladder._gated_household: + # NYC = 2.6% nationally and 2.6/6.2 of New York's mass. + return pd.DataFrame( + { + "household_id": [1, 2, 3, 4, 5], + "state_fips": [36, 36, 6, 48, 1], + "puma": ["3600100", "3600100", "0600100", "4800100", "0100100"], + "county_fips": ["36061", "36001", "06001", "48001", "01001"], + "congressional_district_geoid": [3601, 3602, 601, 4801, 101], + } + ) + + +def test_byte_and_path_joint_decode_and_assignments_match(tmp_path): + payload, _ = _ordinary_lookup_bytes() + path = tmp_path / "invented-ladder.npz" + path.write_bytes(payload) + path_ladder = load_us_puma_ladder(path) + byte_ladder = decode_us_puma_ladder(payload) + for field in fields(path_ladder): + left, right = getattr(path_ladder, field.name), getattr(byte_ladder, field.name) + if isinstance(left, np.ndarray): + np.testing.assert_array_equal(left, right) + assert left.dtype == right.dtype + else: + assert left == right + for assign_tract in (False, True): + pd.testing.assert_frame_equal( + assign_us_puma_ladder( + _households(), path_ladder, seed=17, assign_tract=assign_tract + ), + assign_us_puma_ladder( + _households(), byte_ladder, seed=17, assign_tract=assign_tract + ), + ) + + +@pytest.mark.parametrize("defect", ["schema_v1", "joint_population"]) +def test_byte_and_path_reject_the_same_joint_defect(tmp_path, defect): + payload, _ = _ordinary_lookup_bytes() + with np.load(BytesIO(payload), allow_pickle=False) as archive: + arrays = {key: archive[key] for key in archive.files} + if defect == "schema_v1": + metadata = json.loads(str(arrays["metadata_json"].item())) + metadata["schema_version"] = 1 + arrays["metadata_json"] = np.asarray(json.dumps(metadata)) + else: + arrays["joint_overlap_population"][0] += 1 + path = tmp_path / "invalid-invented-ladder.npz" + np.savez_compressed(path, **arrays) + with pytest.raises(ValueError) as path_error: + load_us_puma_ladder(path) + with pytest.raises(ValueError) as byte_error: + decode_us_puma_ladder(path.read_bytes()) + assert str(path_error.value) == str(byte_error.value) + + +@pytest.mark.parametrize("wrapper", [bytearray, memoryview]) +def test_byte_decoder_requires_immutable_bytes(wrapper): + with pytest.raises(TypeError, match="immutable bytes"): + decode_us_puma_ladder(wrapper(b"not an archive")) + + +@pytest.mark.parametrize("unsupported_weight", [0.0, 1.0]) +def test_graph_gate_rejects_real_unsupported_pair_even_at_zero_weight( + unsupported_weight, +): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = [2.6, 3.6, 73.8, 20.0, unsupported_weight] + kernel = geography.USGeographyGateKernel() + assert ( + kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)).receipt[ + "outcome" + ] + == "pass" + ) + # County 01001 and CD 102 are individually present in PUMA 0100100, + # but the only supported pairs are county 01001/CD 101, county 01002/CD 102. + household.loc[4, "congressional_district_geoid"] = 102 + result = kernel.run(_context(household, weights, ladder_bytes, crosswalk_bytes)) + assert result.receipt["outcome"] == "fail" + expected_mass = geography.us_puma_ladder_gate(household, np.asarray(weights)) + ladder = decode_us_puma_ladder(ladder_bytes) + assert (100100, 1001) in set( + zip(ladder.county_overlap_puma, ladder.county_overlap_county, strict=True) + ) + assert (100100, 102) in set( + zip(ladder.cd_overlap_puma, ladder.cd_overlap_cd, strict=True) + ) + assert (100100, 1001, 102) not in set( + zip( + ladder.joint_overlap_puma, + ladder.joint_overlap_tract // 1_000_000, + ladder.joint_overlap_cd, + strict=True, + ) + ) + expected_joint = geography.us_puma_ladder_joint_support_gate(household, ladder) + assert expected_mass.passed + assert not expected_joint.passed + assert expected_joint.details["unsupported_rows"] == 1 + assert ( + result.receipt["evidence"] + == geography.GateReport((expected_mass, expected_joint)).to_manifest() + ) + + +def test_graph_gate_retains_independent_legacy_mass_failure(): + ladder_bytes, crosswalk_bytes = _ordinary_lookup_bytes() + household = _households() + weights = np.asarray([0.0, 6.2, 73.8, 20.0, 0.0]) + ladder = decode_us_puma_ladder(ladder_bytes) + assert geography.us_puma_ladder_joint_support_gate(household, ladder).passed + assert not geography.us_puma_ladder_gate(household, weights).passed + result = geography.USGeographyGateKernel().run( + _context(household, weights, ladder_bytes, crosswalk_bytes) + ) + assert result.receipt["outcome"] == "fail" + + +def test_graph_gate_declares_both_lookup_producer_dependencies(): + lookup, _, _, gate = _nodes() + assert { + (item.name, item.producer, item.artifact, item.type) + for item in gate.artifact_inputs + } == { + ("ladder", lookup.id, "ladder", geography.US_PUMA_LOOKUP_TYPE), + ("crosswalk", lookup.id, "crosswalk", geography.US_CD_CROSSWALK_TYPE), + } diff --git a/packages/microcosm-build/tests/test_us_runtime_facade_union.py b/packages/microcosm-build/tests/test_us_runtime_facade_union.py new file mode 100644 index 000000000..3287a7a4d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_runtime_facade_union.py @@ -0,0 +1,113 @@ +"""Frozen B/F/J facade contract, without donor or country-engine loading.""" + +import hashlib +from importlib import import_module +from types import SimpleNamespace + +import pytest + +from microcosm.build import us_runtime as facade + + +# Derived independently from __all__ in pinned B 2ebca68e, F eba103bb and +# J 6f6087cb. Hash preimage is UTF-8 newline-joined sorted unique export names. +_UNION_SHA256 = "213ae67eab1640925fade5075a10eae62d7f219b15c1c04090a9feac8c692aa3" +_CONSTANTS = ( + "PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "US_PUF_SUPPORT_STAGE_NAME", + "US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "US_QBI_OUTPUT_COLUMNS", +) + + +def test_facade_preserves_exact_906_name_union_without_duplicates(): + assert len(facade.__all__) == len(set(facade.__all__)) == 906 + preimage = "\n".join(sorted(facade.__all__)).encode() + assert hashlib.sha256(preimage).hexdigest() == _UNION_SHA256 + assert set(facade.__all__) <= set(dir(facade)) + assert all( + name in facade.__dict__ + or name in facade._LAZY_EXPORTS + or name in facade._SOURCE_EXPORTS + or name in facade._SUPPORT_EXPORTS + or name in {"US_DONORS", "US_STAGE_NAMES"} + for name in facade.__all__ + ) + + +def test_graph_constants_keep_canonical_owner_and_object_identity(): + module_name = "microcosm.build.us_runtime.operator_column_contracts" + owner = import_module(module_name) + for name in _CONSTANTS: + assert facade._LAZY_EXPORTS[name] == (module_name, name) + assert getattr(facade, name) is getattr(owner, name) + + +@pytest.mark.parametrize( + ("name", "module"), + [ + ("ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "asec_checkpoint"), + ("load_asec_raw_stage_checkpoint_v4", "asec_checkpoint"), + ("us_puma_ladder_joint_support_gate", "puma_ladder"), + ], +) +def test_new_union_members_resolve_to_real_owner(name, module): + owner = import_module(f"microcosm.build.us_runtime.{module}") + assert getattr(facade, name) is getattr(owner, name) + + +def test_lazy_resolution_preserves_override_installed_during_import(monkeypatch): + name = "us_puma_ladder_joint_support_gate" + imported, override = object(), object() + monkeypatch.delitem(facade.__dict__, name, raising=False) + + def loader(module): + assert module == "microcosm.build.us_runtime.puma_ladder" + monkeypatch.setitem(facade.__dict__, name, override) + return SimpleNamespace(**{name: imported}) + + monkeypatch.setattr(facade, "import_module", loader) + assert facade._resolve_export(name) is override + assert getattr(facade, name) is override + + +def test_existing_override_avoids_lazy_import(monkeypatch): + name = "us_puma_ladder_joint_support_gate" + override = object() + monkeypatch.setitem(facade.__dict__, name, override) + + def refuse_import(module): + raise AssertionError(f"unexpected import of {module}") + + monkeypatch.setattr(facade, "import_module", refuse_import) + assert facade._resolve_export(name) is override + + +def test_plan_keeps_missing_and_unknown_stage_refusal(monkeypatch): + # Ordinary injected plan avoids reading source/donor manifest resources. + monkeypatch.setitem(facade.__dict__, "US_STAGE_NAMES", ("ordinary_stage",)) + monkeypatch.setitem(facade.__dict__, "US_DONORS", {}) + with pytest.raises(ValueError, match="missing"): + facade.us_plan({}) + with pytest.raises(ValueError, match="Unknown stage"): + facade.us_plan( + {"ordinary_stage": lambda frame: frame, "typo": lambda frame: frame} + ) + + def transform(frame): + return frame + + plan = facade.us_plan({"ordinary_stage": transform}) + assert len(plan.stages) == 1 + assert plan.stages[0].name == "ordinary_stage" + assert plan.stages[0].transform is transform diff --git a/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py b/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py new file mode 100644 index 000000000..c9b6db2b2 --- /dev/null +++ b/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py @@ -0,0 +1,163 @@ +"""Ordinary overshoot contracts for the combined US-launch solver. + +The count targets, starts and epoch budgets come from current main's +test_solve.py best-iterate cases. All populations are invented in this module. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd +import pytest +import torch + +from microcosm.calibrate import GroupedUpperBounds, Target, TargetSet, calibrate +from microcosm.calibrate import solve as solve_module +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _count_frame(weights: tuple[float, ...]) -> Frame: + ids = tuple(range(len(weights))) + return Frame( + { + "person": pd.DataFrame({"person_id": ids, "person_household_id": ids}), + "household": pd.DataFrame( + {"household_id": ids, "household_count": np.ones(len(ids))} + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.asarray(weights), WeightKind.DESIGN)}, + ) + + +@pytest.mark.parametrize("sparse", [False, True], ids=["dense", "csr"]) +@pytest.mark.parametrize("fixed_zeros", [False, True], ids=["positive", "fixed-zeros"]) +@pytest.mark.parametrize( + "target,warm,epochs,selected_epoch", + [ + pytest.param(2.01, 2.0, 2, 0, id="warm-start-overshoot"), + pytest.param(1.5, None, 3, 2, id="intermediate-overshoot"), + ], +) +def test_grouped_closing_state_and_ordinary_best_iterate_remain_distinct( + monkeypatch, sparse, fixed_zeros, target, warm, epochs, selected_epoch +) -> None: + """A worse closing step remains grouped; ordinary Adam selects the best. + + The bound of three is deliberately loose, so projection cannot account for + the difference. Fixed-support cases add both a zero-only group and a zero + row in the positive group while keeping the same active count problem. + """ + monkeypatch.setattr(solve_module, "_SPARSE_MIN_CELLS", 1 if sparse else 1000) + monkeypatch.setattr(solve_module, "_SPARSE_DENSITY_CUTOFF", 1.0) + layouts = [] + apply_constraint = solve_module._apply_constraint + + def observe_layout(matrix, weights): + layouts.append(matrix.layout) + return apply_constraint(matrix, weights) + + monkeypatch.setattr(solve_module, "_apply_constraint", observe_layout) + targets = TargetSet( + ( + Target( + name="population", + entity="household", + value=target, + measure="household_count", + ), + ) + ) + common = {"epochs": epochs, "learning_rate": 0.2, "mass": "free", "seed": 0} + ordinary = calibrate( + _count_frame((1.0,)), + targets, + warm_start_weights=None if warm is None else np.array([warm]), + **common, + ) + initial = np.array([0.0, 1.0, 0.0] if fixed_zeros else [1.0]) + grouped_frame = _count_frame(tuple(initial)) + groups = GroupedUpperBounds( + tuple(range(len(initial))), + np.array([0, 1, 1] if fixed_zeros else [0]), + np.array([0.0, 3.0] if fixed_zeros else [3.0]), + ) + observations = [] + grouped = calibrate( + grouped_frame, + targets, + warm_start_weights=None if warm is None else initial * warm, + grouped_upper_bounds=groups, + grouped_preserve_zeros=fixed_zeros, + _post_projection_observer=observations.append, + **common, + ) + expected_layout = torch.sparse_csr if sparse else torch.strided + assert layouts and all(layout == expected_layout for layout in layouts) + assert ordinary.options["iterate_selection"] == "best_feasible_loss" + receipt = ordinary.options["iterate_selection_receipt"] + assert receipt["rule"] == "best_feasible_loss" + assert receipt["selected_epoch"] == selected_epoch + assert receipt["epochs_executed"] == epochs + assert receipt["epoch_convention"] == "completed_optimizer_updates; zero is start" + assert receipt["selected_loss_float32"] == ordinary.loss_trajectory[selected_epoch] + assert receipt["closing_iterate_loss_float32"] > ordinary.final_loss + 0.01 + assert ordinary.final_loss == pytest.approx( + abs(float(ordinary.weights.sum()) - target) / target, abs=1e-12 + ) + assert len(ordinary.loss_trajectory) == epochs + assert ordinary.final_loss <= ordinary.loss_trajectory.min() + 1e-7 + + assert grouped.options["iterate_selection"] == "closing_state" + assert grouped.options["iterate_selection_receipt"] == {} + assert grouped.gate_open_probabilities is None + assert len(observations) == epochs + 1 + assert [item["epoch"] for item in observations] == list(range(epochs + 1)) + assert all(item["corrected_group_count"] == 0 for item in observations) + expected_start = initial if warm is None else initial * warm + assert observations[0]["weights"].tobytes() == expected_start.tobytes() + observed_losses = [] + for item in observations: + weights = item["weights"] + assert weights.dtype == np.dtype("float64") + assert item["household_ids"] == groups.household_ids + np.testing.assert_array_equal(item["group_indices"], groups.group_indices) + np.testing.assert_array_equal(item["absolute_bounds"], groups.absolute_bounds) + np.testing.assert_array_equal( + item["group_totals"], groups.check(weights, positive=not fixed_zeros) + ) + observed_losses.append(abs(float(weights.sum()) - target) / target) + assert int(np.argmin(observed_losses)) == selected_epoch + assert observed_losses[-1] > min(observed_losses) + 0.01 + np.testing.assert_allclose( + grouped.loss_trajectory, observed_losses[:-1], rtol=2e-6, atol=2e-7 + ) + assert grouped.final_loss == pytest.approx(observed_losses[-1], abs=1e-12) + assert grouped.final_loss > ordinary.final_loss + 0.01 + assert grouped.weights.tobytes() == observations[-1]["weights"].tobytes() + stored = grouped.frame.resolve_weights("household").values + assert stored.tobytes() == grouped.weights.tobytes() + assert grouped.initial_weights.tobytes() == initial.tobytes() + assert grouped.options["grouped_upper_bounds"] == groups.diagnostics( + grouped.weights, 0 + ) + for entity in ("person", "household"): + pd.testing.assert_frame_equal( + grouped.frame.table(entity), grouped_frame.table(entity) + ) + if fixed_zeros: + zero_mask = initial == 0 + for item in observations: + assert item["weights"][zero_mask].tobytes() == initial[zero_mask].tobytes() + assert stored[zero_mask].tobytes() == initial[zero_mask].tobytes() + assert grouped.options["grouped_preserve_zeros"] == { + "enabled": True, + "fixed_zero_count": 2, + "ordered_zero_mask_sha256": hashlib.sha256(bytes([1, 0, 1])).hexdigest(), + } + assert np.isfinite(grouped.realized_max_weight_ratio) + assert grouped.realized_max_weight_ratio == grouped.weights[1] + else: + assert "grouped_preserve_zeros" not in grouped.options diff --git a/packages/microcosm-graph/tests/test_lazy_snapshot_metadata_integration.py b/packages/microcosm-graph/tests/test_lazy_snapshot_metadata_integration.py new file mode 100644 index 000000000..568bf3aff --- /dev/null +++ b/packages/microcosm-graph/tests/test_lazy_snapshot_metadata_integration.py @@ -0,0 +1,417 @@ +"""Invented ordinary Frames exercise lazy snapshot publication and replay.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import microcosm.graph.attachments as attachments +import microcosm.graph.executor as executor +import microcosm.graph.store as store_module +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph.manifest import NodeReceipt +from microcosm.graph.population import Population +from microcosm.graph.store import StoreCorrupt, StoreUnavailable + + +def _frame(offset=0): + # Match the ordinary executor fixture's three persons, two households, + # weights and strata. All values and lineage identifiers are invented. + return Frame( + { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_household_id": [10, 10, 20], + "age": np.array([10, 20, 30], dtype="int64") + offset, + "source_person_id": [101, 102, 103], + } + ), + "household": pd.DataFrame( + {"household_id": [10, 20], "source_household_id": [201, 202]} + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([1.0, 2.0]), WeightKind.DESIGN)}, + pd.Series(["a", "a", "b"], name="stratum"), + metadata={ + "source_manifest": { + "source_arm": "invented-ordinary", + "sources": ({"sha256": "a" * 64, "rows": 3},), + "flags": frozenset({"ordinary", "metadata-v2"}), + "nullable": None, + "flag": True, + }, + "signed_zero": -0.0, + }, + ) + + +class _Kernel: + def __init__(self, ref, capabilities, compute): + self.ref = ref + self.capabilities = capabilities + self.compute = compute + self.calls = 0 + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + self.calls += 1 + return self.compute(context) + + +def _case(tmp_path): + # This generated source follows test_graph_executor._source_path; it is + # interpreted by the CREATE kernel and is never a repository resource. + source = tmp_path / "ordinary-source" + source.mkdir() + (source / "value.txt").write_text("0", encoding="utf-8") + + def create(context): + offset = int((context.sources["survey"] / "value.txt").read_text()) + return KernelResult(frame=_frame(offset)) + + def derive(context): + table = context.tables["person"] + return KernelResult( + columns={ + ("person", context.params["target"]): pd.Series( + table[context.params["source"]].to_numpy(dtype="float64") * 2, + index=pd.Index(table["person_id"], name="person_id"), + ) + } + ) + + def reweight(context): + return KernelResult( + weights=Weights( + context.weights["household"].values * 2, WeightKind.IMPORTANCE + ), + # Person mass is 1 + 1 + 2 before and 2 + 2 + 4 after. + receipt={ + "mass": { + "policy": "free", + "before": 4.0, + "after": 8.0, + "stratum_before": {"a": 2.0, "b": 2.0}, + "stratum_after": {"a": 4.0, "b": 4.0}, + } + }, + ) + + graph = Graph( + "toy", + (SourceRef("survey", "csv-tables"),), + ( + Node( + "survey", + "snapshot.source@1", + sources=("survey",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "age", "int64"), + Owned("person", "source_person_id", "int64"), + Owned("household", "source_household_id", "int64"), + ), + ), + Node( + "column_before", + "snapshot.derive@1", + population="survey", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "doubled", "float64"),), + params={"source": "age", "target": "doubled"}, + ), + Node( + "pool", + "snapshot.reweight@1", + structural=StructuralDelta.REWEIGHT, + base="survey", + inputs=( + Slice("person", ("doubled",)), + Slice("household", ("source_household_id",)), + ), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + ), + Node( + "column_after", + "snapshot.derive@1", + population="pool", + inputs=(Slice("person", ("doubled",)),), + outputs=(Owned("person", "quadrupled", "float64"),), + params={"source": "doubled", "target": "quadrupled"}, + ), + ), + ) + registry = KernelRegistry() + for ref, structural, compute in ( + ("snapshot.source@1", StructuralDelta.CREATE, create), + ("snapshot.derive@1", StructuralDelta.NONE, derive), + ("snapshot.reweight@1", StructuralDelta.REWEIGHT, reweight), + ): + registry.register( + _Kernel( + ref, + Capabilities(Determinism.DETERMINISTIC, structural=structural), + compute, + ) + ) + return compile_graph(graph), {"survey": source}, registry + + +def _assert_frame_equal(actual, expected): + assert actual.metadata == expected.metadata + assert store_module._encode_frame_metadata( + actual.metadata + ) == store_module._encode_frame_metadata(expected.metadata) + assert actual.schema == expected.schema + assert actual.mass_log == expected.mass_log + pd.testing.assert_series_equal(actual.strata, expected.strata) + for entity in expected.entities: + pd.testing.assert_frame_equal(actual.table(entity), expected.table(entity)) + for entity in expected.weighted_entities: + assert actual.weights_for(entity).kind is expected.weights_for(entity).kind + assert ( + actual.weights_for(entity).values.tobytes() + == expected.weights_for(entity).values.tobytes() + ) + + +def _assert_snapshots(manifest, eager, store): + assert isinstance(manifest.populations, attachments._LazyPopulations) + assert manifest.key == eager.key + assert manifest.content_addressed == eager.content_addressed + for version in ("survey", "pool"): + reference = manifest.populations._references[version] + # Both versions contain nonstructural columns, so structural reuse + # cannot accidentally hide a broken snapshot publication header. + assert reference.key != manifest.nodes[version].frame_key + header = store.metadata(reference.key, kind="frame") + encoded = json.loads( + (store.object_path(reference.key) / "frame.json").read_text() + ) + expected_hash = hashlib.sha256( + store_module._canonical_json(encoded["metadata"]) + ).hexdigest() + assert header["frame_metadata_sha256"] == expected_hash + assert header["frame_format"] == store_module._FRAME_FORMAT + assert header["node_key"] == manifest.nodes[version].key + restored = store.load_frame(reference.key, node_key=reference.node_key) + _assert_frame_equal(restored, eager.population(version)) + _assert_frame_equal(manifest.population(version), eager.population(version)) + assert manifest.mass_ledger(version) == eager.mass_ledger(version) + pool = manifest.population("pool") + assert pool.weights_for("household").kind is WeightKind.IMPORTANCE + assert pool.person["source_person_id"].tolist() == [101, 102, 103] + assert pool.table("household")["source_household_id"].tolist() == [201, 202] + assert pool.person["quadrupled"].tolist() == [40.0, 80.0, 120.0] + + +def test_nonstructural_snapshots_publish_metadata_and_reload_with_eager_parity( + tmp_path, +): + compiled, sources, kernels = _case(tmp_path) + eager = run_graph( + compiled, + sources=sources, + kernels=kernels, + store=ContentStore(tmp_path / "eager"), + resume="forbid", + ) + store = ContentStore(tmp_path / "lazy") + lazy = run_graph( + compiled, + sources=sources, + kernels=kernels, + store=store, + population_retention="lazy", + resume="forbid", + ) + _assert_snapshots(lazy, eager, store) + + +def test_required_replay_republishes_missing_auxiliary_snapshots_without_kernels( + tmp_path, +): + compiled, sources, kernels = _case(tmp_path) + store = ContentStore(tmp_path / "store") + eager = run_graph(compiled, sources=sources, kernels=kernels, store=store) + calls = {ref: kernel.calls for ref, kernel in kernels.as_mapping().items()} + for attempt in range(2): + replay = run_graph( + compiled, + sources=sources, + kernels=kernels, + store=store, + population_retention="lazy", + resume="require", + ) + assert all(node.hit for node in replay.nodes.values()) + assert calls == { + ref: kernel.calls for ref, kernel in kernels.as_mapping().items() + } + _assert_snapshots(replay, eager, store) + if attempt == 0: + for reference in replay.populations._references.values(): + shutil.rmtree(store.object_path(reference.key)) + + +def test_structural_collision_forces_reloadable_metadata_snapshot(tmp_path): + compiled, sources, kernels = _case(tmp_path) + create_only = compile_graph( + Graph("toy", compiled.graph.sources, (compiled.graph.node("survey"),)) + ) + store = ContentStore(tmp_path / "store") + eager = run_graph(create_only, sources=sources, kernels=kernels, store=store) + receipt = eager.nodes["survey"] + # Leave the structural Frame visible while removing only its node record. + # The next auto run genuinely recomputes into a structural-key collision. + shutil.rmtree(store.object_path(executor._cache_record_key(receipt.key))) + lazy = run_graph( + create_only, + sources=sources, + kernels=kernels, + store=store, + population_retention="lazy", + ) + assert not lazy.nodes["survey"].hit + reference = lazy.populations._references["survey"] + assert reference.key != receipt.frame_key + assert lazy.key == eager.key + _assert_frame_equal(store.load_frame(reference.key), eager.population("survey")) + _assert_frame_equal(lazy.population("survey"), eager.population("survey")) + + +def _snapshot_fixture(): + population = Population.from_frame(_frame(), "survey") + structural = NodeReceipt( + key="a" * 64, + hit=False, + seed=0, + kernel_ref="snapshot.source@1", + kernel_impl_hash="b" * 64, + capabilities=Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ), + frame_key="c" * 64, + ) + return population, structural + + +def _damage_header(path, damage): + header = json.loads(path.read_text()) + if damage == "missing": + header.pop("frame_metadata_sha256") + elif damage == "wrong": + header["frame_metadata_sha256"] = "0" * 64 + elif damage == "old_codec": + header["frame_format"] = "microcosm-graph-frame-v1" + path.write_text(json.dumps(header), encoding="utf-8") + + +@pytest.mark.parametrize("damage", ["missing", "wrong", "old_codec"]) +def test_existing_snapshot_rejects_invalid_metadata_header(tmp_path, damage): + store = ContentStore(tmp_path / "store") + population, structural = _snapshot_fixture() + key, _ = attachments._snapshot( + store, population, structural, (), verify_existing=True + ) + header_path = store.object_path(key) / "meta.json" + _damage_header(header_path, damage) + damaged = header_path.read_bytes() + expected = StoreUnavailable if damage == "old_codec" else StoreCorrupt + with pytest.raises(expected): + store.load_frame(key, node_key=structural.key) + with pytest.raises(expected): + attachments._snapshot(store, population, structural, (), verify_existing=True) + assert header_path.read_bytes() == damaged + + +@pytest.mark.parametrize("damage", ["valid", "missing", "wrong", "old_codec"]) +def test_atomic_publication_race_validates_incumbent_metadata( + tmp_path, monkeypatch, damage +): + store = ContentStore(tmp_path / "store") + population, structural = _snapshot_fixture() + baseline = ContentStore(tmp_path / "baseline") + key, _ = attachments._snapshot( + baseline, population, structural, (), verify_existing=True + ) + destination = store.object_path(key) + real_replace = store_module.os.replace + collisions = [] + + def competing_writer(source, target): + if Path(target) == destination: + # Publish a complete competing object after _put's initial + # exists check, then force its actual EEXIST/ENOTEMPTY branch. + assert not destination.exists() + shutil.copytree(source, destination) + _damage_header(destination / "meta.json", damage) + collisions.append(destination) + raise OSError(errno.ENOTEMPTY, "ordinary competing snapshot") + return real_replace(source, target) + + monkeypatch.setattr(store_module.os, "replace", competing_writer) + if damage == "valid": + actual, _ = attachments._snapshot( + store, population, structural, (), verify_existing=True + ) + assert actual == key + _assert_frame_equal(store.load_frame(key), population.frame) + else: + expected = StoreUnavailable if damage == "old_codec" else StoreCorrupt + with pytest.raises(expected): + attachments._snapshot( + store, population, structural, (), verify_existing=True + ) + assert collisions == [destination] + assert not any(store.tmp.iterdir()) + + +def test_lazy_reload_refuses_decoded_frame_replacement(tmp_path, monkeypatch): + store = ContentStore(tmp_path / "store") + population, structural = _snapshot_fixture() + key, payload_hash = attachments._snapshot( + store, population, structural, (), verify_existing=True + ) + views = attachments._LazyPopulations( + store, + { + "survey": attachments._StoredPopulation( + key, structural.key, population.frame.metadata, payload_hash + ) + }, + ) + # The validated header stays unchanged; the decoded Frame itself changes + # after verification. The attachment must bind that actual decoded value. + monkeypatch.setattr(store, "load_frame", lambda *args, **kwargs: _frame(offset=1)) + with pytest.raises(StoreCorrupt, match="decoded population attachment changed"): + views["survey"] + assert not views._cache From 716980bbc155277dfbf0c09d3b3c42b03b6c8d0f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:52:15 -0400 Subject: [PATCH 016/255] Record US launch integration source staging handoff --- PROGRESS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PROGRESS.md b/PROGRESS.md index 2912cb736..cc5c6beac 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -763,3 +763,12 @@ Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preserv Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +### State +All thirteen source layers staged and committed; behavioral qualification pending. + +### Done +Per-layer pins and actual commit messages checked; store/current-main preservation retained. + +### Next +Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. From 84243d45e1b8b690cd7bfab29b2469c152c2c0ce Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 10:57:12 -0400 Subject: [PATCH 017/255] Include source definitions required by US launch integration --- PROGRESS.md | 4 + .../us_runtime/acs_2024_housing_universe.json | 23 + .../asec_current_money_consumers_v1.json | 628 +++ .../asec_current_money_domains_v1.json | 3418 +++++++++++++++++ .../asec_current_money_price_basis_v1.json | 21 + ...f_2015_monetary_agi_source_projection.json | 1 + .../puf_2015_monetary_source_projection.json | 1 + .../puf_2015_raw_source_definition.json | 1 + 8 files changed, 4097 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_agi_source_projection.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_source_projection.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json diff --git a/PROGRESS.md b/PROGRESS.md index cc5c6beac..dea72ea44 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -772,3 +772,7 @@ Per-layer pins and actual commit messages checked; store/current-main preservati ### Next Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. + +### Integration source-resource closure — 2026-09-09 + +Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json new file mode 100644 index 000000000..272b560dd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "source_kind": "microcosm.acs_housing_universe_source.v1", + "vintage": 2024, + "dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pages": [1, 2, 3, 8, 32, 33] + }, + "vacancy_definition": "acs_2024_source_vacancy_v1", + "vacancy": "TYPEHUGQ=1, NP=0, no linked persons, WGTP=1..9999, blank TEN; separate source vacancy evidence, not a change to the pure classifier", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + "gq": "TYPEHUGQ=2/3, NP=1, WGTP=0, blank TEN, one linked person with PWGTP=1..9999; one source person placeholder at DESIGN weight, not a physical facility or calibrated descendant", + "occupied_unknown_tenure": "Retained in occupied HU total with explicitly unclassified tenure; never infer from model tenure_type", + "origin": "CD source household origin: ACS arm, original household member name and SHA256, vintage integer2024, raw SERIALNO string; person additionally retains original person member and normalized SPORDER", + "parser": "UTF8/noBOM, strict comma QUOTE_MINIMAL, lexical projected fields, no trimming/NA inference; same ordered full role headers; every physical record full width", + "encoding": "acs-housing-lexical-projection/1", + "classifier_columns": ["interview_scope", "physical_unit", "household_kind", "tenure_subtype", "occupied_hu", "hu_tenure_class", "unresolved_reasons", "TEN_valid"], + "us_archive_scope_assumption": "50 US states and DC; observed Puerto Rico refuses rather than silently changing scope", + "release_eligible": false, + "calibrated_descendants_approved": false, + "B19001_B25003_activation": false +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json new file mode 100644 index 000000000..5ac33188e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json @@ -0,0 +1,628 @@ +{ + "addresses": [ + { + "column": "ANN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ANN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "CAP_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CAP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "long_term_capital_gains", + "short_term_capital_gains" + ], + "role": "cps_source" + }, + { + "column": "CHSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CHSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_paid" + ], + "role": "cps_source" + }, + { + "column": "CSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_received" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL1", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL2", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIV_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIV_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "qualified_dividend_income", + "non_qualified_dividend_income" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "ED_VAL", + "consumers": [ + "education_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "educational_assistance" + ], + "role": "cps_source" + }, + { + "column": "FRSE_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "FRSE_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "farm_operations_income" + ], + "role": "cps_source" + }, + { + "column": "asec_HTOTVAL", + "consumers": [ + "asec_household_observations" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "household", + "field": "HTOTVAL", + "grain": "household", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_household_total_income" + ], + "role": "cps_source" + }, + { + "column": "INT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "INT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_interest_income" + ], + "role": "cps_source" + }, + { + "column": "OI_VAL", + "consumers": [ + "asec_other_income_split", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "OI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "alimony_income", + "strike_benefits", + "miscellaneous_income" + ], + "role": "cps_source" + }, + { + "column": "PHIP_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PHIP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "health_insurance_premiums_without_medicare_part_b" + ], + "role": "cps_source" + }, + { + "column": "PMED_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PMED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "other_medical_expenses" + ], + "role": "cps_source" + }, + { + "column": "PNSN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PNSN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "POTC_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "POTC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "over_the_counter_health_expenses" + ], + "role": "cps_source" + }, + { + "column": "PTOTVAL", + "consumers": [ + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PTOTVAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "microunit_current_total_income" + ], + "role": "cps_source" + }, + { + "column": "RETCB_VAL", + "consumers": [ + "retirement_contributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RETCB_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "retirement_contributions" + ], + "role": "cps_source" + }, + { + "column": "RNT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RNT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "rental_income" + ], + "role": "cps_source" + }, + { + "column": "SEMP_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SEMP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "self_employment_income_before_lsr" + ], + "role": "cps_source" + }, + { + "column": "SPM_CAPHOUSESUB", + "consumers": [ + "immigration" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CAPHOUSESUB", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "immigration_amount_predictor" + ], + "role": "cps_source" + }, + { + "column": "SPM_CHILDCAREXPNS", + "consumers": [ + "childcare", + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CHILDCAREXPNS", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_pre_subsidy_childcare_expenses" + ], + "role": "cps_source" + }, + { + "column": "SPM_ENGVAL", + "consumers": [ + "energy_subsidy" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_ENGVAL", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_energy_subsidy" + ], + "role": "cps_source" + }, + { + "column": "SSI_VAL", + "consumers": [ + "acs_release_predictors_reported_ssi_anchor" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SSI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_reported_ssi_predictor" + ], + "role": "cps_source" + }, + { + "column": "SS_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SS_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents" + ], + "role": "cps_source" + }, + { + "column": "UC_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "UC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "unemployment_compensation" + ], + "role": "cps_source" + }, + { + "column": "VET_VAL", + "consumers": [ + "eligibility_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "VET_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "veterans_benefits" + ], + "role": "cps_source" + }, + { + "column": "WC_VAL", + "consumers": [ + "workers_compensation" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "workers_compensation" + ], + "role": "cps_source" + }, + { + "column": "WSAL_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit", + "voluntary_filing_current_leaf" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WSAL_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "employment_income_before_lsr" + ], + "role": "cps_source" + } + ], + "implementation_status": "declarations_only_live_nominal_consumers_unchanged", + "microunit": { + "amount_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "distribution": "microunit", + "module": "microunit/tax_unit_construction.py", + "module_sha256": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "version": "0.1.0" + }, + "ownership_policy": "restater_owns_cpi_only_derived_outputs_keep_registered_phase_role_owners", + "routing_policy": "unchanged_source_codes_no_new_DST_universe_exclusion", + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "unscaled_fields": [ + "PAW_VAL", + "SPM_SNAPSUB" + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json new file mode 100644 index 000000000..7f09e933e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json @@ -0,0 +1,3418 @@ +{ + "fields": [ + { + "column": "ANN_VAL", + "declared_niu_codes": [ + -1 + ], + "domain": { + "declared_negative_nonmoney_codes": [ + -1 + ], + "declared_niu_codes": [ + -1 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -1 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -1, + "name": "ANN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 438, + "dictionary_spelling": "ANN_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars" + }, + { + "column": "CAP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "CAP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 445, + "dictionary_spelling": "CAP_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "CHSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 1 + }, + "zero_semantics": "niu" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CHSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 712, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + } + ], + "zero_semantics": "niu" + }, + { + "column": "CSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2022, + "pdf_page_1based": 50, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2023, + "pdf_page_1based": 51, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 718, + "dictionary_spelling": "CSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 465, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 471, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIV_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIV_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 478, + "dictionary_spelling": "DIV_VAL", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 495, + "dictionary_spelling": "DST_VAL1", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 501, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 507, + "dictionary_spelling": "DST_VAL2", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "ED_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "ED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 521, + "dictionary_spelling": "ED_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "FRSE_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -9999999, + "name": "FRSE_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2022, + "pdf_page_1based": 40, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2023, + "pdf_page_1based": 41, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 390, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "asec_HTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "household", + "grain": "household", + "maximum": 99999999, + "minimum": -999999, + "name": "HTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2022, + "pdf_page_1based": 5, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2023, + "pdf_page_1based": 5, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 108, + "dictionary_spelling": "HTOTVAL", + "income_year": 2024, + "pdf_page_1based": 12, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "INT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "INT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 537, + "dictionary_spelling": "INT_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "OI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "OI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 549, + "dictionary_spelling": "OI_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "PHIP_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PHIP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1292, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2022, + "pdf_page_1based": 78, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1288, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1455, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PMED_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PMED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1304, + "dictionary_spelling": "PMED_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1300, + "dictionary_spelling": "PMED_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1467, + "dictionary_spelling": "PMED_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PNSN_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "PNSN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 571, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "POTC_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "POTC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289, + 290 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1310, + "dictionary_spelling": "POTC_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1306, + "dictionary_spelling": "POTC_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1473, + "dictionary_spelling": "POTC_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999999, + "minimum": -99999, + "name": "PTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 588, + "dictionary_spelling": "PTOTVAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "RETCB_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "RETCB_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 274 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2022, + "pdf_page_1based": 46, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2023, + "pdf_page_1based": 47, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 600, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "RNT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -9999, + "name": "RNT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 621, + "dictionary_spelling": "RNT_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SEMP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -999999, + "name": "SEMP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 413, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SPM_CAPHOUSESUB", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 99999, + "minimum": 0, + "name": "SPM_CAPHOUSESUB", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1436, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1432, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 936, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_CHILDCAREXPNS", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 999999, + "minimum": 0, + "name": "SPM_CHILDCAREXPNS", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 304 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1447, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1443, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 947, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_ENGVAL", + "declared_niu_codes": [], + "domain": { + "authority_range_alternatives": [ + { + "location": "Range header", + "maximum": 10000, + "minimum": 0 + }, + { + "location": "Values description", + "maximum": 99999, + "minimum": 0 + } + ], + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "definitely_outside_both_declared_ranges": { + "above": 99999, + "below": 0 + }, + "encoded_range_inclusive": null, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "uncontested_valid_dollar_range_inclusive": { + "maximum": 10000, + "minimum": 0 + }, + "unresolved_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 10001 + }, + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": null, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "partial_conflicting_upper_bound_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 10000, + "minimum": 0, + "name": "SPM_ENGVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 293 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1463, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1459, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2023, + "pdf_page_1based": 86, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 963, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SSI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SSI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 640, + "dictionary_spelling": "SSI_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SS_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SS_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 634, + "dictionary_spelling": "SS_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "UC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "UC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 672, + "dictionary_spelling": "UC_VAL", + "income_year": 2024, + "pdf_page_1based": 50, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "VET_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "VET_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 684, + "dictionary_spelling": "VET_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "WC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 692, + "dictionary_spelling": "WC_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WSAL_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "WSAL_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 430, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + } + ], + "max_households": 400000, + "max_persons": 1000000, + "recipe": "us_asec_current_money_ccpiu_2024_v1", + "registry_sha256": "bdeb4d2fa1c37c6064530d560a62b12c515118608845b30d01b06c2eeeda8c6e", + "schema_version": 1, + "source_years": [ + [ + 2022, + 2023 + ], + [ + 2023, + 2024 + ], + [ + 2024, + 2025 + ] + ], + "zero_origin_policy": "frozen_fillna_zero_origin_unresolved" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json new file mode 100644 index 000000000..445650d70 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json @@ -0,0 +1,21 @@ +{ + "algorithm": "parse_float64_target_divide_source_then_multiply_nonzero_v1", + "cells": { + "2022": "163.6", + "2023": "170.0", + "2024": "174.4" + }, + "currency": "USD", + "identity_year": "copy_normalized_bytes", + "numeric_scope": "platform_bitwise", + "price_basis": "census_ccpiu_annual_2024", + "schema_version": 1, + "source_finality": "not_stated_in_source", + "target_year": 2024, + "workbook": { + "sha256": "8a5ef078b02c48c9e6414f3ca0cbdfe4a013ce6acf1a3fda42e6bd9c7da49e48", + "size_bytes": 41472, + "url": "https://www2.census.gov/programs-surveys/demo/tables/p60/286/annual-index-value_annual-percent-change.xls" + }, + "zero": "positive_zero_no_arithmetic" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_agi_source_projection.json b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_agi_source_projection.json new file mode 100644 index 000000000..9e27d0a2a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_agi_source_projection.json @@ -0,0 +1 @@ +{"anchors":{"aggregate_recid_footnote":{"booklet":"main_2022","needle":"[1] Records containing aggregated amounts are identified by RECID=999996, 999997, 999998, and 999999","needle_sha256":"2686c04e7238897d583f14f6a152502f55a7d15cf595c2c63dbb5924f2d72e24","page_end":2075,"page_start":1975,"pdf_page":11,"printed_page":10,"text":"[1] Records containing aggregated amounts are identified by RECID=999996, 999997, 999998, and 999999","text_end":18141,"text_sha256":"2686c04e7238897d583f14f6a152502f55a7d15cf595c2c63dbb5924f2d72e24","text_start":18041},"aggregate_records":{"booklet":"main_2022","needle":"Second, returns that contain one or more amount fields with values deeme d extremely","needle_sha256":"bf1d708a634bd26f9ccbf59a7b873b07e22e3f8374921efdf0d565afc57f664a","page_end":1227,"page_start":443,"pdf_page":4,"printed_page":3,"text":"Second, returns that contain one or more amount fields with values deeme d extremely \nlarge\n1 have been excluded from the microdata sample and are aggregated into one of four records2, \nidentified by RECID=999996 for returns reporting negative Adjusted Gross Income (AGI), \nRECID=999997 for returns reporting positive AGI between $0 and $10,000,000, RECID=999998 \nfor returns reporting positive AGI between $10,000,001 and $100,000,000, and RECID=999999 \nfor returns reporting positive AGI of $100,000,001 or more. The rules for identifying extremely \nlarge values are not applied to amount fields that are statutorily capped, subject to income limits, \nor calculated from other fields that are subject to these rules. Information regarding these aggregate \nreturns can be found in ","text_end":3883,"text_sha256":"e53f1ac33b87d6ba5d2274fc83cbc33b245918cfe88a33d33f74b97cbe0e853f","text_start":3099},"amount_field_width":{"booklet":"main_2022","needle":"12 digit amount fields","needle_sha256":"db574766fcc9683b21083244a370659b5b44104660f62c76615916974c819641","page_end":136,"page_start":114,"pdf_page":7,"printed_page":6,"text":"12 digit amount fields","text_end":9696,"text_sha256":"db574766fcc9683b21083244a370659b5b44104660f62c76615916974c819641","text_start":9674},"amount_fields_heading":{"booklet":"main_2022","needle":"Amount Fields and Large Codes","needle_sha256":"02d4caf81fa5527bb8094826e492f2f9d9c7ce880293f3b3d12cc20059d2070a","page_end":98,"page_start":69,"pdf_page":7,"printed_page":6,"text":"Amount Fields and Large Codes","text_end":9658,"text_sha256":"02d4caf81fa5527bb8094826e492f2f9d9c7ce880293f3b3d12cc20059d2070a","text_start":9629},"code_fields_heading":{"booklet":"main_2022","needle":"CORE RECORD LAYOUT","needle_sha256":"50320ab982369f7648e226b45f83bd6eb80a0d293664ea259ad2c21e77ce8496","page_end":72,"page_start":54,"pdf_page":6,"printed_page":5,"text":"CORE RECORD LAYOUT","text_end":8591,"text_sha256":"50320ab982369f7648e226b45f83bd6eb80a0d293664ea259ad2c21e77ce8496","text_start":8573},"csv_order":{"booklet":"main_2022","needle":"Field names have been ordered first","needle_sha256":"20a74ea4f09169699e5b2ece904448b24f7844529a683243a4e3c35ed4ebbb55","page_end":395,"page_start":256,"pdf_page":27,"printed_page":26,"text":"Field names have been ordered first \nby code fields followed by amount fields in the same order as presented on pages 5 through 10. \n \n ","text_end":47599,"text_sha256":"51c519bd5fe4634e29f8a84928ded60babb8ddcf09e49f81bb260f05cb95a7cc","text_start":47460},"demographic_join_section":{"booklet":"demographic_2022","needle":"This section describes the domain of the supplemental file and method for joining to the 2015","needle_sha256":"0de8bafb27d9fa0deb73a67039f545f2f0628edcb8cab89126c695f6fef31d5f","page_end":183,"page_start":90,"pdf_page":10,"printed_page":9,"text":"This section describes the domain of the supplemental file and method for joining to the 2015","text_end":10136,"text_sha256":"0de8bafb27d9fa0deb73a67039f545f2f0628edcb8cab89126c695f6fef31d5f","text_start":10043},"demographic_strata_restriction":{"booklet":"demographic_2022","needle":"found on the 2015 PUF for records in strata 7 through 13, or those returns","needle_sha256":"b6996405e87a53511dd51a9a5243f0d3d17d7ce41bdd2ce6fc12113a7a7b9425","page_end":411,"page_start":337,"pdf_page":10,"printed_page":9,"text":"found on the 2015 PUF for records in strata 7 through 13, or those returns","text_end":10364,"text_sha256":"b6996405e87a53511dd51a9a5243f0d3d17d7ce41bdd2ce6fc12113a7a7b9425","text_start":10290},"e30400_definition":{"booklet":"main_2022","needle":"E30400 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO","needle_sha256":"490778fc1ebf2d244613820d6a90cf2e2c8fe584ffb06d1019adaf9c4e898393","page_end":1554,"page_start":1166,"pdf_page":24,"printed_page":23,"text":"E30400 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO \nSOCIAL SECURITY TAX, PRIMARY AND SECONDARY TAXPAYERS \nThe lesser of Schedule SE, Section B, Part I line 6 (net Schedule SE earnings) or \nPart I line 9 (maximum amount subject to Social Security tax) on both the primary \nand secondary taxpayers’ Schedule SE. For Section A of Schedule SE, this is line \n4. \n \nE30500 NET EARNING","text_end":44200,"text_sha256":"8da0186ad3a92e40587ffcf02c926ebec043b808166092a208d46704d623de8d","text_start":43812},"e30500_definition":{"booklet":"main_2022","needle":"E30500 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO","needle_sha256":"2b9922cbc4a7f61482c7b6f8a931c82d6e88c8939551b0ed389617df99bbfad3","page_end":1914,"page_start":1536,"pdf_page":24,"printed_page":23,"text":"E30500 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO \nSOCIAL SECURITY TAX, SECONDARY TAXPAYER \nThe lesser of Schedule SE, Section B, Part I line 6 (net Schedule SE earnings) or \nPart I line 9 (maximum amount subject to Social Security tax) on the secondary \ntaxpayer’s Schedule SE. For Section A of Schedule SE, this is line 4. \n \n4 Refundable credits include the Making ","text_end":44560,"text_sha256":"4966f97612b401b41a49d40b82d9f879047c4586129fe93edd63e1a60b35cb2d","text_start":44182},"fiscal_year_conversion":{"booklet":"main_2022","needle":"First, fiscal year returns have been converted to reflect the most recent year-end Tax Year","needle_sha256":"e849f8821b08b5075693d85a87df27204351c92bcb1cef439e5b85374c2d622c","page_end":487,"page_start":256,"pdf_page":4,"printed_page":3,"text":"First, fiscal year returns have been converted to reflect the most recent year-end Tax Year \nand returns filed for years earlier than Tax Year 2012 have been removed from the file. \n \n Second, returns that contain one or more amo","text_end":3143,"text_sha256":"03f8559da89549befa950b5c13b0e1e7a39d3472728f4ba28e5f4c3797d67f7c","text_start":2912},"flpdyr_printed_range":{"booklet":"main_2022","needle":"8. FLPDYR Filing (Accounting) Period, Year:","needle_sha256":"d89025f42eade49e5b783e5893d339dbde3aadd1a6136bf18f6a50d9cc38e9b9","page_end":633,"page_start":450,"pdf_page":14,"printed_page":13,"text":"8. FLPDYR Filing (Accounting) Period, Year: \n \nTax Year ........................................................................ 2011-2014 \n \n \n9. FLPDMO Filing (Accounting) Period, M","text_end":24008,"text_sha256":"923c432965a0c6d3713033c8fe8e88b9858ecb098de0cb7944bd1942aded2da0","text_start":23825},"itemized_blurring_007":{"booklet":"main_2022","needle":"Multivariate blurring has been applied to itemized tax returns with nonzero values","needle_sha256":"e13235e26ef16aba4d8992fb9bc1c6e5c6fd84a301c79de030b28fea319e12d1","page_end":1157,"page_start":835,"pdf_page":5,"printed_page":4,"text":"Multivariate blurring has been applied to itemized tax returns with nonzero values in \nat least two of the following fields: wages and salaries, state and local income taxes, \nand real estate taxes. Blurring of these fields is handled in the same manner as those \nrecords sampled at a rate greater than 0.07 percent. \n","text_end":6762,"text_sha256":"a13012f370078a41fd9ec84e21b7ea2c1590a42be1dd9027a7727acaad1caf19","text_start":6440},"multivariate_blurring":{"booklet":"main_2022","needle":"Multivariate blurring has been applied to these returns with nonzero values in at least","needle_sha256":"247d89d50a53cb8d3aca2d7d68bbf614ea0d161f22d4292b3734837ab5dbbe62","page_end":2544,"page_start":2197,"pdf_page":4,"printed_page":3,"text":"Multivariate blurring has been applied to these returns with nonzero values in at least \ntwo of the following fields: wages and salaries, state and local income taxes, and real \nestate taxes. Prior to blurring, these returns are grouped into one of 10 categories \nbased on their filing status and the number of dependents, and then further group","text_end":5200,"text_sha256":"4d2b0c5f244c64ea4a52f0b0aab3b22de9a05e1345edbd4742995106176f84ea","text_start":4853},"national_estimates_only":{"booklet":"main_2022","needle":"2015 Public Use Tax File is only appropriate for national level estima tes.","needle_sha256":"753825a8a47b9f7378dc3007206a4eea4a5ca35f84744d921230339b2ab68883","page_end":625,"page_start":550,"pdf_page":3,"printed_page":2,"text":"2015 Public Use Tax File is only appropriate for national level estima tes.","text_end":2133,"text_sha256":"753825a8a47b9f7378dc3007206a4eea4a5ca35f84744d921230339b2ab68883","text_start":2058},"nonitemized_wage_noise":{"booklet":"main_2022","needle":"For nonitemized tax returns, up to 10 percent of random noise, drawn from a uniform","needle_sha256":"9ceea058079776ba675b63c5194573294e651ef6699d218667eedd1c749e75d6","page_end":1462,"page_start":1159,"pdf_page":5,"printed_page":4,"text":"For nonitemized tax returns, up to 10 percent of random noise, drawn from a uniform \ndistribution, has been a dded to wages and salaries. The random noise has been \nconstrained as to not disqualify previously qualifying Earned Income Tax Credit \nrecipients or Retirement Saver’s Credit recipients. \n \n","text_end":7067,"text_sha256":"b51a7c2443aaeea041f2bbbd108e7571a39d8d65222c0cd082397eb239650a33","text_start":6764},"not_one_taxpayer":{"booklet":"main_2022","needle":"Since individual records in this file may or may not contain data from just one tax return","needle_sha256":"5f651403aea3e17b13254dfb1d4f6b48001ea3007a5dc40e36ff73d2869ee319","page_end":2913,"page_start":2538,"pdf_page":5,"printed_page":4,"text":"Since individual records in this file may or may not contain data from just one tax return--\nand never contain the full item content of any one tax return-- we request that researchers make \nthese facts clear in publishing their findings. In particular, they should refrain from using any \nlanguage that would imply that they had access to individual taxpayers' records. \n ","text_end":8518,"text_sha256":"99af06c9990f3dbb468724f54de3b711fd6d4a9e78b9c83672f5829db76b7ecb","text_start":8143},"p22250_definition":{"booklet":"main_2022","needle":"P22250 SHORT-TERM GAINS LESS LOSSES NET OF CARRYOVER","needle_sha256":"9ff26c5fade441cfb71aa8e9472922bbdf19a61b20955a053962097c76bc69c5","page_end":702,"page_start":450,"pdf_page":24,"printed_page":23,"text":"P22250 SHORT-TERM GAINS LESS LOSSES NET OF CARRYOVER \nAll short-term capital gains less losses shown in column (h) of Schedule D except \nfor the short-term capital loss carryover. \n \nP23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYO","text_end":43348,"text_sha256":"93098c0bfddb1298db9019e2a47657d95fd943c7cd5f6a76a239efc948518189","text_start":43096},"p23250_definition":{"booklet":"main_2022","needle":"P23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYOVER","needle_sha256":"3d19ab3284dd523bee3cb36f38f7abb88419fbfc3038b819a66f05a8d068b1bd","page_end":895,"page_start":643,"pdf_page":24,"printed_page":23,"text":"P23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYOVER \nAll long-term capital gains less losses shown in column (h) of Schedule D except \nfor the long-term capital loss carryover \n \nP25350 TOTAL RENTS AND ROYALTIES RECEIVED \n The sum of lines ","text_end":43541,"text_sha256":"419b64e90741764c0ed815a165821259e5860b27c7ef1fad217cc2a1f47d69a1","text_start":43289},"rebalancing":{"booklet":"main_2022","needle":"Finally, all records in the file are rebalanced to ensure accounting accuracy","needle_sha256":"b8255f0567486c82414832afad9cc3304cb7e294b39a430579976675f8164306","page_end":2572,"page_start":2405,"pdf_page":5,"printed_page":4,"text":"Finally, all records in the file are rebalanced to ensure accounting accuracy after the above \ndisclosure procedures are applied. \n \nSince individual records in this f","text_end":8177,"text_sha256":"4248cde7dd22d71756c2400db4837550b96aa519df2b7b6f9c6be71225e94347","text_start":8010},"record_count_intro":{"booklet":"main_2022","needle":"The Internal Revenue Service 2015 Public Use Tax File, which contains 207,696 records","needle_sha256":"27a686d1fbf3e6850267f7616c1e1dd87baa0af8ea0cc740a88a7052c1809add","page_end":155,"page_start":70,"pdf_page":3,"printed_page":2,"text":"The Internal Revenue Service 2015 Public Use Tax File, which contains 207,696 records","text_end":1663,"text_sha256":"27a686d1fbf3e6850267f7616c1e1dd87baa0af8ea0cc740a88a7052c1809add","text_start":1578},"record_count_technical":{"booklet":"main_2022","needle":"This comma separated file contains 207,696","needle_sha256":"7a30ba3ee0b6719fbdf0ccfdf9105407da36683a09d7876be92b62bbd6cda930","page_end":282,"page_start":150,"pdf_page":27,"printed_page":26,"text":"This comma separated file contains 207,696 \nrecords where the first record consists of only field names. Field names have been orde","text_end":47486,"text_sha256":"5ea7fb638fd255ff77436d39fd3f806ee64265ea2824f875e035c976fd83916e","text_start":47354},"rounding_grid":{"booklet":"main_2022","needle":"Ninth, for all records on the file, all amount fields have been rounded.","needle_sha256":"d9ab4ab8efe86cf0c235da1904e664c3a643cdfd2e10e15259a4d16d824d6a38","page_end":2447,"page_start":1955,"pdf_page":5,"printed_page":4,"text":"Ninth, for all records on the file, all amount fields have been rounded. Amounts, in absolute \nvalues, above $100,000 are rounded to the four most significant digits from left to right(e.g., \n$228,867 = $228,900 and $1,158,235 = $1,158,000). Amounts between $10,000 and $100,000 are \nrounded to the nearest $100. Amounts between $5 and $10,000 are rounded to the nearest $10. \nNonzero amounts less than $5 are set to $2, with sign retained. \n \n Finally, all records in the file are rebal","text_end":8052,"text_sha256":"eb56b16079f3e2bf633b7d72cecf8e20ca6e7837c7fc5570790129dcee517531","text_start":7560},"s006_definition":{"booklet":"main_2022","needle":"S006 DECIMAL WEIGHT","needle_sha256":"981896b68f40bde5639176999f2f2f856aca3cfb9b1ac4f967fe9e83c0830c81","page_end":1237,"page_start":958,"pdf_page":25,"printed_page":24,"text":"S006 DECIMAL WEIGHT \nThis field was derived by dividing the computer population count of returns in a \nsample stratum by the number of sample returns for that stratum (carried to two \ndecimal places). The decimal place is implied. Estimates derived from each \nreturn record mus","text_end":46079,"text_sha256":"92641e3f7dfc09690b3aed4c110767d01a15819f82ff04fee4fda546584c06bf","text_start":45800},"txrt_implied_decimal":{"booklet":"main_2022","needle":"that this field uses an implied decimal point (28.0 percent is represented by 280).","needle_sha256":"ef23bfa821ae8f4f3aa57420ae2ed24b987d6ea93780917e1288a70ce8f43be6","page_end":407,"page_start":324,"pdf_page":26,"printed_page":25,"text":"that this field uses an implied decimal point (28.0 percent is represented by 280).","text_end":46825,"text_sha256":"ef23bfa821ae8f4f3aa57420ae2ed24b987d6ea93780917e1288a70ce8f43be6","text_start":46742}},"authority":"irs_soi_2015_public_use_file_retained_booklets","booklets":{"demographic_2022":{"edition":"September 2022","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":345247,"pdf_pages":13,"pdf_sha256":"7bb623c7c7631e30eb620f0a522d74bff59487c1778d0b2023b2536cead5f38b","role":"primary_layout_authority","span_digest":"sha256 of the span encoded as UTF-8","text_characters":19352,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"d2074c3a2e2dabb6d25da4a9a62b8d9302e001cc5d86939edc0bdd09719d8c9a","title":"2015 Public Use Tax Demographic File"},"main_2022":{"edition":"September 2022","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":23417059,"pdf_pages":88,"pdf_sha256":"dfb2a6c85f98cba95624ece7c2bd239a0fd09457ca2498777c8242208ff7295b","role":"primary_layout_authority","span_digest":"sha256 of the span encoded as UTF-8","text_characters":93621,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"c2446b589f60592f337ed45f0d82406593fddafd16ec45d06440c07c35418765","title":"2015 Public Use Tax File General Description Booklet"},"main_2023":{"edition":"Updated February 2023","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":23413549,"pdf_pages":88,"pdf_sha256":"93a1d863b5a4af21b624714fb2bb3f9d30509defdd8f2c052c2c5ca4d9cce35c","role":"corroborating_edition","span_digest":"sha256 of the span encoded as UTF-8","text_characters":93783,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"c7f84a1ec266d341b713ed1251c7056b6636e6a8ac23ef1124f21fe367f34353","title":"2015 Public Use Tax File General Description Booklet"}},"generation":{"generator":"tools/build_us_puf_agi_source_layout.py","generator_sha256":"6fa9175f0e1dffd358b8118adb16d0d5ef8f797e3a5555bdba19ef842ec3f747","parent_generator":"tools/build_us_puf_source_layout.py","parent_generator_sha256":"c35b8b20342376d404427bb00f522b81e115e1f079ed53e81f923f9a7660b48b","reads":["the three pinned booklet text extractions","the packaged raw source definition","the packaged twelve-field projection document"],"reads_microdata":false},"layout":{"demographic":{"code_fields":7,"columns":[{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":115,"page_start":86,"pdf_page":6,"printed_page":5,"text":"1. AGEDP1 Age of Dependent 1 ","text_end":3806,"text_sha256":"209cf4c864128785757d242779538f0ee5b0bb935cca291d020ab7ed7efade3c","text_start":3777},"delivered_index":0,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":116,"page_start":104,"pdf_page":5,"printed_page":4,"text":" 1. AGEDP11 ","text_end":3224,"text_sha256":"e032518be848a70629a50c3b886430b0ec8b200475c4fa8b469d698f7794e478","text_start":3212},"name":"AGEDP1","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 1","publisher_position":1,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":1182,"page_start":1153,"pdf_page":6,"printed_page":5,"text":"2. AGEDP2 Age of Dependent 2 ","text_end":4873,"text_sha256":"52d4f54ef51fd158513cb53774a9b8c98d3c8b1e6334729bd22791f4937e6d5d","text_start":4844},"delivered_index":1,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":129,"page_start":117,"pdf_page":5,"printed_page":4,"text":" 2. AGEDP21 ","text_end":3237,"text_sha256":"586f75b3d4b4702611f91b89a26b212f0a62b5286740e55d440cb7644e897425","text_start":3225},"name":"AGEDP2","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 2","publisher_position":2,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":95,"page_start":66,"pdf_page":7,"printed_page":6,"text":"3. AGEDP3 Age of Dependent 3 ","text_end":6054,"text_sha256":"01751c44a83a96e86a4fcd33696ba5de01f4c9cae9e12a4090d433b747001ae1","text_start":6025},"delivered_index":2,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":142,"page_start":130,"pdf_page":5,"printed_page":4,"text":" 3. AGEDP31 ","text_end":3250,"text_sha256":"75e66f62320bbcd1968a85db6cae7a52578303cdfab10cb7266b2e68f0e047d7","text_start":3238},"name":"AGEDP3","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 3","publisher_position":3,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":1231,"page_start":1192,"pdf_page":7,"printed_page":6,"text":"4. AGERANGE Age range of Primary Filer ","text_end":7190,"text_sha256":"0d896e896bb5e35eed7e1cb2c70237c01d924333de5aef052e20aab5d5bbc042","text_start":7151},"delivered_index":3,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":156,"page_start":143,"pdf_page":5,"printed_page":4,"text":" 4. AGERANGE ","text_end":3264,"text_sha256":"137b0c36ae39e9423e67b23a641f6775455ff711e295633a910035d7ef7478ec","text_start":3251},"name":"AGERANGE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age range of Primary Filer","publisher_position":4,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":110,"page_start":64,"pdf_page":8,"printed_page":7,"text":"5. EARNSPLIT Earnings Split on Joint Returns ","text_end":8608,"text_sha256":"9e8326c12d2f58e09f9ce9d83568569dcc6468731e7dd34f58715992c19f2070","text_start":8562},"delivered_index":4,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":171,"page_start":157,"pdf_page":5,"printed_page":4,"text":" 5. EARNSPLIT ","text_end":3279,"text_sha256":"cd6115b551a6da5a2b21c7191327ea923514aee715de0b30a2e61f26c16d6f25","text_start":3265},"name":"EARNSPLIT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Earnings Split on Joint Returns","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":674,"page_start":640,"pdf_page":8,"printed_page":7,"text":"6. GENDER Gender of Primary filer ","text_end":9172,"text_sha256":"2a358c87a94b63cd540a29b76cc359a67de90819cfd3694430894b3df43ce631","text_start":9138},"delivered_index":5,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":183,"page_start":172,"pdf_page":5,"printed_page":4,"text":" 6. GENDER ","text_end":3291,"text_sha256":"af2cf4870ae8abfc99f88c9e3c90cc71c2d8b54eb9cba902fdfdb0e719b63670","text_start":3280},"name":"GENDER","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Gender of Primary filer","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":911,"page_start":890,"pdf_page":8,"printed_page":7,"text":"7. RECID RETURN ID ","text_end":9409,"text_sha256":"1415d1f01fe6e062d13b8eeb929909980bc0b9da00becda6f1b24b3570fc8144","text_start":9388},"delivered_index":6,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":194,"page_start":184,"pdf_page":5,"printed_page":4,"text":" 7. RECID ","text_end":3302,"text_sha256":"98dfcfd8c77226b0297f30f25fde9283b73b6ac87ef268d4f3de27ac109a7f01","text_start":3292},"name":"RECID","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"RETURN ID","publisher_position":7,"publisher_signed_marker":false}],"delivered_header_width":7,"printed_order_matches_delivered_order":true},"main":{"amount_or_large_code_fields":179,"code_fields":43,"columns":[{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":119,"page_start":75,"pdf_page":12,"printed_page":11,"text":"1. AGIR1 Adjusted Gross Income (AGI) Range: ","text_end":18261,"text_sha256":"f304d466d6ad08a9d463641c9b4667edae19aeb36eeca325da90db983145f1a4","text_start":18217},"delivered_index":0,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":102,"page_start":92,"pdf_page":6,"printed_page":5,"text":" 1. AGIR1 ","text_end":8621,"text_sha256":"ef23b0e0e7abf66606a29cda9d566a187f525c1373ed6f09fcb38398b5ca6023","text_start":8611},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":102,"page_start":92,"pdf_page":6,"printed_page":5,"text":" 1. AGIR1 ","text_end":8643,"text_sha256":"ef23b0e0e7abf66606a29cda9d566a187f525c1373ed6f09fcb38398b5ca6023","text_start":8633},"name":"AGIR1","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Adjusted Gross Income (AGI) Range:","publisher_position":1,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":490,"page_start":449,"pdf_page":13,"printed_page":12,"text":"2. CLAIM8965 Health Coverage Exemptions ","text_end":22012,"text_sha256":"377cee7d29bf4e186ead2daccd837bd50f9d7d4fd7b15fe22d4f1508c7fe49e5","text_start":21971},"delivered_index":1,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":120,"page_start":103,"pdf_page":6,"printed_page":5,"text":" 2. CLAIM89651,2 ","text_end":8639,"text_sha256":"34d92e2e3e846dd2917000322e0f7e05378045aebf1c9a2ef2982a07eb4be909","text_start":8622},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":120,"page_start":103,"pdf_page":6,"printed_page":5,"text":" 2. CLAIM89651,2 ","text_end":8661,"text_sha256":"34d92e2e3e846dd2917000322e0f7e05378045aebf1c9a2ef2982a07eb4be909","text_start":8644},"name":"CLAIM8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Health Coverage Exemptions","publisher_position":2,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":674,"page_start":639,"pdf_page":13,"printed_page":12,"text":"3. DSI Dependent Status Indicator: ","text_end":22196,"text_sha256":"907528bcc8c57c09dfb71bdf79228e3d3de69adf38663789ce8903ae3967a424","text_start":22161},"delivered_index":2,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":129,"page_start":121,"pdf_page":6,"printed_page":5,"text":" 3. DSI ","text_end":8648,"text_sha256":"cc9c81fed4dabdf7a1e6074ac8ea0aa291945cf834f127affadbec903e56d53f","text_start":8640},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":129,"page_start":121,"pdf_page":6,"printed_page":5,"text":" 3. DSI ","text_end":8670,"text_sha256":"cc9c81fed4dabdf7a1e6074ac8ea0aa291945cf834f127affadbec903e56d53f","text_start":8662},"name":"DSI","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Dependent Status Indicator:","publisher_position":3,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":949,"page_start":912,"pdf_page":13,"printed_page":12,"text":"4. EFI Electronic Filing Indicator: ","text_end":22471,"text_sha256":"a4467e5ca8e46f72746fb6491f40d66f94edcfaaaa123c4f0d071416f9c6ed70","text_start":22434},"delivered_index":3,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":138,"page_start":130,"pdf_page":6,"printed_page":5,"text":" 4. EFI ","text_end":8657,"text_sha256":"224488a72c014cd6368c5de18c7f75b3e313f13015e9c509ad995ba198c049d3","text_start":8649},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":138,"page_start":130,"pdf_page":6,"printed_page":5,"text":" 4. EFI ","text_end":8679,"text_sha256":"224488a72c014cd6368c5de18c7f75b3e313f13015e9c509ad995ba198c049d3","text_start":8671},"name":"EFI","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Electronic Filing Indicator:","publisher_position":4,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1163,"page_start":1129,"pdf_page":13,"printed_page":12,"text":"5. EIC Earned Income Credit Code: ","text_end":22685,"text_sha256":"9ef3462adfb166c47c84ebd57ffeeaf95f226274ce146d37f9964e2bd7b3545c","text_start":22651},"delivered_index":4,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":147,"page_start":139,"pdf_page":6,"printed_page":5,"text":" 5. EIC ","text_end":8666,"text_sha256":"ddfc450e354ad0e909bc48204bbdd4c82ca9d20981f5252575e9af54c88a2724","text_start":8658},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":147,"page_start":139,"pdf_page":6,"printed_page":5,"text":" 5. EIC ","text_end":8688,"text_sha256":"ddfc450e354ad0e909bc48204bbdd4c82ca9d20981f5252575e9af54c88a2724","text_start":8680},"name":"EIC","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Earned Income Credit Code:","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1594,"page_start":1542,"pdf_page":13,"printed_page":12,"text":"6. ELECT Presidential Election Campaign Fund Boxes: ","text_end":23116,"text_sha256":"e9e30b1db453e79532cef891c8990253a1909c6cc8323e6238e21a28ee71dfa2","text_start":23064},"delivered_index":5,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":158,"page_start":148,"pdf_page":6,"printed_page":5,"text":" 6. ELECT ","text_end":8677,"text_sha256":"7c5188295524711beef5fab70c2dd0960d52793a9da2647e2a1f20c362fd7a53","text_start":8667},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":158,"page_start":148,"pdf_page":6,"printed_page":5,"text":" 6. ELECT ","text_end":8699,"text_sha256":"7c5188295524711beef5fab70c2dd0960d52793a9da2647e2a1f20c362fd7a53","text_start":8689},"name":"ELECT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Presidential Election Campaign Fund Boxes:","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":85,"page_start":53,"pdf_page":14,"printed_page":13,"text":"7. FDED Form of Deduction Code: ","text_end":23460,"text_sha256":"c328a37c4c3f209b45630cdb9df3e0fcb9ab9eae39088e5d510ac1720ac068df","text_start":23428},"delivered_index":6,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":168,"page_start":159,"pdf_page":6,"printed_page":5,"text":" 7. FDED ","text_end":8687,"text_sha256":"e8e6d023b0c3a10a18cf3f5fb54a65342299f83be75b3378ab3f974ae93f8d1c","text_start":8678},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":168,"page_start":159,"pdf_page":6,"printed_page":5,"text":" 7. FDED ","text_end":8709,"text_sha256":"e8e6d023b0c3a10a18cf3f5fb54a65342299f83be75b3378ab3f974ae93f8d1c","text_start":8700},"name":"FDED","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form of Deduction Code:","publisher_position":7,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":494,"page_start":450,"pdf_page":14,"printed_page":13,"text":"8. FLPDYR Filing (Accounting) Period, Year: ","text_end":23869,"text_sha256":"e655ae86dedfa56d7356d721981cf206e43e290f2ab9d821f5ed75dc08447de0","text_start":23825},"delivered_index":7,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":180,"page_start":169,"pdf_page":6,"printed_page":5,"text":" 8. FLPDYR ","text_end":8699,"text_sha256":"2d43a4bd7b53098a69fad6d88ff71c93eac7f4209274a802d99fbb04102a83ce","text_start":8688},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":180,"page_start":169,"pdf_page":6,"printed_page":5,"text":" 8. FLPDYR ","text_end":8721,"text_sha256":"2d43a4bd7b53098a69fad6d88ff71c93eac7f4209274a802d99fbb04102a83ce","text_start":8710},"name":"FLPDYR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Filing (Accounting) Period, Year:","publisher_position":8,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":639,"page_start":594,"pdf_page":14,"printed_page":13,"text":"9. FLPDMO Filing (Accounting) Period, Month: ","text_end":24014,"text_sha256":"1052efbe06ec372bda396ff6873b18f310663c9000a439a954ffb0e62a190da3","text_start":23969},"delivered_index":8,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":192,"page_start":181,"pdf_page":6,"printed_page":5,"text":" 9. FLPDMO ","text_end":8711,"text_sha256":"21b628067e0e0cae93f82779561150bde7ed78afb74d69e455ed1e3186bd3fa4","text_start":8700},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":192,"page_start":181,"pdf_page":6,"printed_page":5,"text":" 9. FLPDMO ","text_end":8733,"text_sha256":"21b628067e0e0cae93f82779561150bde7ed78afb74d69e455ed1e3186bd3fa4","text_start":8722},"name":"FLPDMO","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Filing (Accounting) Period, Month:","publisher_position":9,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":793,"page_start":732,"pdf_page":14,"printed_page":13,"text":"10. F2441 Form 2441, Child Care Credit Qualified Individual: ","text_end":24168,"text_sha256":"b152765931e89ba3fb8448e01cfa70bbb8aa71b29e701486f78b2ce1ef17bee6","text_start":24107},"delivered_index":9,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":203,"page_start":193,"pdf_page":6,"printed_page":5,"text":"10. F2441 ","text_end":8722,"text_sha256":"01c52f4b2fe0d9d5165a17189ab4e5282b5b18d69ff2cad7e18d8c8a4f372c53","text_start":8712},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":203,"page_start":193,"pdf_page":6,"printed_page":5,"text":"10. F2441 ","text_end":8744,"text_sha256":"01c52f4b2fe0d9d5165a17189ab4e5282b5b18d69ff2cad7e18d8c8a4f372c53","text_start":8734},"name":"F2441","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 2441, Child Care Credit Qualified Individual:","publisher_position":10,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1016,"page_start":970,"pdf_page":14,"printed_page":13,"text":"11. F3800 Form 3800, General Business credit: ","text_end":24391,"text_sha256":"14e338b0441e0cb6c76e480bc1ba0b4e483c372dd5733f009574a1c8445d453d","text_start":24345},"delivered_index":10,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":214,"page_start":204,"pdf_page":6,"printed_page":5,"text":"11. F3800 ","text_end":8733,"text_sha256":"bb537dbb5a7aade7fc2c214184c697b080473fffe36d511244ebfffac3ba3778","text_start":8723},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":214,"page_start":204,"pdf_page":6,"printed_page":5,"text":"11. F3800 ","text_end":8755,"text_sha256":"bb537dbb5a7aade7fc2c214184c697b080473fffe36d511244ebfffac3ba3778","text_start":8745},"name":"F3800","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 3800, General Business credit:","publisher_position":11,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1252,"page_start":1201,"pdf_page":14,"printed_page":13,"text":" 12. F6251 Form 6251, Alternative Minimum Tax ","text_end":24627,"text_sha256":"83056f2f4b5d89d70af9812d722a74a903c11bc8171f4c288bd65372211ad4db","text_start":24576},"delivered_index":11,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":225,"page_start":215,"pdf_page":6,"printed_page":5,"text":"12. F6251 ","text_end":8744,"text_sha256":"07948b9784d84be63148935b012474f06a2f3c41b4fc9bef606b53b0600ef979","text_start":8734},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":225,"page_start":215,"pdf_page":6,"printed_page":5,"text":"12. F6251 ","text_end":8766,"text_sha256":"07948b9784d84be63148935b012474f06a2f3c41b4fc9bef606b53b0600ef979","text_start":8756},"name":"F6251","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 6251, Alternative Minimum Tax","publisher_position":12,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1489,"page_start":1433,"pdf_page":14,"printed_page":13,"text":"13. F8582 Form 8582, Passive Activity Loss Limitation: ","text_end":24864,"text_sha256":"f4fdb0f6d622efb5553da40f994ff677a5510b0cc317825441200248a6469f9e","text_start":24808},"delivered_index":12,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":237,"page_start":226,"pdf_page":6,"printed_page":5,"text":"13. F8582 ","text_end":8756,"text_sha256":"68a09f329357da168ad01490dbee20c1658f4f77e77bd10c7da496cc6b2fd5e5","text_start":8745},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":237,"page_start":226,"pdf_page":6,"printed_page":5,"text":"13. F8582 ","text_end":8778,"text_sha256":"68a09f329357da168ad01490dbee20c1658f4f77e77bd10c7da496cc6b2fd5e5","text_start":8767},"name":"F8582","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8582, Passive Activity Loss Limitation:","publisher_position":13,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":108,"page_start":53,"pdf_page":15,"printed_page":14,"text":"14. F8606 Form 8606, Nondeductible IRA Contributions: ","text_end":25160,"text_sha256":"dc8cb5749d573bb72e1c2b70d860134f5d815688cf28f497bb3511beba6f5cf8","text_start":25105},"delivered_index":13,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":248,"page_start":238,"pdf_page":6,"printed_page":5,"text":"14. F8606 ","text_end":8767,"text_sha256":"b0339bb6ca69ad94d567242f788c12cacef4870df79eba42789992fcc9aa4ef6","text_start":8757},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":248,"page_start":238,"pdf_page":6,"printed_page":5,"text":"14. F8606 ","text_end":8789,"text_sha256":"b0339bb6ca69ad94d567242f788c12cacef4870df79eba42789992fcc9aa4ef6","text_start":8779},"name":"F8606","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8606, Nondeductible IRA Contributions:","publisher_position":14,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":352,"page_start":290,"pdf_page":15,"printed_page":14,"text":"15. F8829 Form 8829, Expenses for business use of your Home: ","text_end":25404,"text_sha256":"9fd3c1b619f18c307090369c2e6ee66db0acc06fb5162eceb37d2bd5f4f3b2e0","text_start":25342},"delivered_index":14,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":259,"page_start":249,"pdf_page":6,"printed_page":5,"text":"15. F8829 ","text_end":8778,"text_sha256":"b0d9f7bfd115bf296b265117053937e4c8ce62bb4cdef9f8ab444767000b47ae","text_start":8768},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":259,"page_start":249,"pdf_page":6,"printed_page":5,"text":"15. F8829 ","text_end":8800,"text_sha256":"b0d9f7bfd115bf296b265117053937e4c8ce62bb4cdef9f8ab444767000b47ae","text_start":8790},"name":"F8829","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8829, Expenses for business use of your Home:","publisher_position":15,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":605,"page_start":534,"pdf_page":15,"printed_page":14,"text":"16. F8867 Form 8867, Paid Preparer’s Earned Income Credit Checklist ","text_end":25657,"text_sha256":"e1145d516646e92197631ec15785a0e97e8c2524882913c6ef91e728f48b17c3","text_start":25586},"delivered_index":15,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":260,"pdf_page":6,"printed_page":5,"text":"16. F8867 ","text_end":8789,"text_sha256":"61b533fadde404dbbeaed16ed60c54263faa32f9485e48cc4bb832d4ff20644e","text_start":8779},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":270,"page_start":260,"pdf_page":6,"printed_page":5,"text":"16. F8867 ","text_end":8811,"text_sha256":"61b533fadde404dbbeaed16ed60c54263faa32f9485e48cc4bb832d4ff20644e","text_start":8801},"name":"F8867","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8867, Paid Preparer’s Earned Income Credit Checklist","publisher_position":16,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":861,"page_start":793,"pdf_page":15,"printed_page":14,"text":"17. F8949 Form 8949, Sales and Other Dispositions of Capital Assets ","text_end":25913,"text_sha256":"e6f94330f8238e110113e378c222dd640b75cfede2fccd561f9851e9bc4f8d69","text_start":25845},"delivered_index":16,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":281,"page_start":271,"pdf_page":6,"printed_page":5,"text":"17. F8949 ","text_end":8800,"text_sha256":"ee402923ac3783fddb2f269f3a7910872dc6ed0fff422e5909e5fe5c704f4790","text_start":8790},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":281,"page_start":271,"pdf_page":6,"printed_page":5,"text":"17. F8949 ","text_end":8822,"text_sha256":"ee402923ac3783fddb2f269f3a7910872dc6ed0fff422e5909e5fe5c704f4790","text_start":8812},"name":"F8949","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8949, Sales and Other Dispositions of Capital Assets","publisher_position":17,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1094,"page_start":1049,"pdf_page":15,"printed_page":14,"text":"18. F8959 Form 8959, Additional Medicare Tax ","text_end":26146,"text_sha256":"568a604f5d031c5444be004ad8b02bb9f0946ad6f82b7673c05f5f8c953637ba","text_start":26101},"delivered_index":17,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":292,"page_start":282,"pdf_page":6,"printed_page":5,"text":"18. F8959 ","text_end":8811,"text_sha256":"3e61f77e7e3bdaae2ea51f63f7db2c68e0baeff67d48380adbbb3d504e485729","text_start":8801},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":292,"page_start":282,"pdf_page":6,"printed_page":5,"text":"18. F8959 ","text_end":8833,"text_sha256":"3e61f77e7e3bdaae2ea51f63f7db2c68e0baeff67d48380adbbb3d504e485729","text_start":8823},"name":"F8959","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8959, Additional Medicare Tax","publisher_position":18,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1330,"page_start":1282,"pdf_page":15,"printed_page":14,"text":"19. F8960 Form 8960, Net Investment Income Tax ","text_end":26382,"text_sha256":"bc4fce44984c18f262036bbe816dda8b4ef1df87b54fd8cb5ec175846ede6c70","text_start":26334},"delivered_index":18,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":303,"page_start":293,"pdf_page":6,"printed_page":5,"text":"19. F8960 ","text_end":8822,"text_sha256":"ee66926934ed3a726a77dc689b9b6ab98ca6aa9c05c13d438c723ba2119ee189","text_start":8812},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":303,"page_start":293,"pdf_page":6,"printed_page":5,"text":"19. F8960 ","text_end":8844,"text_sha256":"ee66926934ed3a726a77dc689b9b6ab98ca6aa9c05c13d438c723ba2119ee189","text_start":8834},"name":"F8960","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8960, Net Investment Income Tax","publisher_position":19,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1559,"page_start":1518,"pdf_page":15,"printed_page":14,"text":"20. F8962 Form 8962, Premium Tax Credit ","text_end":26611,"text_sha256":"961f04e8544622ebc45ffd98d69321f5bef6ee2fecc12a29e0da6e685173fe60","text_start":26570},"delivered_index":19,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":314,"page_start":304,"pdf_page":6,"printed_page":5,"text":"20. F8962 ","text_end":8833,"text_sha256":"2f63031c7b761dcc11e5828103e54bff5dfe3dc61cfb9bc882e41d4099db5a8e","text_start":8823},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":314,"page_start":304,"pdf_page":6,"printed_page":5,"text":"20. F8962 ","text_end":8855,"text_sha256":"2f63031c7b761dcc11e5828103e54bff5dfe3dc61cfb9bc882e41d4099db5a8e","text_start":8845},"name":"F8962","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8962, Premium Tax Credit","publisher_position":20,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":104,"page_start":55,"pdf_page":16,"printed_page":15,"text":"21. F8965 Form 8965, Health Coverage Exemptions ","text_end":26905,"text_sha256":"ae896abb0ad3155680f67895ba158bfa6231ca868f603f0766874e0b82dd5ef6","text_start":26856},"delivered_index":20,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":325,"page_start":315,"pdf_page":6,"printed_page":5,"text":"21. F8965 ","text_end":8844,"text_sha256":"73855028c4832b198c2180bdf2470509b5d4651da2c9cb0f7e59b8d46aed2503","text_start":8834},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":325,"page_start":315,"pdf_page":6,"printed_page":5,"text":"21. F8965 ","text_end":8866,"text_sha256":"73855028c4832b198c2180bdf2470509b5d4651da2c9cb0f7e59b8d46aed2503","text_start":8856},"name":"F8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8965, Health Coverage Exemptions","publisher_position":21,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":373,"page_start":292,"pdf_page":16,"printed_page":15,"text":"22. IE Itemized Deductions Election Indicator (Taxpayer elects to claim itemized ","text_end":27174,"text_sha256":"89bd2433f9d9626758da01462b21f40b8b2bd4718aa51d4a14e4f6e6b2752ba9","text_start":27093},"delivered_index":21,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":334,"page_start":326,"pdf_page":6,"printed_page":5,"text":"22. IE ","text_end":8853,"text_sha256":"a057fca32e6df91862eb45cced31eb61b4e244507f8ad327a386d9c49f1707e0","text_start":8845},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":334,"page_start":326,"pdf_page":6,"printed_page":5,"text":"22. IE ","text_end":8875,"text_sha256":"a057fca32e6df91862eb45cced31eb61b4e244507f8ad327a386d9c49f1707e0","text_start":8867},"name":"IE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Itemized Deductions Election Indicator (Taxpayer elects to claim itemized","publisher_position":22,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":703,"page_start":669,"pdf_page":16,"printed_page":15,"text":"23. MARS Marital (Filing) Status: ","text_end":27504,"text_sha256":"f6e3a499c660e5d7a98a143fe4112646260b5c1f4b044f1b9cbb579526a1e6ff","text_start":27470},"delivered_index":22,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":344,"page_start":335,"pdf_page":6,"printed_page":5,"text":"23. MARS ","text_end":8863,"text_sha256":"6ab746b927b75d8ca8116a16cf94f76d543c1a6b07d870bec86a67c3cbc6fa3a","text_start":8854},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":344,"page_start":335,"pdf_page":6,"printed_page":5,"text":"23. MARS ","text_end":8885,"text_sha256":"6ab746b927b75d8ca8116a16cf94f76d543c1a6b07d870bec86a67c3cbc6fa3a","text_start":8876},"name":"MARS","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Marital (Filing) Status:","publisher_position":23,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1480,"page_start":1403,"pdf_page":16,"printed_page":15,"text":"24. MIDR Married Filing Separately Itemized Deductions Requirement Indicator ","text_end":28281,"text_sha256":"54f50b4c999632fb255c8e4e089bac30d227f5c5264a1843b4dfaba4a0b2818c","text_start":28204},"delivered_index":23,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":354,"page_start":345,"pdf_page":6,"printed_page":5,"text":"24. MIDR ","text_end":8873,"text_sha256":"715abc6dfd87c8db8911ce5d31132613e16507d82be608ce30d84d3bd488792c","text_start":8864},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":354,"page_start":345,"pdf_page":6,"printed_page":5,"text":"24. MIDR ","text_end":8895,"text_sha256":"715abc6dfd87c8db8911ce5d31132613e16507d82be608ce30d84d3bd488792c","text_start":8886},"name":"MIDR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Married Filing Separately Itemized Deductions Requirement Indicator","publisher_position":24,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1726,"page_start":1669,"pdf_page":16,"printed_page":15,"text":"25. N24 Number of Children for Child Tax Credit ","text_end":28527,"text_sha256":"d7fa0439ae8cc3bca825a9e9cd326a09f2d63d5a14514af5daed980f2f1c4de3","text_start":28470},"delivered_index":24,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":362,"page_start":355,"pdf_page":6,"printed_page":5,"text":"25. N24","text_end":8881,"text_sha256":"64ee893dbe036d6f8e92fe63eb8e981dd69c7dfc4673a298c463c9b5bfd10491","text_start":8874},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":362,"page_start":355,"pdf_page":6,"printed_page":5,"text":"25. N24","text_end":8903,"text_sha256":"64ee893dbe036d6f8e92fe63eb8e981dd69c7dfc4673a298c463c9b5bfd10491","text_start":8896},"name":"N24","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Children for Child Tax Credit","publisher_position":25,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1851,"page_start":1782,"pdf_page":16,"printed_page":15,"text":"26. N25 Number of Qualified Students for Lifetime Learning Credit ","text_end":28652,"text_sha256":"71b80f1051188dd397ca82a77fd2ea48d31ed257cd027dac4502c2787edd1a9b","text_start":28583},"delivered_index":25,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":375,"page_start":366,"pdf_page":6,"printed_page":5,"text":"26. N251 ","text_end":8894,"text_sha256":"e67dbf65f66e23d7eeb486fcc555a5f28db5b6825f70dc148518fc19e2f443e4","text_start":8885},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":375,"page_start":366,"pdf_page":6,"printed_page":5,"text":"26. N251 ","text_end":8916,"text_sha256":"e67dbf65f66e23d7eeb486fcc555a5f28db5b6825f70dc148518fc19e2f443e4","text_start":8907},"name":"N25","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Qualified Students for Lifetime Learning Credit","publisher_position":26,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":134,"page_start":57,"pdf_page":17,"printed_page":16,"text":"27. N30 Number of Qualified Students for American Opportunity Credit ","text_end":28842,"text_sha256":"c42aead4d3c5742c32259a4ac711e3656e85576230e16f9006b6bcdb23d08634","text_start":28765},"delivered_index":26,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":386,"page_start":376,"pdf_page":6,"printed_page":5,"text":"27. N301 ","text_end":8905,"text_sha256":"271fb00168a07968994777a34e3403bc21928b7dc3ee7ac9bd5aee54c8c57c92","text_start":8895},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":386,"page_start":376,"pdf_page":6,"printed_page":5,"text":"27. N301 ","text_end":8927,"text_sha256":"271fb00168a07968994777a34e3403bc21928b7dc3ee7ac9bd5aee54c8c57c92","text_start":8917},"name":"N30","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Qualified Students for American Opportunity Credit","publisher_position":27,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":216,"page_start":193,"pdf_page":17,"printed_page":16,"text":"28. PREP Tax Preparer: ","text_end":28924,"text_sha256":"8a34bb702a90c77cad2ea1dd34526f242dd73867c82b5e0c651fc91ca3ed0fed","text_start":28901},"delivered_index":27,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":397,"page_start":387,"pdf_page":6,"printed_page":5,"text":"28. PREP ","text_end":8916,"text_sha256":"d1c5c0755db5e7466e609a50cd025069ef2545510b72edf16083dd3bc731f81f","text_start":8906},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":397,"page_start":387,"pdf_page":6,"printed_page":5,"text":"28. PREP ","text_end":8938,"text_sha256":"d1c5c0755db5e7466e609a50cd025069ef2545510b72edf16083dd3bc731f81f","text_start":8928},"name":"PREP","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Tax Preparer:","publisher_position":28,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":816,"page_start":744,"pdf_page":17,"printed_page":16,"text":"29. PREMNTHS Number of months enrolled in Health Insurance Marketplace: ","text_end":29524,"text_sha256":"32994b013e555ac88bc339cd4170dafe046092d35360e0e1b0e528a29aed8c93","text_start":29452},"delivered_index":28,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":411,"page_start":398,"pdf_page":6,"printed_page":5,"text":"29. PREMNTHS ","text_end":8930,"text_sha256":"73de2a8a9136440ff88b507a4e71d51c50ff10be496b23b168c5218d7609500d","text_start":8917},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":411,"page_start":398,"pdf_page":6,"printed_page":5,"text":"29. PREMNTHS ","text_end":8952,"text_sha256":"73de2a8a9136440ff88b507a4e71d51c50ff10be496b23b168c5218d7609500d","text_start":8939},"name":"PREMNTHS","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of months enrolled in Health Insurance Marketplace:","publisher_position":29,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":905,"page_start":873,"pdf_page":17,"printed_page":16,"text":"30. SCHB Schedule B Indicator: ","text_end":29613,"text_sha256":"3480701404d3bd7153400771e38fbfdbf2ef3192d494f5ef90d6fa48ae4c3ac2","text_start":29581},"delivered_index":29,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":421,"page_start":412,"pdf_page":6,"printed_page":5,"text":"30. SCHB ","text_end":8940,"text_sha256":"81699bc01628f8f813147f62b9ee4b0de71234bcdd3cae4b5b903119ffd1573f","text_start":8931},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":421,"page_start":412,"pdf_page":6,"printed_page":5,"text":"30. SCHB ","text_end":8962,"text_sha256":"81699bc01628f8f813147f62b9ee4b0de71234bcdd3cae4b5b903119ffd1573f","text_start":8953},"name":"SCHB","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule B Indicator:","publisher_position":30,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1134,"page_start":1097,"pdf_page":17,"printed_page":16,"text":"31. SCHCF Schedule C or F Indicator: ","text_end":29842,"text_sha256":"c6b2101f4c86a5908e5952f55a9cb68f4ca3bb6b5a95b22334e868458ee60023","text_start":29805},"delivered_index":30,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":432,"page_start":422,"pdf_page":6,"printed_page":5,"text":"31. SCHCF ","text_end":8951,"text_sha256":"c8d3cc667c96dd120043a8f2800ee14845c10eef8e9e444ec3c7423935891c4a","text_start":8941},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":432,"page_start":422,"pdf_page":6,"printed_page":5,"text":"31. SCHCF ","text_end":8973,"text_sha256":"c8d3cc667c96dd120043a8f2800ee14845c10eef8e9e444ec3c7423935891c4a","text_start":8963},"name":"SCHCF","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule C or F Indicator:","publisher_position":31,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1590,"page_start":1559,"pdf_page":17,"printed_page":16,"text":"32. SCHE Schedule E Indicator: ","text_end":30298,"text_sha256":"919c31e69c49c54b2210be9213c7e1bc005c84e1dd9d035df2914d6a1f54475c","text_start":30267},"delivered_index":31,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":442,"page_start":433,"pdf_page":6,"printed_page":5,"text":"32. SCHE ","text_end":8961,"text_sha256":"a0c8f93f41c9d9c5e2044a9f75b05f73d114be91707f44c59f9f1f825256fa3f","text_start":8952},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":442,"page_start":433,"pdf_page":6,"printed_page":5,"text":"32. SCHE ","text_end":8983,"text_sha256":"a0c8f93f41c9d9c5e2044a9f75b05f73d114be91707f44c59f9f1f825256fa3f","text_start":8974},"name":"SCHE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule E Indicator:","publisher_position":32,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":79,"page_start":53,"pdf_page":18,"printed_page":17,"text":"33. TFORM Form of Return: ","text_end":30562,"text_sha256":"3f48ca202e984c862e8b008783e2f9b0dd2c765af5d1d4d0cc0826e3080bd15d","text_start":30536},"delivered_index":32,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":453,"page_start":443,"pdf_page":6,"printed_page":5,"text":"33. TFORM ","text_end":8972,"text_sha256":"43c0167976d05afcca47000325ec120354c485d1b028898e6eaf970f792a9cc8","text_start":8962},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":453,"page_start":443,"pdf_page":6,"printed_page":5,"text":"33. TFORM ","text_end":8994,"text_sha256":"43c0167976d05afcca47000325ec120354c485d1b028898e6eaf970f792a9cc8","text_start":8984},"name":"TFORM","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form of Return:","publisher_position":33,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":432,"page_start":411,"pdf_page":18,"printed_page":17,"text":"34. TXST TAX STATUS: ","text_end":30915,"text_sha256":"1cc17672f79629fcb8faf909e2b10b4bd4bb21a5b183477d16f1f646eca8266b","text_start":30894},"delivered_index":33,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":463,"page_start":454,"pdf_page":6,"printed_page":5,"text":"34. TXST ","text_end":8982,"text_sha256":"a66590af6e4892e80d9a4cf549a5d06848b59fed91d548988c5b86f63237778e","text_start":8973},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":463,"page_start":454,"pdf_page":6,"printed_page":5,"text":"34. TXST ","text_end":9004,"text_sha256":"a66590af6e4892e80d9a4cf549a5d06848b59fed91d548988c5b86f63237778e","text_start":8995},"name":"TXST","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"TAX STATUS:","publisher_position":34,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1724,"page_start":1685,"pdf_page":18,"printed_page":17,"text":"35. XFPT Primary Taxpayer Exemption: ","text_end":32207,"text_sha256":"71b564ede2584f8fbd3452d53da0111f388951420b3e1346de10ff8bc68d3d3a","text_start":32168},"delivered_index":34,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":473,"page_start":464,"pdf_page":6,"printed_page":5,"text":"35. XFPT ","text_end":8992,"text_sha256":"d80624a20d2b2d40220541a1ee9a182ba47d57ff21453ca0a4e7ce9c4f574269","text_start":8983},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":473,"page_start":464,"pdf_page":6,"printed_page":5,"text":"35. XFPT ","text_end":9014,"text_sha256":"d80624a20d2b2d40220541a1ee9a182ba47d57ff21453ca0a4e7ce9c4f574269","text_start":9005},"name":"XFPT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Primary Taxpayer Exemption:","publisher_position":35,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1944,"page_start":1903,"pdf_page":18,"printed_page":17,"text":"36. XFST Secondary Taxpayer Exemption: ","text_end":32427,"text_sha256":"694ace59a6e0e0e1af6c5ead4b5bae1d33a608bb072d25e269660cf4c20d443c","text_start":32386},"delivered_index":35,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":483,"page_start":474,"pdf_page":6,"printed_page":5,"text":"36. XFST ","text_end":9002,"text_sha256":"01dc6c79a08a86010c3a477707ce8d180aa7e9f06850e41c97ae15df54bda59b","text_start":8993},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":483,"page_start":474,"pdf_page":6,"printed_page":5,"text":"36. XFST ","text_end":9024,"text_sha256":"01dc6c79a08a86010c3a477707ce8d180aa7e9f06850e41c97ae15df54bda59b","text_start":9015},"name":"XFST","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Secondary Taxpayer Exemption:","publisher_position":36,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":2175,"page_start":2125,"pdf_page":18,"printed_page":17,"text":"37. XOCAH Exemptions for Children Living at Home: ","text_end":32658,"text_sha256":"183eb94b7c211eb2d7c65a2a299f485007bd23be5d922f6a92c31558e988f106","text_start":32608},"delivered_index":36,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":495,"page_start":484,"pdf_page":6,"printed_page":5,"text":"37. XOCAH1 ","text_end":9014,"text_sha256":"74da3cf13cd316e9f3e5ef5479cfc61384d91777a65b487867565257ac96fc66","text_start":9003},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":493,"page_start":484,"pdf_page":6,"printed_page":5,"text":"37. XOCAH","text_end":9034,"text_sha256":"9f339784357264867577084eeff2324f32af7e6a78bfbb835195eb0156b92ced","text_start":9025},"name":"XOCAH","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Children Living at Home:","publisher_position":37,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":114,"page_start":55,"pdf_page":19,"printed_page":18,"text":"38. XOCAWH Exemptions for Children Living Away from Home: ","text_end":32976,"text_sha256":"6eb94eb248ac20032207633ec9fef8dede4d355e1ad0ce109cbd1b3688e9a96f","text_start":32917},"delivered_index":37,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":508,"page_start":496,"pdf_page":6,"printed_page":5,"text":"38. XOCAWH1 ","text_end":9027,"text_sha256":"70998a2ceb5b54883dc74cdfa3556083f867466ef55d4b278ff2ff8fb61a9f38","text_start":9015},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":509,"page_start":497,"pdf_page":6,"printed_page":5,"text":"38. XOCAWH1 ","text_end":9050,"text_sha256":"70998a2ceb5b54883dc74cdfa3556083f867466ef55d4b278ff2ff8fb61a9f38","text_start":9038},"name":"XOCAWH","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Children Living Away from Home:","publisher_position":38,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":378,"page_start":333,"pdf_page":19,"printed_page":18,"text":"39. XOODEP Exemptions for Other Dependents: ","text_end":33240,"text_sha256":"8428645125fbf4e150038bebb8d4132fafd9d844954bc3e424715f3d3778fff6","text_start":33195},"delivered_index":38,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":521,"page_start":509,"pdf_page":6,"printed_page":5,"text":"39. XOODEP1 ","text_end":9040,"text_sha256":"bc8e1545ee428c93ea46ef63d2043dcbbc3632a21be60b9da409dbed80b06c1d","text_start":9028},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":522,"page_start":510,"pdf_page":6,"printed_page":5,"text":"39. XOODEP1 ","text_end":9063,"text_sha256":"bc8e1545ee428c93ea46ef63d2043dcbbc3632a21be60b9da409dbed80b06c1d","text_start":9051},"name":"XOODEP","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Other Dependents:","publisher_position":39,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":680,"page_start":613,"pdf_page":19,"printed_page":18,"text":"40. XOPAR Exemptions for Parents Living at Home or Away from Home: ","text_end":33542,"text_sha256":"60838bc09ccfa6c113ba042622d48900789c3c806d7b5b0b5c24bf33b274bcf5","text_start":33475},"delivered_index":39,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":533,"page_start":522,"pdf_page":6,"printed_page":5,"text":"40. XOPAR1 ","text_end":9052,"text_sha256":"2134b12224986f1b7a019090eff091f29c2707dd253d98d17d50bb3c2fef28cd","text_start":9041},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":534,"page_start":523,"pdf_page":6,"printed_page":5,"text":"40. XOPAR1 ","text_end":9075,"text_sha256":"2134b12224986f1b7a019090eff091f29c2707dd253d98d17d50bb3c2fef28cd","text_start":9064},"name":"XOPAR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Parents Living at Home or Away from Home:","publisher_position":40,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":955,"page_start":927,"pdf_page":19,"printed_page":18,"text":"41. XTOT Total Exemptions: ","text_end":33817,"text_sha256":"161e71174e6137bfe5573edcc93198e876df78d5a4f1ea0a4e0a59201519de39","text_start":33789},"delivered_index":40,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":546,"page_start":534,"pdf_page":6,"printed_page":5,"text":"41. XTOT1,2 ","text_end":9065,"text_sha256":"b4edcb7ff97942f842ab84fdf1655dbbb9b6db60529a8b7cc35a83c74b086890","text_start":9053},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":547,"page_start":535,"pdf_page":6,"printed_page":5,"text":"41. XTOT1,2 ","text_end":9088,"text_sha256":"b4edcb7ff97942f842ab84fdf1655dbbb9b6db60529a8b7cc35a83c74b086890","text_start":9076},"name":"XTOT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Total Exemptions:","publisher_position":41,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1195,"page_start":1145,"pdf_page":19,"printed_page":18,"text":"42. XTOT8962 Number of Exemptions on Form 8962: ","text_end":34057,"text_sha256":"6e46fb37b1b3c3b9b1908649a4a0078c12a5712b627ad22942e14a0a8c165331","text_start":34007},"delivered_index":41,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":563,"page_start":547,"pdf_page":6,"printed_page":5,"text":"42. XTOT89621,2 ","text_end":9082,"text_sha256":"4de83366114efa6b6f2cb99bb131fb8dbebb8462de9010a91c17d6250848c87b","text_start":9066},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":564,"page_start":548,"pdf_page":6,"printed_page":5,"text":"42. XTOT89621,2 ","text_end":9105,"text_sha256":"4de83366114efa6b6f2cb99bb131fb8dbebb8462de9010a91c17d6250848c87b","text_start":9089},"name":"XTOT8962","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Exemptions on Form 8962:","publisher_position":42,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1351,"page_start":1301,"pdf_page":19,"printed_page":18,"text":"43. XTOT8965 Number of Exemptions on Form 8965: ","text_end":34213,"text_sha256":"f774c483511496dacf032950cc61f8a8dd14a92fe65ae958eccc3c674776f0de","text_start":34163},"delivered_index":42,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":580,"page_start":564,"pdf_page":6,"printed_page":5,"text":"43. XTOT89651,2 ","text_end":9099,"text_sha256":"59ed964dd25a600a3006f29fd696db60520b9091defce95a7df7c0f9676b12b5","text_start":9083},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":581,"page_start":565,"pdf_page":6,"printed_page":5,"text":"43. XTOT89651,2 ","text_end":9122,"text_sha256":"59ed964dd25a600a3006f29fd696db60520b9091defce95a7df7c0f9676b12b5","text_start":9106},"name":"XTOT8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Exemptions on Form 8965:","publisher_position":43,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":43,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9732,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9697},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9755,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9720},"name":"E00200","publisher_group":"amount_or_large_code","publisher_label":"Salaries and wages","publisher_position":1,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":44,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9759,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9733},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9782,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9756},"name":"E00300","publisher_group":"amount_or_large_code","publisher_label":"Interest received","publisher_position":2,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":45,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9795,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9760},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9818,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9783},"name":"E00400","publisher_group":"amount_or_large_code","publisher_label":"Tax-exempt interest income","publisher_position":3,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":46,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9830,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9796},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9853,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9819},"name":"E00600","publisher_group":"amount_or_large_code","publisher_label":"Dividends included in AGI","publisher_position":4,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":47,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9859,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9831},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9882,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9854},"name":"E00650","publisher_group":"amount_or_large_code","publisher_label":"Qualified Dividends","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":48,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":333,"page_start":300,"pdf_page":7,"printed_page":6,"text":"6 E00700 State income tax refunds","text_end":9893,"text_sha256":"f2b052b44e74bdf5f4870a1e46127391246bd3ee2edabe260c3388208dae7ab6","text_start":9860},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":333,"page_start":300,"pdf_page":7,"printed_page":6,"text":"6 E00700 State income tax refunds","text_end":9916,"text_sha256":"f2b052b44e74bdf5f4870a1e46127391246bd3ee2edabe260c3388208dae7ab6","text_start":9883},"name":"E00700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"State income tax refunds","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":49,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":360,"page_start":334,"pdf_page":7,"printed_page":6,"text":"7 E00800 Alimony received ","text_end":9920,"text_sha256":"95a7d728cb5f15e14fc8a10efecf4caee43cc2534ee505c5dab4f7d20faea4ea","text_start":9894},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":360,"page_start":334,"pdf_page":7,"printed_page":6,"text":"7 E00800 Alimony received ","text_end":9943,"text_sha256":"95a7d728cb5f15e14fc8a10efecf4caee43cc2534ee505c5dab4f7d20faea4ea","text_start":9917},"name":"E00800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alimony received","publisher_position":7,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":50,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":9987,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9921},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":10010,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9944},"name":"E00900","publisher_group":"amount_or_large_code","publisher_label":"Business or profession (Schedule C) net profit/loss (+/-)","publisher_position":8,"publisher_signed_marker":true},{"admission":"monetary_source_projection","delivered_index":51,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10027,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":9988},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10050,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":10011},"name":"E01000","publisher_group":"amount_or_large_code","publisher_label":"Net capital gain or loss (+/-)","publisher_position":9,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":52,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":526,"page_start":468,"pdf_page":7,"printed_page":6,"text":"10 E01100 Capital gain distributions reported on Form 1040","text_end":10086,"text_sha256":"c87bd839a618d48c0471a13631800dd39d82a9cd514d60bb6e693c6c1b7d3b90","text_start":10028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":526,"page_start":468,"pdf_page":7,"printed_page":6,"text":"10 E01100 Capital gain distributions reported on Form 1040","text_end":10109,"text_sha256":"c87bd839a618d48c0471a13631800dd39d82a9cd514d60bb6e693c6c1b7d3b90","text_start":10051},"name":"E01100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Capital gain distributions reported on Form 1040","publisher_position":10,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":53,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":565,"page_start":527,"pdf_page":7,"printed_page":6,"text":"11 E01200 Other gains (or loss) (+/-)","text_end":10125,"text_sha256":"642ca8e20a5621d511d4b6a77204b7f45ceab14cf33c85cce7ea54874daa28d3","text_start":10087},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":565,"page_start":527,"pdf_page":7,"printed_page":6,"text":"11 E01200 Other gains (or loss) (+/-)","text_end":10148,"text_sha256":"642ca8e20a5621d511d4b6a77204b7f45ceab14cf33c85cce7ea54874daa28d3","text_start":10110},"name":"E01200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other gains (or loss) (+/-)","publisher_position":11,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":54,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":600,"page_start":566,"pdf_page":7,"printed_page":6,"text":"12 E01400 Taxable IRA distribution","text_end":10160,"text_sha256":"310cb5b974160d517c423872da0cc305c6f9ce0beadc81aeca21971240b310b7","text_start":10126},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":600,"page_start":566,"pdf_page":7,"printed_page":6,"text":"12 E01400 Taxable IRA distribution","text_end":10183,"text_sha256":"310cb5b974160d517c423872da0cc305c6f9ce0beadc81aeca21971240b310b7","text_start":10149},"name":"E01400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Taxable IRA distribution","publisher_position":12,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":55,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10208,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10231,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10184},"name":"E01500","publisher_group":"amount_or_large_code","publisher_label":"Total pensions and annuities received","publisher_position":13,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":56,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10257,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10209},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10280,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10232},"name":"E01700","publisher_group":"amount_or_large_code","publisher_label":"Pensions and annuities included in AGI","publisher_position":14,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":57,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":743,"page_start":698,"pdf_page":7,"printed_page":6,"text":"15 E02000 Schedule E net income or loss (+/-)","text_end":10303,"text_sha256":"529435b1456ddff50c1d6d5cb2a84462492cbe3bcb2e87b571711a5c9cfe17f0","text_start":10258},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":743,"page_start":698,"pdf_page":7,"printed_page":6,"text":"15 E02000 Schedule E net income or loss (+/-)","text_end":10326,"text_sha256":"529435b1456ddff50c1d6d5cb2a84462492cbe3bcb2e87b571711a5c9cfe17f0","text_start":10281},"name":"E02000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule E net income or loss (+/-)","publisher_position":15,"publisher_signed_marker":true},{"admission":"monetary_source_projection","delivered_index":58,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10346,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10304},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10369,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10327},"name":"E02100","publisher_group":"amount_or_large_code","publisher_label":"Schedule F net profit/loss (+/-)","publisher_position":16,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":59,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":829,"page_start":787,"pdf_page":7,"printed_page":6,"text":"17 E02300 Unemployment compensation in AGI","text_end":10389,"text_sha256":"49fa031dafe5db53dac70ad0a8551a55d659d559c106f315d005fc8331f25199","text_start":10347},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":829,"page_start":787,"pdf_page":7,"printed_page":6,"text":"17 E02300 Unemployment compensation in AGI","text_end":10412,"text_sha256":"49fa031dafe5db53dac70ad0a8551a55d659d559c106f315d005fc8331f25199","text_start":10370},"name":"E02300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unemployment compensation in AGI","publisher_position":17,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":60,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":7,"printed_page":6,"text":"18 E02400 Gross Social Security benefits","text_end":10430,"text_sha256":"93ac4847542fa632ac7da30fa7a66bcc905a0a9f440f0eebb343a0fe20ace190","text_start":10390},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":7,"printed_page":6,"text":"18 E02400 Gross Social Security benefits","text_end":10453,"text_sha256":"93ac4847542fa632ac7da30fa7a66bcc905a0a9f440f0eebb343a0fe20ace190","text_start":10413},"name":"E02400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Gross Social Security benefits","publisher_position":18,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":61,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":912,"page_start":871,"pdf_page":7,"printed_page":6,"text":"19 E02500 Social Security benefits in AGI","text_end":10472,"text_sha256":"b2e9a9fd99b7b17ca804b24a09c86a6d62570404ec46b49900d0594139775f13","text_start":10431},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":912,"page_start":871,"pdf_page":7,"printed_page":6,"text":"19 E02500 Social Security benefits in AGI","text_end":10495,"text_sha256":"b2e9a9fd99b7b17ca804b24a09c86a6d62570404ec46b49900d0594139775f13","text_start":10454},"name":"E02500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Social Security benefits in AGI","publisher_position":19,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":62,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":994,"page_start":913,"pdf_page":7,"printed_page":6,"text":"Statutory 20 E03150 Total deductible individual retirement account (IRA) payments","text_end":10554,"text_sha256":"3ed3ece015295a7a5b0b3046629af4c012b0eeb5f6066ea87c963ae66e9b9c81","text_start":10473},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":994,"page_start":913,"pdf_page":7,"printed_page":6,"text":"Statutory 20 E03150 Total deductible individual retirement account (IRA) payments","text_end":10577,"text_sha256":"3ed3ece015295a7a5b0b3046629af4c012b0eeb5f6066ea87c963ae66e9b9c81","text_start":10496},"name":"E03150","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total deductible individual retirement account (IRA) payments","publisher_position":20,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":63,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1050,"page_start":995,"pdf_page":7,"printed_page":6,"text":"Adjustments 21 E03210 Student Loan Interest Deduction","text_end":10610,"text_sha256":"41e03c2ee377762c689e57a03f67ba4e2e031f3688f2ac02de62da16666b3885","text_start":10555},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1050,"page_start":995,"pdf_page":7,"printed_page":6,"text":"Adjustments 21 E03210 Student Loan Interest Deduction","text_end":10633,"text_sha256":"41e03c2ee377762c689e57a03f67ba4e2e031f3688f2ac02de62da16666b3885","text_start":10578},"name":"E03210","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Student Loan Interest Deduction","publisher_position":21,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":64,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1078,"page_start":1051,"pdf_page":7,"printed_page":6,"text":"22 E03220 Educator Expenses","text_end":10638,"text_sha256":"cb43a76c4c81b7ef03bf093a6cf492bb595b81ecf74804838adcf955aa699afa","text_start":10611},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1078,"page_start":1051,"pdf_page":7,"printed_page":6,"text":"22 E03220 Educator Expenses","text_end":10661,"text_sha256":"cb43a76c4c81b7ef03bf093a6cf492bb595b81ecf74804838adcf955aa699afa","text_start":10634},"name":"E03220","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Educator Expenses","publisher_position":22,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":65,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1115,"page_start":1079,"pdf_page":7,"printed_page":6,"text":"23 E03230 Tuition and Fees Deduction","text_end":10675,"text_sha256":"af3ba4332089496cb164d163ee3428b90f57b8301a6b2cb6af793224d24d0f0e","text_start":10639},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1115,"page_start":1079,"pdf_page":7,"printed_page":6,"text":"23 E03230 Tuition and Fees Deduction","text_end":10698,"text_sha256":"af3ba4332089496cb164d163ee3428b90f57b8301a6b2cb6af793224d24d0f0e","text_start":10662},"name":"E03230","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tuition and Fees Deduction","publisher_position":23,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":66,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1159,"page_start":1116,"pdf_page":7,"printed_page":6,"text":"24 E03260 Deduction for self-employment tax","text_end":10719,"text_sha256":"1697766007704455dbf9bef8986c2bf0975e5a71a0c87e9bd1e34c9b698db1e6","text_start":10676},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1159,"page_start":1116,"pdf_page":7,"printed_page":6,"text":"24 E03260 Deduction for self-employment tax","text_end":10742,"text_sha256":"1697766007704455dbf9bef8986c2bf0975e5a71a0c87e9bd1e34c9b698db1e6","text_start":10699},"name":"E03260","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Deduction for self-employment tax","publisher_position":24,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":67,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1210,"page_start":1160,"pdf_page":7,"printed_page":6,"text":"25 E03270 Self-employed health insurance deduction","text_end":10770,"text_sha256":"cf87ff838847798de4b572b71e86d7da07ff3090a60bf6fa4e616452d17412f8","text_start":10720},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1210,"page_start":1160,"pdf_page":7,"printed_page":6,"text":"25 E03270 Self-employed health insurance deduction","text_end":10793,"text_sha256":"cf87ff838847798de4b572b71e86d7da07ff3090a60bf6fa4e616452d17412f8","text_start":10743},"name":"E03270","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employed health insurance deduction","publisher_position":25,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":68,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1261,"page_start":1211,"pdf_page":7,"printed_page":6,"text":"26 E03240 Domestic Production Activities deduction","text_end":10821,"text_sha256":"0e8f3a8c01b8b891ffac6474d8cdd32ac5d56aecf8a7caca58959a71c05d0989","text_start":10771},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1261,"page_start":1211,"pdf_page":7,"printed_page":6,"text":"26 E03240 Domestic Production Activities deduction","text_end":10844,"text_sha256":"0e8f3a8c01b8b891ffac6474d8cdd32ac5d56aecf8a7caca58959a71c05d0989","text_start":10794},"name":"E03240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Domestic Production Activities deduction","publisher_position":26,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":69,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1304,"page_start":1262,"pdf_page":7,"printed_page":6,"text":"27 E03290 Health Savings Account deduction","text_end":10864,"text_sha256":"12be5316231352f334c79873cf0ebef86e8fdcc81687b2656ba7866290550838","text_start":10822},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1304,"page_start":1262,"pdf_page":7,"printed_page":6,"text":"27 E03290 Health Savings Account deduction","text_end":10887,"text_sha256":"12be5316231352f334c79873cf0ebef86e8fdcc81687b2656ba7866290550838","text_start":10845},"name":"E03290","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Health Savings Account deduction","publisher_position":27,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":70,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1342,"page_start":1305,"pdf_page":7,"printed_page":6,"text":"28 E03300 Payments to KEOGH accounts ","text_end":10902,"text_sha256":"7dae2808be86e6f0bbf5855f0fee2a388e485d2b1a99dcb6406c859b3f8536ad","text_start":10865},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1342,"page_start":1305,"pdf_page":7,"printed_page":6,"text":"28 E03300 Payments to KEOGH accounts ","text_end":10925,"text_sha256":"7dae2808be86e6f0bbf5855f0fee2a388e485d2b1a99dcb6406c859b3f8536ad","text_start":10888},"name":"E03300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Payments to KEOGH accounts","publisher_position":28,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":71,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1380,"page_start":1343,"pdf_page":7,"printed_page":6,"text":"29 E03400 Forfeited interest penalty ","text_end":10940,"text_sha256":"867401e61c00cab7dde702e81c565e3f757bdbcbf2af32477834dfdc0157a4da","text_start":10903},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1380,"page_start":1343,"pdf_page":7,"printed_page":6,"text":"29 E03400 Forfeited interest penalty ","text_end":10963,"text_sha256":"867401e61c00cab7dde702e81c565e3f757bdbcbf2af32477834dfdc0157a4da","text_start":10926},"name":"E03400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Forfeited interest penalty","publisher_position":29,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":72,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1404,"page_start":1381,"pdf_page":7,"printed_page":6,"text":"30 E03500 Alimony paid ","text_end":10964,"text_sha256":"a242a0ae85f5b2e7bc3d0bc4c8c4265435beb5299fc68f7a05f1b06108a9615c","text_start":10941},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1404,"page_start":1381,"pdf_page":7,"printed_page":6,"text":"30 E03500 Alimony paid ","text_end":10987,"text_sha256":"a242a0ae85f5b2e7bc3d0bc4c8c4265435beb5299fc68f7a05f1b06108a9615c","text_start":10964},"name":"E03500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alimony paid","publisher_position":30,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":73,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1464,"page_start":1405,"pdf_page":7,"printed_page":6,"text":"AGI 31 E00100 Adjusted Gross Income (deficit) (AGI) (+/-)","text_end":11024,"text_sha256":"40277b800d8de41a73e6df8b5a317a17bf4434f5b70f0176d19b390cbe993012","text_start":10965},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1464,"page_start":1405,"pdf_page":7,"printed_page":6,"text":"AGI 31 E00100 Adjusted Gross Income (deficit) (AGI) (+/-)","text_end":11047,"text_sha256":"40277b800d8de41a73e6df8b5a317a17bf4434f5b70f0176d19b390cbe993012","text_start":10988},"name":"E00100","publisher_group":"amount_or_large_code","publisher_label":"Adjusted Gross Income (deficit) (AGI) (+/-)","publisher_position":31,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":74,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":113,"page_start":59,"pdf_page":8,"printed_page":7,"text":"Tax 32 P04470 Total deductions (standard or itemized) ","text_end":11138,"text_sha256":"7217c3a84ba6199a4568c02ecfe0b4e7f6595ddb3750a5747ed673b2fcecf75a","text_start":11084},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":114,"page_start":59,"pdf_page":8,"printed_page":7,"text":"Tax 32 P 04470 Total deductions (standard or itemized) ","text_end":11162,"text_sha256":"e64b39c4f1d50937158869f0ab4f13c5e99d5fd0f04b534cba45e6b011587889","text_start":11107},"name":"P04470","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total deductions (standard or itemized)","publisher_position":32,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":75,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":152,"page_start":114,"pdf_page":8,"printed_page":7,"text":"Computation 33 E04600 Exemption amount","text_end":11177,"text_sha256":"10be05859eaeec4a0842c04c01138f0c5d68d521ec82d36102844cf3fa652c6a","text_start":11139},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":154,"page_start":115,"pdf_page":8,"printed_page":7,"text":"Computation 33 E 04600 Exemption amount","text_end":11202,"text_sha256":"4a96690a9d89c712a55dd39d78f8a14a62ef3ef74d00fabc87b793ab57c50a9c","text_start":11163},"name":"E04600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Exemption amount","publisher_position":33,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":76,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":177,"page_start":153,"pdf_page":8,"printed_page":7,"text":"34 E04800 Taxable income","text_end":11202,"text_sha256":"225956f7e91356f9f90468a4395e1ed1451117a10725a5582a366af9ebb73665","text_start":11178},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":180,"page_start":155,"pdf_page":8,"printed_page":7,"text":"34 E04800 T axable income","text_end":11228,"text_sha256":"a6647ad88a4fb78304bb79035e3acbc2255212638c8a5c9f4ed9c7e08d1abd50","text_start":11203},"name":"E04800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Taxable income","publisher_position":34,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":77,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":209,"page_start":178,"pdf_page":8,"printed_page":7,"text":"35 E05100 Tax on taxable income","text_end":11234,"text_sha256":"f07c9259b17f51670ba7efd80c7b97ab1ecaec58f086b2684c76ec05bd23e7c2","text_start":11203},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":213,"page_start":181,"pdf_page":8,"printed_page":7,"text":"35 E05100 T ax on taxable income","text_end":11261,"text_sha256":"ba3d243665c0afd6cf245154c47ff83a499abb58a5acd53f88cc2fec45506c10","text_start":11229},"name":"E05100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax on taxable income","publisher_position":35,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":78,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":240,"page_start":210,"pdf_page":8,"printed_page":7,"text":"36 E05200 Computed regular tax","text_end":11265,"text_sha256":"7fe4593d18d6f23f85a0d57e4cc326a7d703a8df348bac50c6ca0ca94d33141e","text_start":11235},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":245,"page_start":214,"pdf_page":8,"printed_page":7,"text":"36 E05200 C omputed regular tax","text_end":11293,"text_sha256":"610ccde83a35567b71be3f7b134f6c2b7fd49a5002144b9fdb9adab087a18071","text_start":11262},"name":"E05200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Computed regular tax","publisher_position":36,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":79,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":276,"page_start":241,"pdf_page":8,"printed_page":7,"text":"37 E05800 Income tax before credits","text_end":11301,"text_sha256":"d6c09afb0eb5859b5f2303f4c57a3b439ab09c3d50b55e30fdd42f95cbbfe1f3","text_start":11266},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":282,"page_start":246,"pdf_page":8,"printed_page":7,"text":"37 E05800 I ncome tax before credits","text_end":11330,"text_sha256":"5cfba19950c904e1b2ed68779913fa3dca9a195d55aaed231f3f23a340f5a83f","text_start":11294},"name":"E05800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income tax before credits","publisher_position":37,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":80,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":308,"page_start":277,"pdf_page":8,"printed_page":7,"text":"38 E06000 Income subject to tax","text_end":11333,"text_sha256":"3aab3e844780e493265bfee48edef4d92c8e523e3af3bae76a3024c5de49bdde","text_start":11302},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":315,"page_start":283,"pdf_page":8,"printed_page":7,"text":"38 E06000 I ncome subject to tax","text_end":11363,"text_sha256":"3d72d60a6987087644a188050422acb8ca44ee4741dc53049113416d0a64f2fb","text_start":11331},"name":"E06000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income subject to tax","publisher_position":38,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":81,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":336,"page_start":309,"pdf_page":8,"printed_page":7,"text":"39 E06200 Marginal tax base","text_end":11361,"text_sha256":"cc9cabdcad93c863ec588b36ec7398ea2799fee802d92d209fc351647ec06854","text_start":11334},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":344,"page_start":316,"pdf_page":8,"printed_page":7,"text":"39 E06200 M arginal tax base","text_end":11392,"text_sha256":"05bba14fbadd5949179ca9ecdb80a849e9d9dc81f66ff24c932ec24b8b55a410","text_start":11364},"name":"E06200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Marginal tax base","publisher_position":39,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":82,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":378,"page_start":337,"pdf_page":8,"printed_page":7,"text":"40 E06300 Tax generated (tax rate tables)","text_end":11403,"text_sha256":"2d912e7ead6d8b12b0e1576cf402112dec8ed1f00ddb95d70eb96115a1e46727","text_start":11362},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":387,"page_start":345,"pdf_page":8,"printed_page":7,"text":"40 E06300 T ax generated (tax rate tables)","text_end":11435,"text_sha256":"3b2e76a22d70602f18562832d61d4c5386a4504648490bef9a7f964bd4c2861c","text_start":11393},"name":"E06300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax generated (tax rate tables)","publisher_position":40,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":83,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":412,"page_start":379,"pdf_page":8,"printed_page":7,"text":"41 E09600 Alternative minimum tax","text_end":11437,"text_sha256":"21de44978c2d3b1c06219dce1f35457e2d63c70380b2c3b508730e5a9b729920","text_start":11404},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":422,"page_start":388,"pdf_page":8,"printed_page":7,"text":"41 E09600 A lternative minimum tax","text_end":11470,"text_sha256":"4f1df3f395a937de11095cc2726d9adc9fdda85de4436a28ff4bdcdd1e23615e","text_start":11436},"name":"E09600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum tax","publisher_position":41,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":84,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":462,"page_start":413,"pdf_page":8,"printed_page":7,"text":"Credits 42 E07180 Child and Dependent Care Credit","text_end":11487,"text_sha256":"c7f8b0a16eed143191bdc26d8673a4fa7f0bb26b77163f4725e7b1a6fef829c3","text_start":11438},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":473,"page_start":423,"pdf_page":8,"printed_page":7,"text":"Credits 42 E 07180 Child and Dependent Care Credit","text_end":11521,"text_sha256":"6c7fb4017550d824a8580a9f58783f80834bfcd18c87eb7221495274eb83c03b","text_start":11471},"name":"E07180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Child and Dependent Care Credit","publisher_position":42,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":85,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":499,"page_start":463,"pdf_page":8,"printed_page":7,"text":"43 E07200 Elderly or Disabled Credit","text_end":11524,"text_sha256":"86a25fa2ff94fabc225ba5db603f961c7ddfc77a505a105cac63779c8a1bbeb2","text_start":11488},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":511,"page_start":474,"pdf_page":8,"printed_page":7,"text":"43 E07200 E lderly or Disabled Credit","text_end":11559,"text_sha256":"dbd830a3a7c0fd4c8518a9c65fa2aa01bbee66f26e05021e536fb7e785238121","text_start":11522},"name":"E07200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Elderly or Disabled Credit","publisher_position":43,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":86,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":528,"page_start":500,"pdf_page":8,"printed_page":7,"text":"44 E07220 Child Tax Credit","text_end":11553,"text_sha256":"4665ac1c4be769c23bd4e874cf7ac0be0e3fc159c3eb99cd45755c75788fec10","text_start":11525},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":541,"page_start":512,"pdf_page":8,"printed_page":7,"text":"44 E07220 C hild Tax Credit","text_end":11589,"text_sha256":"2e8c10e2c4b8c72efcb18d8b1b8ac1371dce6ad2cb64dabd8c0f01f7f54e06d7","text_start":11560},"name":"E07220","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Child Tax Credit","publisher_position":44,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":87,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":556,"page_start":529,"pdf_page":8,"printed_page":7,"text":"45 E07230 Education Credits","text_end":11581,"text_sha256":"d22ce5cc8537272d49db695f282bc049e59da6d3f7ba71f8a2d659cda3cec245","text_start":11554},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":570,"page_start":542,"pdf_page":8,"printed_page":7,"text":"45 E07230 E ducation Credits","text_end":11618,"text_sha256":"cf53661d3150bb55482e3f4bf94cd39b1634b576f99f2056af72ffe888463a01","text_start":11590},"name":"E07230","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Education Credits","publisher_position":45,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":88,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":592,"page_start":557,"pdf_page":8,"printed_page":7,"text":"46 E07240 Retirement Savings Credit","text_end":11617,"text_sha256":"ed0581e66c4d315b00544c6c5870f50d59b78866d11ba70ab7b8f1c806530828","text_start":11582},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":607,"page_start":571,"pdf_page":8,"printed_page":7,"text":"46 E07240 R etirement Savings Credit","text_end":11655,"text_sha256":"01c2172fb6749206e21e3ac233baedc6f036f9c658a26495018ba37eabada65e","text_start":11619},"name":"E07240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Retirement Savings Credit","publisher_position":46,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":89,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":628,"page_start":593,"pdf_page":8,"printed_page":7,"text":"47 E07260 Residential Energy Credit","text_end":11653,"text_sha256":"e4b21e879dead9af53a59d70b1ad032f1df39ab3c0f0a58779d932d7b0303ad1","text_start":11618},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":644,"page_start":608,"pdf_page":8,"printed_page":7,"text":"47 E07260 R esidential Energy Credit","text_end":11692,"text_sha256":"5bfb63e57d318883c240ab16740b02956147a641a5af9e1053af50bc180e1ab2","text_start":11656},"name":"E07260","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Residential Energy Credit","publisher_position":47,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":90,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":657,"page_start":629,"pdf_page":8,"printed_page":7,"text":"48 E07300 Foreign Tax Credit","text_end":11682,"text_sha256":"b811c7f90feb7afd9cdd277dee25be672d7e35627f734a106464c6fb1c3a4c9a","text_start":11654},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":674,"page_start":645,"pdf_page":8,"printed_page":7,"text":"48 E07300 F oreign Tax Credit","text_end":11722,"text_sha256":"b995a12a2eb3a80e72a1c84879e2785fb79b77a2facd459e532a0d96dd4c81e0","text_start":11693},"name":"E07300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Foreign Tax Credit","publisher_position":48,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":91,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":691,"page_start":658,"pdf_page":8,"printed_page":7,"text":"49 E07400 General Business Credit","text_end":11716,"text_sha256":"ce11f898797d8cfb0d3c67d6a3fe1786a7bf4c6deb9c35ab962a1bb568694e44","text_start":11683},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":709,"page_start":675,"pdf_page":8,"printed_page":7,"text":"49 E07400 G eneral Business Credit","text_end":11757,"text_sha256":"18b5a70022d438fb26656e16d8df7f40b5ce6a4dda85d539cf47abeb304c0185","text_start":11723},"name":"E07400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"General Business Credit","publisher_position":49,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":92,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":735,"page_start":692,"pdf_page":8,"printed_page":7,"text":"50 E07600 Credit for prior year minimum tax","text_end":11760,"text_sha256":"5bda2c85cf5a6668471b1a32d74e1647ca9faa346522bad6acc959bdb8575e94","text_start":11717},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":754,"page_start":710,"pdf_page":8,"printed_page":7,"text":"50 E07600 C redit for prior year minimum tax","text_end":11802,"text_sha256":"879f9c80102582a44633fb593718e762162ad1efa2e827282931eb169033d5cb","text_start":11758},"name":"E07600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit for prior year minimum tax","publisher_position":50,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":93,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":763,"page_start":736,"pdf_page":8,"printed_page":7,"text":"51 P08000 Other Tax Credits","text_end":11788,"text_sha256":"a06e7b7c20d74977127cbbe60401c09eae0a6d6fd1108537e90f6978f0e10699","text_start":11761},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":783,"page_start":755,"pdf_page":8,"printed_page":7,"text":"51 P08000 O ther Tax Credits","text_end":11831,"text_sha256":"bdcb3d710589efc74e919875a86a1529492ef024d550db7993e73d78248b2490","text_start":11803},"name":"P08000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other Tax Credits","publisher_position":51,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":94,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":796,"page_start":764,"pdf_page":8,"printed_page":7,"text":"52 E07150 Total tax credit (SOI)","text_end":11821,"text_sha256":"2bdd809fb4a2b5fad127d86f1919a4eb45e4a13e88553e5849d1b42691793dc4","text_start":11789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":817,"page_start":784,"pdf_page":8,"printed_page":7,"text":"52 E07150 T otal tax credit (SOI)","text_end":11865,"text_sha256":"7ed6338920c3da222745d61d65971c36f37303971c4a98ab432942f3e2405b75","text_start":11832},"name":"E07150","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax credit (SOI)","publisher_position":52,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":95,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":829,"page_start":797,"pdf_page":8,"printed_page":7,"text":"Taxes 53 E06500 Total income tax","text_end":11854,"text_sha256":"1469e4a861ea96162067edf32f7c0f8c7ae68a498e754ee9f14ec96fd48560d5","text_start":11822},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":851,"page_start":818,"pdf_page":8,"printed_page":7,"text":"Taxes 53 E 06500 Total income tax","text_end":11899,"text_sha256":"0762def63b6cd9e122bc748d2727e061ddc698c4d84aff1d7abfd16f273c5570","text_start":11866},"name":"E06500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total income tax","publisher_position":53,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":96,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":8,"printed_page":7,"text":"54 E08800 Income Tax after Credits (SOI)","text_end":11895,"text_sha256":"a9a103dc7547c727ba65a7d6b64450ed88d4885943ae06015eccc657f399f6aa","text_start":11855},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":893,"page_start":852,"pdf_page":8,"printed_page":7,"text":"54 E08800 I ncome Tax after Credits (SOI)","text_end":11941,"text_sha256":"bfe3a1c2c853909de7ad99d58a24f82482cd65ded6de5062ebe6d6754a1bf57a","text_start":11900},"name":"E08800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income Tax after Credits (SOI)","publisher_position":54,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":97,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":900,"page_start":871,"pdf_page":8,"printed_page":7,"text":"55 E09400 Self-employment tax","text_end":11925,"text_sha256":"7aae3feca6c81e9898b38d2470e6d52d320ca078a5a3f5d7bdcda51081702022","text_start":11896},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":924,"page_start":894,"pdf_page":8,"printed_page":7,"text":"55 E09400 S elf-employment tax","text_end":11972,"text_sha256":"cf8699e596032b9fd4764f46b334bb2ea3503dfc55f97310705e534ff943c397","text_start":11942},"name":"E09400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment tax","publisher_position":55,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":98,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":926,"page_start":901,"pdf_page":8,"printed_page":7,"text":"56 E09700 Recapture taxes","text_end":11951,"text_sha256":"95614e6f4cb3a76ceffb9e17d6d31eee5be0df29a4b8650bda98e8a996a56dd7","text_start":11926},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":951,"page_start":925,"pdf_page":8,"printed_page":7,"text":"56 E09700 R ecapture taxes","text_end":11999,"text_sha256":"d69e68dc57d670091cc988385d97b58e46d749157bab78600ef07f44e5bb3461","text_start":11973},"name":"E09700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Recapture taxes","publisher_position":56,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":99,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":966,"page_start":927,"pdf_page":8,"printed_page":7,"text":"57 E09730 Total Additional Medicare Tax","text_end":11991,"text_sha256":"76e56e65c3ea9269de1ac7c4f517f6a6e3e7a7190e817728f9c87f00be8900bd","text_start":11952},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":992,"page_start":952,"pdf_page":8,"printed_page":7,"text":"57 E09730 T otal Additional Medicare Tax","text_end":12040,"text_sha256":"fc1bdab89bd58d4992f505a48ba4bb50fe3489533304c7f3d0093864b5f4dd0d","text_start":12000},"name":"E09730","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Additional Medicare Tax","publisher_position":57,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":100,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1002,"page_start":967,"pdf_page":8,"printed_page":7,"text":"58 E09740 Net Investment Income Tax","text_end":12027,"text_sha256":"ae779721c98e842e8f3c73eb8c052ea9437311f2fcf4a78596d78b6aedbcf0f5","text_start":11992},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1029,"page_start":993,"pdf_page":8,"printed_page":7,"text":"58 E09740 N et Investment Income Tax","text_end":12077,"text_sha256":"2eea0be3d95ca8e28846e33d87213963674c000be2dcea9164ac84fb03b1f02c","text_start":12041},"name":"E09740","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Investment Income Tax","publisher_position":58,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":101,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1058,"page_start":1003,"pdf_page":8,"printed_page":7,"text":"59 E09750 Health Care Individual Responsibility Payment","text_end":12083,"text_sha256":"2fc634d9369fdfd2e26fcf87d86b9cad2a1d3f19045f84a538757332d437de2f","text_start":12028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1086,"page_start":1030,"pdf_page":8,"printed_page":7,"text":"59 E09750 H ealth Care Individual Responsibility Payment","text_end":12134,"text_sha256":"81052da246ea49aaf486e06f39b37d3c4e8953091e45fc44079beb76b8553f33","text_start":12078},"name":"E09750","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Health Care Individual Responsibility Payment","publisher_position":59,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":102,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1102,"page_start":1059,"pdf_page":8,"printed_page":7,"text":"60 E09800 Social security tax on tip income","text_end":12127,"text_sha256":"dfe787c463d2f18754a247ff0e9f2e2afa79f7e7a5ce8e1cc157053c1ba85559","text_start":12084},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1131,"page_start":1087,"pdf_page":8,"printed_page":7,"text":"60 E09800 S ocial security tax on tip income","text_end":12179,"text_sha256":"758076146bbb9e4271a02412e0002e077dc923ef78436fd97950230e6f833d18","text_start":12135},"name":"E09800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Social security tax on tip income","publisher_position":60,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":103,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1131,"page_start":1103,"pdf_page":8,"printed_page":7,"text":"61 E09900 Penalty tax on IRA","text_end":12156,"text_sha256":"28bac09e2a8048c28bf53b7c47990cf81509b535662409223b7a9427ee34c031","text_start":12128},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1161,"page_start":1132,"pdf_page":8,"printed_page":7,"text":"61 E09900 P enalty tax on IRA","text_end":12209,"text_sha256":"b99dbce1d89eba9969a0306947a1c7457d14bd6b57851c24248c391017932c7a","text_start":12180},"name":"E09900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Penalty tax on IRA","publisher_position":61,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":104,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1167,"page_start":1132,"pdf_page":8,"printed_page":7,"text":"62 E10300 Total tax liability (SOI)","text_end":12192,"text_sha256":"e556f48e984fbab4c1a608f63717e57b75e7d6987ad01672a5e5864ba6ac373c","text_start":12157},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1198,"page_start":1162,"pdf_page":8,"printed_page":7,"text":"62 E10300 T otal tax liability (SOI)","text_end":12246,"text_sha256":"5c4be64fc37f325dc96809408c1baab97b0537551ae9d110e7c183bfbbc4918e","text_start":12210},"name":"E10300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax liability (SOI)","publisher_position":62,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":105,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":95,"page_start":57,"pdf_page":9,"printed_page":8,"text":"Payments 63 E10700 Income tax withheld","text_end":12288,"text_sha256":"2ec1675834a00e512a58c9381864b776b714ed4373cdddae32895d985a0a7c3a","text_start":12250},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":95,"page_start":57,"pdf_page":9,"printed_page":8,"text":"Payments 63 E10700 Income tax withheld","text_end":12342,"text_sha256":"2ec1675834a00e512a58c9381864b776b714ed4373cdddae32895d985a0a7c3a","text_start":12304},"name":"E10700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income tax withheld","publisher_position":63,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":106,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":130,"page_start":96,"pdf_page":9,"printed_page":8,"text":"64 E10900 Estimated tax payments","text_end":12323,"text_sha256":"75627724cfa477e23bbd43e0a7debdf4c4b6b25cb76fbc60d6105f235643da60","text_start":12289},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":130,"page_start":96,"pdf_page":9,"printed_page":8,"text":"64 E10900 Estimated tax payments","text_end":12377,"text_sha256":"75627724cfa477e23bbd43e0a7debdf4c4b6b25cb76fbc60d6105f235643da60","text_start":12343},"name":"E10900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Estimated tax payments","publisher_position":64,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":107,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":179,"page_start":131,"pdf_page":9,"printed_page":8,"text":"65 E10960 Refundable American Opportunity Credit","text_end":12372,"text_sha256":"ab0ab6d30d1af12adea7720e88748dc063811593b58152ac9789909ff1bd8eed","text_start":12324},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":179,"page_start":131,"pdf_page":9,"printed_page":8,"text":"65 E10960 Refundable American Opportunity Credit","text_end":12426,"text_sha256":"ab0ab6d30d1af12adea7720e88748dc063811593b58152ac9789909ff1bd8eed","text_start":12378},"name":"E10960","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Refundable American Opportunity Credit","publisher_position":65,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":108,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":234,"page_start":180,"pdf_page":9,"printed_page":8,"text":"66 E59560 Earned income for earned income credit (EIC)","text_end":12427,"text_sha256":"fe390c9b179c68068bf3cc5e920b5a89129d372f2c7fb81634d68d0f73a85ddd","text_start":12373},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":234,"page_start":180,"pdf_page":9,"printed_page":8,"text":"66 E59560 Earned income for earned income credit (EIC)","text_end":12481,"text_sha256":"fe390c9b179c68068bf3cc5e920b5a89129d372f2c7fb81634d68d0f73a85ddd","text_start":12427},"name":"E59560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Earned income for earned income credit (EIC)","publisher_position":66,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":109,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":289,"page_start":235,"pdf_page":9,"printed_page":8,"text":"67 E59680 EIC used to offset income tax before credits","text_end":12482,"text_sha256":"2f948c4d5be959a7aab4abc2107d24427f1bc1df7f202cb8781f99d684e1b262","text_start":12428},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":289,"page_start":235,"pdf_page":9,"printed_page":8,"text":"67 E59680 EIC used to offset income tax before credits","text_end":12536,"text_sha256":"2f948c4d5be959a7aab4abc2107d24427f1bc1df7f202cb8781f99d684e1b262","text_start":12482},"name":"E59680","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC used to offset income tax before credits","publisher_position":67,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":110,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":353,"page_start":290,"pdf_page":9,"printed_page":8,"text":"68 E59700 EIC used to offset all other taxes except advance EIC","text_end":12546,"text_sha256":"31b02c1b8ebc8097080e693556f9ccebaabf07ec1a8acf72213baa96372a9989","text_start":12483},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":353,"page_start":290,"pdf_page":9,"printed_page":8,"text":"68 E59700 EIC used to offset all other taxes except advance EIC","text_end":12600,"text_sha256":"31b02c1b8ebc8097080e693556f9ccebaabf07ec1a8acf72213baa96372a9989","text_start":12537},"name":"E59700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC used to offset all other taxes except advance EIC","publisher_position":68,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":111,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":386,"page_start":354,"pdf_page":9,"printed_page":8,"text":"69 E59720 EIC refundable portion","text_end":12579,"text_sha256":"061911af9638aa1707e023d2655cda4c06437b76a4a2b8ae7900156c8fbb8747","text_start":12547},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":386,"page_start":354,"pdf_page":9,"printed_page":8,"text":"69 E59720 EIC refundable portion","text_end":12633,"text_sha256":"061911af9638aa1707e023d2655cda4c06437b76a4a2b8ae7900156c8fbb8747","text_start":12601},"name":"E59720","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC refundable portion","publisher_position":69,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":112,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":437,"page_start":387,"pdf_page":9,"printed_page":8,"text":"70 E11550 Refundable prior year minimum tax credit","text_end":12630,"text_sha256":"92a84d3f42a9d9ec071ea4571d7caaff0fb80d1132fad1fa7a2380f85425f553","text_start":12580},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":437,"page_start":387,"pdf_page":9,"printed_page":8,"text":"70 E11550 Refundable prior year minimum tax credit","text_end":12684,"text_sha256":"92a84d3f42a9d9ec071ea4571d7caaff0fb80d1132fad1fa7a2380f85425f553","text_start":12634},"name":"E11550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Refundable prior year minimum tax credit","publisher_position":70,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":113,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":470,"page_start":438,"pdf_page":9,"printed_page":8,"text":"71 E11560 Net Premium Tax Credit","text_end":12663,"text_sha256":"8f25388839d6cc3c3cd2c1d95d23eda2b98c4baaa0843245f346eb045aee4f0a","text_start":12631},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":470,"page_start":438,"pdf_page":9,"printed_page":8,"text":"71 E11560 Net Premium Tax Credit","text_end":12717,"text_sha256":"8f25388839d6cc3c3cd2c1d95d23eda2b98c4baaa0843245f346eb045aee4f0a","text_start":12685},"name":"E11560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit","publisher_position":71,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":114,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":544,"page_start":471,"pdf_page":9,"printed_page":8,"text":"72 E11561 Net Premium Tax Credit Used to Offset Income Tax Before Credits","text_end":12737,"text_sha256":"adc352b7a716e01f12838eddd579ba35172879a827f6d7479e2535d8202fa6a1","text_start":12664},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":544,"page_start":471,"pdf_page":9,"printed_page":8,"text":"72 E11561 Net Premium Tax Credit Used to Offset Income Tax Before Credits","text_end":12791,"text_sha256":"adc352b7a716e01f12838eddd579ba35172879a827f6d7479e2535d8202fa6a1","text_start":12718},"name":"E11561","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Used to Offset Income Tax Before Credits","publisher_position":72,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":115,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":608,"page_start":545,"pdf_page":9,"printed_page":8,"text":"73 E11562 Net Premium Tax Credit Used to Offset All Other Taxes","text_end":12801,"text_sha256":"c991a6bbefc24b16715685110d277b081078fa19cb554264f7b516fcfd7183d6","text_start":12738},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":608,"page_start":545,"pdf_page":9,"printed_page":8,"text":"73 E11562 Net Premium Tax Credit Used to Offset All Other Taxes","text_end":12855,"text_sha256":"c991a6bbefc24b16715685110d277b081078fa19cb554264f7b516fcfd7183d6","text_start":12792},"name":"E11562","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Used to Offset All Other Taxes","publisher_position":73,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":116,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":660,"page_start":609,"pdf_page":9,"printed_page":8,"text":"74 E11563 Net Premium Tax Credit Refundable Portion","text_end":12853,"text_sha256":"39936472c1503f60a139eddfe93bcce36d03d664fad3a30b23323daab3917ade","text_start":12802},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":660,"page_start":609,"pdf_page":9,"printed_page":8,"text":"74 E11563 Net Premium Tax Credit Refundable Portion","text_end":12907,"text_sha256":"39936472c1503f60a139eddfe93bcce36d03d664fad3a30b23323daab3917ade","text_start":12856},"name":"E11563","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Refundable Portion","publisher_position":74,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":117,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":700,"page_start":661,"pdf_page":9,"printed_page":8,"text":"75 E11070 Additional Child Tax Credit","text_end":12893,"text_sha256":"040dc313e83f019d1a7fdd86ac6fa9fa1dd4ea6e3bcd92c3ae66de5e09a557af","text_start":12854},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":700,"page_start":661,"pdf_page":9,"printed_page":8,"text":"75 E11070 Additional Child Tax Credit","text_end":12947,"text_sha256":"040dc313e83f019d1a7fdd86ac6fa9fa1dd4ea6e3bcd92c3ae66de5e09a557af","text_start":12908},"name":"E11070","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Additional Child Tax Credit","publisher_position":75,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":118,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":761,"page_start":701,"pdf_page":9,"printed_page":8,"text":"76 E11100 Amount paid with Form 4868 (Request for Extension)","text_end":12954,"text_sha256":"91a1c51438810477329c58b1633cdb8948a4dfe77239a39790e26f4be94d7914","text_start":12894},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":761,"page_start":701,"pdf_page":9,"printed_page":8,"text":"76 E11100 Amount paid with Form 4868 (Request for Extension)","text_end":13008,"text_sha256":"91a1c51438810477329c58b1633cdb8948a4dfe77239a39790e26f4be94d7914","text_start":12948},"name":"E11100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Amount paid with Form 4868 (Request for Extension)","publisher_position":76,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":119,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":788,"page_start":762,"pdf_page":9,"printed_page":8,"text":"77 E11200 Excess FICA/RRTA","text_end":12981,"text_sha256":"0c24759a396f0f273edc9dde02cc4562d41b1a8ac24615d03ba23c7c77feec8f","text_start":12955},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":788,"page_start":762,"pdf_page":9,"printed_page":8,"text":"77 E11200 Excess FICA/RRTA","text_end":13035,"text_sha256":"0c24759a396f0f273edc9dde02cc4562d41b1a8ac24615d03ba23c7c77feec8f","text_start":13009},"name":"E11200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Excess FICA/RRTA","publisher_position":77,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":120,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":847,"page_start":789,"pdf_page":9,"printed_page":8,"text":"78 E11300 Credit for federal tax on special fuels and oils","text_end":13040,"text_sha256":"7304b8c8e2a6223db69f4a0fc11b59de90d57ff9ed998f9ea24ba2d444191e00","text_start":12982},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":847,"page_start":789,"pdf_page":9,"printed_page":8,"text":"78 E11300 Credit for federal tax on special fuels and oils","text_end":13094,"text_sha256":"7304b8c8e2a6223db69f4a0fc11b59de90d57ff9ed998f9ea24ba2d444191e00","text_start":13036},"name":"E11300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit for federal tax on special fuels and oils","publisher_position":78,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":121,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":893,"page_start":848,"pdf_page":9,"printed_page":8,"text":"79 E11400 Regulated investment company credit","text_end":13086,"text_sha256":"929c13813a8b909fefaf29e06a1ea8d87ae6a798b760e6b05d09707656cd30ea","text_start":13041},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":893,"page_start":848,"pdf_page":9,"printed_page":8,"text":"79 E11400 Regulated investment company credit","text_end":13140,"text_sha256":"929c13813a8b909fefaf29e06a1ea8d87ae6a798b760e6b05d09707656cd30ea","text_start":13095},"name":"E11400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Regulated investment company credit","publisher_position":79,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":122,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":969,"page_start":894,"pdf_page":9,"printed_page":8,"text":"80 E11601 Total Refundable Credits Used to Offset Income Tax Before Credits","text_end":13162,"text_sha256":"901e4b62dd84f28c11773d525adce98423ad11db209d0acb64ee54f3f38e0f48","text_start":13087},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":969,"page_start":894,"pdf_page":9,"printed_page":8,"text":"80 E11601 Total Refundable Credits Used to Offset Income Tax Before Credits","text_end":13216,"text_sha256":"901e4b62dd84f28c11773d525adce98423ad11db209d0acb64ee54f3f38e0f48","text_start":13141},"name":"E11601","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits Used to Offset Income Tax Before Credits","publisher_position":80,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":123,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1035,"page_start":970,"pdf_page":9,"printed_page":8,"text":"81 E11602 Total Refundable Credits Used to Offset All Other Taxes","text_end":13228,"text_sha256":"990e11754e4558b12c74042a2dfab2766c5fa2a2f68f0e09ee8c30ff127b3c3b","text_start":13163},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1035,"page_start":970,"pdf_page":9,"printed_page":8,"text":"81 E11602 Total Refundable Credits Used to Offset All Other Taxes","text_end":13282,"text_sha256":"990e11754e4558b12c74042a2dfab2766c5fa2a2f68f0e09ee8c30ff127b3c3b","text_start":13217},"name":"E11602","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits Used to Offset All Other Taxes","publisher_position":81,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":124,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1087,"page_start":1036,"pdf_page":9,"printed_page":8,"text":"82 E11603 Total Refundable Credits refundable parts","text_end":13280,"text_sha256":"4c5413d9f359b29220824f2fd019c847b16b38d48e4f05aa8b2c232c7e8e06d4","text_start":13229},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1087,"page_start":1036,"pdf_page":9,"printed_page":8,"text":"82 E11603 Total Refundable Credits refundable parts","text_end":13334,"text_sha256":"4c5413d9f359b29220824f2fd019c847b16b38d48e4f05aa8b2c232c7e8e06d4","text_start":13283},"name":"E11603","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits refundable parts","publisher_position":82,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":125,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1122,"page_start":1088,"pdf_page":9,"printed_page":8,"text":"83 E10605 Total tax payments (SOI)","text_end":13315,"text_sha256":"bafa4982b2bc104cd05f8e4a8ace72c10c10caa7bd404d5d12093d7fdae3f6f6","text_start":13281},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1122,"page_start":1088,"pdf_page":9,"printed_page":8,"text":"83 E10605 Total tax payments (SOI)","text_end":13369,"text_sha256":"bafa4982b2bc104cd05f8e4a8ace72c10c10caa7bd404d5d12093d7fdae3f6f6","text_start":13335},"name":"E10605","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax payments (SOI)","publisher_position":83,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":126,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1176,"page_start":1123,"pdf_page":9,"printed_page":8,"text":"Refund etc. 84 E11900 Balance due (overpayment) (+/-)","text_end":13369,"text_sha256":"e1fa87d21b6f5be5368d265edc33a4e2782145385b7b5d37d7f7846e6ab5da48","text_start":13316},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1176,"page_start":1123,"pdf_page":9,"printed_page":8,"text":"Refund etc. 84 E11900 Balance due (overpayment) (+/-)","text_end":13423,"text_sha256":"e1fa87d21b6f5be5368d265edc33a4e2782145385b7b5d37d7f7846e6ab5da48","text_start":13370},"name":"E11900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Balance due (overpayment) (+/-)","publisher_position":84,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":127,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1199,"page_start":1177,"pdf_page":9,"printed_page":8,"text":"85 E12000 Credit elect","text_end":13392,"text_sha256":"5d78d040c8635ffc9f4169045377df4dd3f633b65c9bc2faafaf74b60ba327b2","text_start":13370},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1199,"page_start":1177,"pdf_page":9,"printed_page":8,"text":"85 E12000 Credit elect","text_end":13446,"text_sha256":"5d78d040c8635ffc9f4169045377df4dd3f633b65c9bc2faafaf74b60ba327b2","text_start":13424},"name":"E12000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit elect","publisher_position":85,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":128,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1245,"page_start":1200,"pdf_page":9,"printed_page":8,"text":"86 E12200 Predetermined estimated tax penalty","text_end":13438,"text_sha256":"d9ae52686557efdf00899962c7ced85b0a7c7b1456476672166687a6160f0b06","text_start":13393},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1245,"page_start":1200,"pdf_page":9,"printed_page":8,"text":"86 E12200 Predetermined estimated tax penalty","text_end":13492,"text_sha256":"d9ae52686557efdf00899962c7ced85b0a7c7b1456476672166687a6160f0b06","text_start":13447},"name":"E12200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Predetermined estimated tax penalty","publisher_position":86,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":129,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1328,"page_start":1246,"pdf_page":9,"printed_page":8,"text":"Schedule A 87 E17500 Medical and dental expenses subject to reduction by AGI limit","text_end":13521,"text_sha256":"e32903c1132bc1d3e5b64f301ff0bf4d3286af26c821cf59d501505bd45cbd32","text_start":13439},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1328,"page_start":1246,"pdf_page":9,"printed_page":8,"text":"Schedule A 87 E17500 Medical and dental expenses subject to reduction by AGI limit","text_end":13575,"text_sha256":"e32903c1132bc1d3e5b64f301ff0bf4d3286af26c821cf59d501505bd45cbd32","text_start":13493},"name":"E17500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Medical and dental expenses subject to reduction by AGI limit","publisher_position":87,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":130,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1360,"page_start":1329,"pdf_page":9,"printed_page":8,"text":"88 E18400 State and local taxes","text_end":13553,"text_sha256":"ddfa187258de45c2b29771f40d3c5ca194d94230e598b9b177faea049aa6b4aa","text_start":13522},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1360,"page_start":1329,"pdf_page":9,"printed_page":8,"text":"88 E18400 State and local taxes","text_end":13607,"text_sha256":"ddfa187258de45c2b29771f40d3c5ca194d94230e598b9b177faea049aa6b4aa","text_start":13576},"name":"E18400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"State and local taxes","publisher_position":88,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":131,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1397,"page_start":1361,"pdf_page":9,"printed_page":8,"text":"89 E18500 Real estate tax deductions","text_end":13590,"text_sha256":"8f00348204122a4830340660f171bb3c3821fdec2c22cc97d136d8ab05f41eb3","text_start":13554},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1397,"page_start":1361,"pdf_page":9,"printed_page":8,"text":"89 E18500 Real estate tax deductions","text_end":13644,"text_sha256":"8f00348204122a4830340660f171bb3c3821fdec2c22cc97d136d8ab05f41eb3","text_start":13608},"name":"E18500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Real estate tax deductions","publisher_position":89,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":132,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1437,"page_start":1398,"pdf_page":9,"printed_page":8,"text":"90 E19200 Total interest paid deduction","text_end":13630,"text_sha256":"439821e914f73fc45684f88a5cf2b8a33b108e9f661823070941548f3936f8bc","text_start":13591},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1437,"page_start":1398,"pdf_page":9,"printed_page":8,"text":"90 E19200 Total interest paid deduction","text_end":13684,"text_sha256":"439821e914f73fc45684f88a5cf2b8a33b108e9f661823070941548f3936f8bc","text_start":13645},"name":"E19200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total interest paid deduction","publisher_position":90,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":133,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1485,"page_start":1438,"pdf_page":9,"printed_page":8,"text":"91 E19550 Qualified Mortgage Insurance Premiums","text_end":13678,"text_sha256":"b40c4937f620b500331b1761429127aab74734aa246b5d5984372ca90a9aa383","text_start":13631},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1485,"page_start":1438,"pdf_page":9,"printed_page":8,"text":"91 E19550 Qualified Mortgage Insurance Premiums","text_end":13732,"text_sha256":"b40c4937f620b500331b1761429127aab74734aa246b5d5984372ca90a9aa383","text_start":13685},"name":"E19550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Qualified Mortgage Insurance Premiums","publisher_position":91,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":134,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1514,"page_start":1486,"pdf_page":9,"printed_page":8,"text":"92 E19800 Cash contributions","text_end":13707,"text_sha256":"1f645a012922ff0adaff19fb9bcc239e615bd77859877cfc6095f33bef086592","text_start":13679},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1514,"page_start":1486,"pdf_page":9,"printed_page":8,"text":"92 E19800 Cash contributions","text_end":13761,"text_sha256":"1f645a012922ff0adaff19fb9bcc239e615bd77859877cfc6095f33bef086592","text_start":13733},"name":"E19800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Cash contributions","publisher_position":92,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":135,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1554,"page_start":1515,"pdf_page":9,"printed_page":8,"text":"93 E20100 Other than cash contributions","text_end":13747,"text_sha256":"7b2ae0eda7289cdc24da53278d55944fd14e28383ae760756379c879c1902fca","text_start":13708},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1554,"page_start":1515,"pdf_page":9,"printed_page":8,"text":"93 E20100 Other than cash contributions","text_end":13801,"text_sha256":"7b2ae0eda7289cdc24da53278d55944fd14e28383ae760756379c879c1902fca","text_start":13762},"name":"E20100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other than cash contributions","publisher_position":93,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":136,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1595,"page_start":1555,"pdf_page":9,"printed_page":8,"text":"94 E19700 Contributions deduction, total","text_end":13788,"text_sha256":"751df3b920c087fb69355fae1da856a1078970fd840324bc79f855dca29327a2","text_start":13748},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1595,"page_start":1555,"pdf_page":9,"printed_page":8,"text":"94 E19700 Contributions deduction, total","text_end":13842,"text_sha256":"751df3b920c087fb69355fae1da856a1078970fd840324bc79f855dca29327a2","text_start":13802},"name":"E19700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Contributions deduction, total","publisher_position":94,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":137,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1644,"page_start":1596,"pdf_page":9,"printed_page":8,"text":"95 E20550 Unreimbursed employee business expense","text_end":13837,"text_sha256":"3280ed6476b27be4243fbc9e564c1e5e9912ead010448d59581d39d72e6ded2b","text_start":13789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1644,"page_start":1596,"pdf_page":9,"printed_page":8,"text":"95 E20550 Unreimbursed employee business expense","text_end":13891,"text_sha256":"3280ed6476b27be4243fbc9e564c1e5e9912ead010448d59581d39d72e6ded2b","text_start":13843},"name":"E20550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unreimbursed employee business expense","publisher_position":95,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":138,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1674,"page_start":1645,"pdf_page":9,"printed_page":8,"text":"96 E20600 Tax preparation fee","text_end":13867,"text_sha256":"9a90312e099c6324a38e35937d5d21a9a9d8e2a8c72fd9529ea7cf5dddcb61d0","text_start":13838},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1674,"page_start":1645,"pdf_page":9,"printed_page":8,"text":"96 E20600 Tax preparation fee","text_end":13921,"text_sha256":"9a90312e099c6324a38e35937d5d21a9a9d8e2a8c72fd9529ea7cf5dddcb61d0","text_start":13892},"name":"E20600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax preparation fee","publisher_position":96,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":139,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1742,"page_start":1675,"pdf_page":9,"printed_page":8,"text":"97 E20400 Miscellaneous deductions subject to AGI limitation, total","text_end":13935,"text_sha256":"db6224d05b7e4e7320dc041033f9b221b17b11bf1ddf91d6927a2c7a425982c1","text_start":13868},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1742,"page_start":1675,"pdf_page":9,"printed_page":8,"text":"97 E20400 Miscellaneous deductions subject to AGI limitation, total","text_end":13989,"text_sha256":"db6224d05b7e4e7320dc041033f9b221b17b11bf1ddf91d6927a2c7a425982c1","text_start":13922},"name":"E20400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Miscellaneous deductions subject to AGI limitation, total","publisher_position":97,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":140,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1789,"page_start":1743,"pdf_page":9,"printed_page":8,"text":"98 E20800 Net limited miscellaneous deductions","text_end":13982,"text_sha256":"b5735f97598efaaa6630b2f6d6f570a491bcdf4ab9769974b9a123467f7b67f0","text_start":13936},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1789,"page_start":1743,"pdf_page":9,"printed_page":8,"text":"98 E20800 Net limited miscellaneous deductions","text_end":14036,"text_sha256":"b5735f97598efaaa6630b2f6d6f570a491bcdf4ab9769974b9a123467f7b67f0","text_start":13990},"name":"E20800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net limited miscellaneous deductions","publisher_position":98,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":141,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1826,"page_start":1790,"pdf_page":9,"printed_page":8,"text":"99 E20500 Net casualty or theft loss","text_end":14019,"text_sha256":"fb66a5a3fcc72d2e64d8b5e5e226a4a12e1d1da8c5d00026a7273d3d181893d8","text_start":13983},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1826,"page_start":1790,"pdf_page":9,"printed_page":8,"text":"99 E20500 Net casualty or theft loss","text_end":14073,"text_sha256":"fb66a5a3fcc72d2e64d8b5e5e226a4a12e1d1da8c5d00026a7273d3d181893d8","text_start":14037},"name":"E20500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net casualty or theft loss","publisher_position":99,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":142,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1867,"page_start":1827,"pdf_page":9,"printed_page":8,"text":"100 E21040 Itemized deduction limitation","text_end":14060,"text_sha256":"dae3528bdda2a0b790242911e4b9cc1d18936bfb2fc3bd29ccfca74c4274a052","text_start":14020},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1867,"page_start":1827,"pdf_page":9,"printed_page":8,"text":"100 E21040 Itemized deduction limitation","text_end":14114,"text_sha256":"dae3528bdda2a0b790242911e4b9cc1d18936bfb2fc3bd29ccfca74c4274a052","text_start":14074},"name":"E21040","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Itemized deduction limitation","publisher_position":100,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":143,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14185,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14118},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14239,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14172},"name":"P22250","publisher_group":"amount_or_large_code","publisher_label":"Short-term gains less losses net of carryover","publisher_position":101,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":144,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":195,"page_start":125,"pdf_page":10,"printed_page":9,"text":"102 E22320 Long term gain/loss from other forms on Schedule D","text_end":14256,"text_sha256":"089560a8b3a740a645cdfec1838d2eb8006906fbd8932a94e1d91e8ad3381ba8","text_start":14186},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":195,"page_start":125,"pdf_page":10,"printed_page":9,"text":"102 E22320 Long term gain/loss from other forms on Schedule D","text_end":14310,"text_sha256":"089560a8b3a740a645cdfec1838d2eb8006906fbd8932a94e1d91e8ad3381ba8","text_start":14240},"name":"E22320","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Long term gain/loss from other forms on Schedule D","publisher_position":102,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":145,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":245,"page_start":196,"pdf_page":10,"printed_page":9,"text":"103 E22370 Schedule D capital gain distributions ","text_end":14306,"text_sha256":"9e3c59a530d0730058c9fba5bb7e99e090e8c389c585fa49f360d2107fd1f7a9","text_start":14257},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":245,"page_start":196,"pdf_page":10,"printed_page":9,"text":"103 E22370 Schedule D capital gain distributions ","text_end":14360,"text_sha256":"9e3c59a530d0730058c9fba5bb7e99e090e8c389c585fa49f360d2107fd1f7a9","text_start":14311},"name":"E22370","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D capital gain distributions","publisher_position":103,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":146,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14362,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14307},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14416,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14361},"name":"P23250","publisher_group":"amount_or_large_code","publisher_label":"Long-term gains less losses net of carryover","publisher_position":104,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":147,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":343,"page_start":302,"pdf_page":10,"printed_page":9,"text":"105 E24515 Unrecaptured Section 1250 gain","text_end":14404,"text_sha256":"b3530cde2dfbb1b4c6fb5a0b51e5313493dd594709a0cc3970a76bab97453461","text_start":14363},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":343,"page_start":302,"pdf_page":10,"printed_page":9,"text":"105 E24515 Unrecaptured Section 1250 gain","text_end":14458,"text_sha256":"b3530cde2dfbb1b4c6fb5a0b51e5313493dd594709a0cc3970a76bab97453461","text_start":14417},"name":"E24515","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unrecaptured Section 1250 gain","publisher_position":105,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":148,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":391,"page_start":344,"pdf_page":10,"printed_page":9,"text":"106 E24516 Capital gain less investment expense","text_end":14452,"text_sha256":"9a4e3aff9c97e63b0d70dbe3b562faaef715367aee826d47dc278ece6dc903a3","text_start":14405},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":391,"page_start":344,"pdf_page":10,"printed_page":9,"text":"106 E24516 Capital gain less investment expense","text_end":14506,"text_sha256":"9a4e3aff9c97e63b0d70dbe3b562faaef715367aee826d47dc278ece6dc903a3","text_start":14459},"name":"E24516","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Capital gain less investment expense","publisher_position":106,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":149,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":424,"page_start":392,"pdf_page":10,"printed_page":9,"text":"107 E24518 28% Rate Gain or Loss","text_end":14485,"text_sha256":"6f1db8c750d7ce65f18c9f43563781a4116fa6ee8871e18c9310618d6324d75b","text_start":14453},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":424,"page_start":392,"pdf_page":10,"printed_page":9,"text":"107 E24518 28% Rate Gain or Loss","text_end":14539,"text_sha256":"6f1db8c750d7ce65f18c9f43563781a4116fa6ee8871e18c9310618d6324d75b","text_start":14507},"name":"E24518","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"28% Rate Gain or Loss","publisher_position":107,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":150,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":454,"page_start":425,"pdf_page":10,"printed_page":9,"text":"108 E24560 Non-Schedule D tax","text_end":14515,"text_sha256":"fa4bc43dd1207b390e04fa7d4e76787dc25fb318077540f034fcc3ce0d286465","text_start":14486},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":454,"page_start":425,"pdf_page":10,"printed_page":9,"text":"108 E24560 Non-Schedule D tax","text_end":14569,"text_sha256":"fa4bc43dd1207b390e04fa7d4e76787dc25fb318077540f034fcc3ce0d286465","text_start":14540},"name":"E24560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Non-Schedule D tax","publisher_position":108,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":151,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":498,"page_start":455,"pdf_page":10,"printed_page":9,"text":"109 E24598 Schedule D 15 percent tax amount","text_end":14559,"text_sha256":"f74d6810780d586c9b08e9aa21b3e88c868528245a7761950f06ef70f4e4b7fd","text_start":14516},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":498,"page_start":455,"pdf_page":10,"printed_page":9,"text":"109 E24598 Schedule D 15 percent tax amount","text_end":14613,"text_sha256":"f74d6810780d586c9b08e9aa21b3e88c868528245a7761950f06ef70f4e4b7fd","text_start":14570},"name":"E24598","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 15 percent tax amount","publisher_position":109,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":152,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":542,"page_start":499,"pdf_page":10,"printed_page":9,"text":"110 E24615 Schedule D 25 percent tax amount","text_end":14603,"text_sha256":"2c70781b25a472c4b0f60ddd2020b6ccb429c0f6d3f9164e31315f973c902380","text_start":14560},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":542,"page_start":499,"pdf_page":10,"printed_page":9,"text":"110 E24615 Schedule D 25 percent tax amount","text_end":14657,"text_sha256":"2c70781b25a472c4b0f60ddd2020b6ccb429c0f6d3f9164e31315f973c902380","text_start":14614},"name":"E24615","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 25 percent tax amount","publisher_position":110,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":153,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":586,"page_start":543,"pdf_page":10,"printed_page":9,"text":"111 E24570 Schedule D 28 percent tax amount","text_end":14647,"text_sha256":"92ebc9580c0fb81e2158e5b24e0893662c90255b3170bae820bc60ad7254fd0e","text_start":14604},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":586,"page_start":543,"pdf_page":10,"printed_page":9,"text":"111 E24570 Schedule D 28 percent tax amount","text_end":14701,"text_sha256":"92ebc9580c0fb81e2158e5b24e0893662c90255b3170bae820bc60ad7254fd0e","text_start":14658},"name":"E24570","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 28 percent tax amount","publisher_position":111,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":154,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":643,"page_start":587,"pdf_page":10,"printed_page":9,"text":"Schedule E 112 P25350 Total rents and royalties received","text_end":14704,"text_sha256":"b640f21795387c7dcee9b8b5eacf67e4afa2f4ea930fb4d85da852d634834ea5","text_start":14648},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":643,"page_start":587,"pdf_page":10,"printed_page":9,"text":"Schedule E 112 P25350 Total rents and royalties received","text_end":14758,"text_sha256":"b640f21795387c7dcee9b8b5eacf67e4afa2f4ea930fb4d85da852d634834ea5","text_start":14702},"name":"P25350","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total rents and royalties received","publisher_position":112,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":155,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":686,"page_start":644,"pdf_page":10,"printed_page":9,"text":"113 P25380 Rent/royalty interest expenses ","text_end":14747,"text_sha256":"682d74494bd432d1d5f49e4db91b96eed7a631d7c57717b5173c9069a7922f3a","text_start":14705},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":686,"page_start":644,"pdf_page":10,"printed_page":9,"text":"113 P25380 Rent/royalty interest expenses ","text_end":14801,"text_sha256":"682d74494bd432d1d5f49e4db91b96eed7a631d7c57717b5173c9069a7922f3a","text_start":14759},"name":"P25380","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty interest expenses","publisher_position":113,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":156,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":746,"page_start":687,"pdf_page":10,"printed_page":9,"text":"114 E25550 Total Depreciation and Depletion of all Property","text_end":14807,"text_sha256":"0ed56e992c2784b17db201b27c6061ca7d997a5bb32696270268b882c7b05e83","text_start":14748},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":746,"page_start":687,"pdf_page":10,"printed_page":9,"text":"114 E25550 Total Depreciation and Depletion of all Property","text_end":14861,"text_sha256":"0ed56e992c2784b17db201b27c6061ca7d997a5bb32696270268b882c7b05e83","text_start":14802},"name":"E25550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Depreciation and Depletion of all Property","publisher_position":114,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":157,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":797,"page_start":747,"pdf_page":10,"printed_page":9,"text":"115 P25700 Rent/royalty net income or loss (+/-) ","text_end":14858,"text_sha256":"bd63446a758ed0811529a953dd89e11eac58807a32a2837aa8c4022735a8b0b3","text_start":14808},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":797,"page_start":747,"pdf_page":10,"printed_page":9,"text":"115 P25700 Rent/royalty net income or loss (+/-) ","text_end":14912,"text_sha256":"bd63446a758ed0811529a953dd89e11eac58807a32a2837aa8c4022735a8b0b3","text_start":14862},"name":"P25700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net income or loss (+/-)","publisher_position":115,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":158,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":831,"page_start":798,"pdf_page":10,"printed_page":9,"text":"116 E25820 Deductible rental loss","text_end":14892,"text_sha256":"9cb63df4b98225bfd3cb3e46ece699649bc9702fabb83f5777386461ce79d11b","text_start":14859},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":831,"page_start":798,"pdf_page":10,"printed_page":9,"text":"116 E25820 Deductible rental loss","text_end":14946,"text_sha256":"9cb63df4b98225bfd3cb3e46ece699649bc9702fabb83f5777386461ce79d11b","text_start":14913},"name":"E25820","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Deductible rental loss","publisher_position":116,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":159,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":866,"page_start":832,"pdf_page":10,"printed_page":9,"text":"117 E25850 Rent/royalty net income","text_end":14927,"text_sha256":"fa94a810b3593755108189021145c382351b0447afa5cfb9594bde8deda4a2e5","text_start":14893},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":866,"page_start":832,"pdf_page":10,"printed_page":9,"text":"117 E25850 Rent/royalty net income","text_end":14981,"text_sha256":"fa94a810b3593755108189021145c382351b0447afa5cfb9594bde8deda4a2e5","text_start":14947},"name":"E25850","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net income","publisher_position":117,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":160,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":899,"page_start":867,"pdf_page":10,"printed_page":9,"text":"118 E25860 Rent/royalty net loss","text_end":14960,"text_sha256":"6bde54987abb7b45eda76d5e9647fcf750298d941f9919bf63a0604a34b19d48","text_start":14928},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":899,"page_start":867,"pdf_page":10,"printed_page":9,"text":"118 E25860 Rent/royalty net loss","text_end":15014,"text_sha256":"6bde54987abb7b45eda76d5e9647fcf750298d941f9919bf63a0604a34b19d48","text_start":14982},"name":"E25860","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net loss","publisher_position":118,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":161,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":944,"page_start":900,"pdf_page":10,"printed_page":9,"text":"Partnerships 119 E25940 Total passive income","text_end":15005,"text_sha256":"3db4b0dd75ddecf1bffa7e10d35822db4e1a6f86d1a1758db9ddacfb3a4b71ed","text_start":14961},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":944,"page_start":900,"pdf_page":10,"printed_page":9,"text":"Partnerships 119 E25940 Total passive income","text_end":15059,"text_sha256":"3db4b0dd75ddecf1bffa7e10d35822db4e1a6f86d1a1758db9ddacfb3a4b71ed","text_start":15015},"name":"E25940","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive income","publisher_position":119,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":162,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":980,"page_start":945,"pdf_page":10,"printed_page":9,"text":"120 E25980 Total non-passive income","text_end":15041,"text_sha256":"75021dd4dd31369b59b66d904b46f5b62ac6b0693fe6254531a197859051f047","text_start":15006},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":980,"page_start":945,"pdf_page":10,"printed_page":9,"text":"120 E25980 Total non-passive income","text_end":15095,"text_sha256":"75021dd4dd31369b59b66d904b46f5b62ac6b0693fe6254531a197859051f047","text_start":15060},"name":"E25980","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive income","publisher_position":120,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":163,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1010,"page_start":981,"pdf_page":10,"printed_page":9,"text":"121 E25920 Total passive loss","text_end":15071,"text_sha256":"0870768b18a3633a2023c645a78aa15aa9dd80b8032b7b31505c004329ee5e33","text_start":15042},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1010,"page_start":981,"pdf_page":10,"printed_page":9,"text":"121 E25920 Total passive loss","text_end":15125,"text_sha256":"0870768b18a3633a2023c645a78aa15aa9dd80b8032b7b31505c004329ee5e33","text_start":15096},"name":"E25920","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive loss","publisher_position":121,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":164,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1045,"page_start":1011,"pdf_page":10,"printed_page":9,"text":"122 E25960 Total non-passive loss ","text_end":15106,"text_sha256":"95bd8d56b53fd5087246f2535647572b80f35c81a3d1cfceeb37f02ef313d90f","text_start":15072},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1045,"page_start":1011,"pdf_page":10,"printed_page":9,"text":"122 E25960 Total non-passive loss ","text_end":15160,"text_sha256":"95bd8d56b53fd5087246f2535647572b80f35c81a3d1cfceeb37f02ef313d90f","text_start":15126},"name":"E25960","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive loss","publisher_position":122,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":165,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1099,"page_start":1046,"pdf_page":10,"printed_page":9,"text":"123 E26110 Partnership Section 179 expense deduction ","text_end":15160,"text_sha256":"8aab34c08db2c37001058efb2db7ed3bb4c768551e0f9ab9a90c8b6fa5aee250","text_start":15107},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1099,"page_start":1046,"pdf_page":10,"printed_page":9,"text":"123 E26110 Partnership Section 179 expense deduction ","text_end":15214,"text_sha256":"8aab34c08db2c37001058efb2db7ed3bb4c768551e0f9ab9a90c8b6fa5aee250","text_start":15161},"name":"E26110","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Partnership Section 179 expense deduction","publisher_position":123,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":166,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1137,"page_start":1100,"pdf_page":10,"printed_page":9,"text":"Small 124 E26170 Total passive income","text_end":15198,"text_sha256":"851d2824b95073f372ea07f402e7fb4f2f8bda41f8384e25a3282f88dce713e6","text_start":15161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1137,"page_start":1100,"pdf_page":10,"printed_page":9,"text":"Small 124 E26170 Total passive income","text_end":15252,"text_sha256":"851d2824b95073f372ea07f402e7fb4f2f8bda41f8384e25a3282f88dce713e6","text_start":15215},"name":"E26170","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive income","publisher_position":124,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":167,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1182,"page_start":1138,"pdf_page":10,"printed_page":9,"text":"Business 125 E26190 Total non-passive income","text_end":15243,"text_sha256":"747af42f61dced71b614799a3f5682ba4506b1d09b36957568d755e91b3aed49","text_start":15199},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1182,"page_start":1138,"pdf_page":10,"printed_page":9,"text":"Business 125 E26190 Total non-passive income","text_end":15297,"text_sha256":"747af42f61dced71b614799a3f5682ba4506b1d09b36957568d755e91b3aed49","text_start":15253},"name":"E26190","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive income","publisher_position":125,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":168,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1217,"page_start":1183,"pdf_page":10,"printed_page":9,"text":"Corp 126 E26160 Total passive loss","text_end":15278,"text_sha256":"1a777ab5ce50cf58d0422059c263de66fe7e1f724a26f9bc8ee14ec253b38066","text_start":15244},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1217,"page_start":1183,"pdf_page":10,"printed_page":9,"text":"Corp 126 E26160 Total passive loss","text_end":15332,"text_sha256":"1a777ab5ce50cf58d0422059c263de66fe7e1f724a26f9bc8ee14ec253b38066","text_start":15298},"name":"E26160","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive loss","publisher_position":126,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":169,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1252,"page_start":1218,"pdf_page":10,"printed_page":9,"text":"127 E26180 Total non-passive loss ","text_end":15313,"text_sha256":"048170ad209b195d4da910c9bc6eff93d10996098925e7d9c457ce4b07b04f36","text_start":15279},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1252,"page_start":1218,"pdf_page":10,"printed_page":9,"text":"127 E26180 Total non-passive loss ","text_end":15367,"text_sha256":"048170ad209b195d4da910c9bc6eff93d10996098925e7d9c457ce4b07b04f36","text_start":15333},"name":"E26180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive loss","publisher_position":127,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":170,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1324,"page_start":1253,"pdf_page":10,"printed_page":9,"text":"128 E26270 Combined partnership and S corporation net income/loss (+/-)","text_end":15385,"text_sha256":"31a1191bd5b7096b5d5e1f25b7c1bf70c014437c5ced1571411caf4321790779","text_start":15314},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1324,"page_start":1253,"pdf_page":10,"printed_page":9,"text":"128 E26270 Combined partnership and S corporation net income/loss (+/-)","text_end":15439,"text_sha256":"31a1191bd5b7096b5d5e1f25b7c1bf70c014437c5ced1571411caf4321790779","text_start":15368},"name":"E26270","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Combined partnership and S corporation net income/loss (+/-)","publisher_position":128,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":171,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1373,"page_start":1325,"pdf_page":10,"printed_page":9,"text":"129 E26100 S-corp. Section 179 expense deduction","text_end":15434,"text_sha256":"64da1fc754e9d2ddc722d9593e9f7457b8db6d95893ac8d2178120731f51903b","text_start":15386},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1373,"page_start":1325,"pdf_page":10,"printed_page":9,"text":"129 E26100 S-corp. Section 179 expense deduction","text_end":15488,"text_sha256":"64da1fc754e9d2ddc722d9593e9f7457b8db6d95893ac8d2178120731f51903b","text_start":15440},"name":"E26100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"S-corp. Section 179 expense deduction","publisher_position":129,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":172,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1413,"page_start":1374,"pdf_page":10,"printed_page":9,"text":"Estate or Trust 130 E26390 Total income","text_end":15474,"text_sha256":"2f0898262ee6425d4ee7d4033a22819627afbeec6ecd4a00fa73b3c0761161f6","text_start":15435},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1413,"page_start":1374,"pdf_page":10,"printed_page":9,"text":"Estate or Trust 130 E26390 Total income","text_end":15528,"text_sha256":"2f0898262ee6425d4ee7d4033a22819627afbeec6ecd4a00fa73b3c0761161f6","text_start":15489},"name":"E26390","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total income","publisher_position":130,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":173,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1435,"page_start":1414,"pdf_page":10,"printed_page":9,"text":"131 E26400 Total loss","text_end":15496,"text_sha256":"0dd69be8e4d8a255afd523dabd6c4dd2c9a4ce1145510ef6e5eb16f598679d8f","text_start":15475},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1435,"page_start":1414,"pdf_page":10,"printed_page":9,"text":"131 E26400 Total loss","text_end":15550,"text_sha256":"0dd69be8e4d8a255afd523dabd6c4dd2c9a4ce1145510ef6e5eb16f598679d8f","text_start":15529},"name":"E26400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total loss","publisher_position":131,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":174,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1493,"page_start":1436,"pdf_page":10,"printed_page":9,"text":"Farm Rental 132 E27200 Farm rent net income or loss (+/-)","text_end":15554,"text_sha256":"4d10ccb6bb7947db33cf75041f47bd5ce5139849f0e1a67fc1adb951537346db","text_start":15497},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1493,"page_start":1436,"pdf_page":10,"printed_page":9,"text":"Farm Rental 132 E27200 Farm rent net income or loss (+/-)","text_end":15608,"text_sha256":"4d10ccb6bb7947db33cf75041f47bd5ce5139849f0e1a67fc1adb951537346db","text_start":15551},"name":"E27200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Farm rent net income or loss (+/-)","publisher_position":132,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":175,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1575,"page_start":1494,"pdf_page":10,"printed_page":9,"text":"Schedule SE 133 E30400 Self-employment income subject to SS tax, primary taxpayer","text_end":15636,"text_sha256":"0bb1fa0b3d54c49628e277295abce0b052e2fae074957df9386626e1158c1b97","text_start":15555},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1575,"page_start":1494,"pdf_page":10,"printed_page":9,"text":"Schedule SE 133 E30400 Self-employment income subject to SS tax, primary taxpayer","text_end":15690,"text_sha256":"0bb1fa0b3d54c49628e277295abce0b052e2fae074957df9386626e1158c1b97","text_start":15609},"name":"E30400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment income subject to SS tax, primary taxpayer","publisher_position":133,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":176,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1647,"page_start":1576,"pdf_page":10,"printed_page":9,"text":"134 E30500 Self-employment income subject to SS tax, secondary taxpayer","text_end":15708,"text_sha256":"f03ef146af52bdb8e74f0e445fd123bb3398b9633eb21e61263dd8b86b9ca9dd","text_start":15637},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1647,"page_start":1576,"pdf_page":10,"printed_page":9,"text":"134 E30500 Self-employment income subject to SS tax, secondary taxpayer","text_end":15762,"text_sha256":"f03ef146af52bdb8e74f0e445fd123bb3398b9633eb21e61263dd8b86b9ca9dd","text_start":15691},"name":"E30500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment income subject to SS tax, secondary taxpayer","publisher_position":134,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":177,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1701,"page_start":1648,"pdf_page":10,"printed_page":9,"text":"Form 2441 135 E32800 Qualifying individuals' Expenses","text_end":15762,"text_sha256":"a96204e1552bf8811d3c3ea05f88afad8fc1e452c183b45ceed6504ec1be5b1e","text_start":15709},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1701,"page_start":1648,"pdf_page":10,"printed_page":9,"text":"Form 2441 135 E32800 Qualifying individuals' Expenses","text_end":15816,"text_sha256":"a96204e1552bf8811d3c3ea05f88afad8fc1e452c183b45ceed6504ec1be5b1e","text_start":15763},"name":"E32800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Qualifying individuals' Expenses","publisher_position":135,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":178,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1746,"page_start":1702,"pdf_page":10,"printed_page":9,"text":"136 E33000 Expenses limited to earned income","text_end":15807,"text_sha256":"c179dcba5a5bfab5d99c5ca6fe984c9469bfacff81b0fa37ee43481843393703","text_start":15763},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1746,"page_start":1702,"pdf_page":10,"printed_page":9,"text":"136 E33000 Expenses limited to earned income","text_end":15861,"text_sha256":"c179dcba5a5bfab5d99c5ca6fe984c9469bfacff81b0fa37ee43481843393703","text_start":15817},"name":"E33000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Expenses limited to earned income","publisher_position":136,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":179,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1829,"page_start":1747,"pdf_page":10,"printed_page":9,"text":"Form 3800 137 E53240 Work Opportunity - Jobs (included in general business credit)","text_end":15890,"text_sha256":"04ffe18138bd89fc8e2ed9b6a255a172545bd7d10f8f1ee51d21fbcab4074f06","text_start":15808},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1829,"page_start":1747,"pdf_page":10,"printed_page":9,"text":"Form 3800 137 E53240 Work Opportunity - Jobs (included in general business credit)","text_end":15944,"text_sha256":"04ffe18138bd89fc8e2ed9b6a255a172545bd7d10f8f1ee51d21fbcab4074f06","text_start":15862},"name":"E53240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Work Opportunity - Jobs (included in general business credit)","publisher_position":137,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":180,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1907,"page_start":1830,"pdf_page":10,"printed_page":9,"text":"138 E53280 Research and experimentation (included in general business credit)","text_end":15968,"text_sha256":"c7f398db6179e25e290cfa86be9a505ed073e18d379a91f7f2021c02ecda2d6e","text_start":15891},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1907,"page_start":1830,"pdf_page":10,"printed_page":9,"text":"138 E53280 Research and experimentation (included in general business credit)","text_end":16022,"text_sha256":"c7f398db6179e25e290cfa86be9a505ed073e18d379a91f7f2021c02ecda2d6e","text_start":15945},"name":"E53280","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Research and experimentation (included in general business credit)","publisher_position":138,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":181,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1944,"page_start":1908,"pdf_page":10,"printed_page":9,"text":"139 E53300 Low Income Housing Credit","text_end":16005,"text_sha256":"5e6ad4b9c3949946111e65220b5a39cc22a498f6addd0d9ea6c95dea26d0c50f","text_start":15969},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1944,"page_start":1908,"pdf_page":10,"printed_page":9,"text":"139 E53300 Low Income Housing Credit","text_end":16059,"text_sha256":"5e6ad4b9c3949946111e65220b5a39cc22a498f6addd0d9ea6c95dea26d0c50f","text_start":16023},"name":"E53300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Low Income Housing Credit","publisher_position":139,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":182,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":2004,"page_start":1945,"pdf_page":10,"printed_page":9,"text":"140 E53317 Employer Credit for Social Security Tax on Tips ","text_end":16065,"text_sha256":"3ff76b72ecc28a54974e1b4b03b09533be8fa6914d3045885eddaa50878c949f","text_start":16006},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":2004,"page_start":1945,"pdf_page":10,"printed_page":9,"text":"140 E53317 Employer Credit for Social Security Tax on Tips ","text_end":16119,"text_sha256":"3ff76b72ecc28a54974e1b4b03b09533be8fa6914d3045885eddaa50878c949f","text_start":16060},"name":"E53317","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Employer Credit for Social Security Tax on Tips","publisher_position":140,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":183,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":118,"page_start":64,"pdf_page":11,"printed_page":10,"text":"Form 4952 141 E58950 Total investment Interest Expense","text_end":16184,"text_sha256":"5e89a62c32c9b5f4d0f8c8c2e8125e4a9e36fec55e118a28236158a7dd203d2c","text_start":16130},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":119,"page_start":64,"pdf_page":11,"printed_page":10,"text":"Form 4952 141 E 58950 Total investment Interest Expense","text_end":16239,"text_sha256":"252b38b9a5704bde9efc197e7a808036d8834aff474a41ad3705e4283f762123","text_start":16184},"name":"E58950","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total investment Interest Expense","publisher_position":141,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":184,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":190,"page_start":119,"pdf_page":11,"printed_page":10,"text":"142 E58990 Investment income Elected Amount (Form 4952 part 2 line 4g) ","text_end":16256,"text_sha256":"64688151556142b16e63690f72f67a35deb45e35a60ee1ab53ccd532b8806c6d","text_start":16185},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":192,"page_start":120,"pdf_page":11,"printed_page":10,"text":"142 E58990 I nvestment income Elected Amount (Form 4952 part 2 line 4g) ","text_end":16312,"text_sha256":"d3db63182f82930facb1ceaf405f367af661295d6633fa5448402dbbc5e20522","text_start":16240},"name":"E58990","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Investment income Elected Amount (Form 4952 part 2 line 4g)","publisher_position":142,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":185,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":268,"page_start":191,"pdf_page":11,"printed_page":10,"text":"Form 6251 143 P60100 Net operating loss tax preference plus other adjustments","text_end":16334,"text_sha256":"072bb169590f7846991ef28bb423645172a6e15ce3586e22cbe40c2f3825db20","text_start":16257},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":271,"page_start":193,"pdf_page":11,"printed_page":10,"text":"Form 6251 143 P 60100 Net operating loss tax preference plus other adjustments","text_end":16391,"text_sha256":"2bf5756fbd6496a7e8be5558aa19f5086f021f4584a1539bf7c41e69c9d8c577","text_start":16313},"name":"P60100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net operating loss tax preference plus other adjustments","publisher_position":143,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":186,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":320,"page_start":269,"pdf_page":11,"printed_page":10,"text":"144 P61850 Total adjustments and preferences (+/-) ","text_end":16386,"text_sha256":"255e12d7e57c6002eb605e9c2edcf9bdce94fb2920530a0b3acba4cd09108aa0","text_start":16335},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":324,"page_start":272,"pdf_page":11,"printed_page":10,"text":"144 P61850 T otal adjustments and preferences (+/-) ","text_end":16444,"text_sha256":"f287aeeaf4b188642d977a6cf4b59f27df24c4364a08068098c801bf0a78a4b3","text_start":16392},"name":"P61850","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total adjustments and preferences (+/-)","publisher_position":144,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":187,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":384,"page_start":321,"pdf_page":11,"printed_page":10,"text":"145 E60000 Form 1040 Taxable Income for AMT (Form 6251, Line 1)","text_end":16450,"text_sha256":"31fa42fd1d1eafc9f6a3d2739c76dd770faaf9f6c608b6a6d1c5622139248e40","text_start":16387},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":389,"page_start":325,"pdf_page":11,"printed_page":10,"text":"145 E60000 F orm 1040 Taxable Income for AMT (Form 6251, Line 1)","text_end":16509,"text_sha256":"d4379bb4e120357f1c29c8a73b4008da40ad5899f5865a5bcf49a6aa84a88c27","text_start":16445},"name":"E60000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Form 1040 Taxable Income for AMT (Form 6251, Line 1)","publisher_position":145,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":188,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":430,"page_start":385,"pdf_page":11,"printed_page":10,"text":"146 E62100 Alternative minimum taxable income","text_end":16496,"text_sha256":"28fa4b9e9eb924190bcfcb0b41a748f7a0c3aac07cafa84db6446302c55c3ac3","text_start":16451},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":436,"page_start":390,"pdf_page":11,"printed_page":10,"text":"146 E62100 A lternative minimum taxable income","text_end":16556,"text_sha256":"11a936b34f7f0873a3352cf0c705915559f906f357a39db2d7389cc9438c264f","text_start":16510},"name":"E62100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum taxable income","publisher_position":146,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":189,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":477,"page_start":431,"pdf_page":11,"printed_page":10,"text":"147 E62900 Alternative tax foreign tax credit ","text_end":16543,"text_sha256":"62649650ef597f51c764fce34dc395c78434e5e3e0120253791be153408a0f52","text_start":16497},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":484,"page_start":437,"pdf_page":11,"printed_page":10,"text":"147 E62900 A lternative tax foreign tax credit ","text_end":16604,"text_sha256":"6975e8ed410762963bcdb26a7a3d41b8f26e0bbb373879003a3fb5938868f778","text_start":16557},"name":"E62900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative tax foreign tax credit","publisher_position":147,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":190,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":551,"page_start":478,"pdf_page":11,"printed_page":10,"text":"148 E62720 Alternative minimum Schedule D less investment interest amount","text_end":16617,"text_sha256":"34af121bbf5475ac2597cde5cde08e116b6398e6dd1df8f4745a492aba43c2f1","text_start":16544},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":559,"page_start":485,"pdf_page":11,"printed_page":10,"text":"148 E62720 A lternative minimum Schedule D less investment interest amount","text_end":16679,"text_sha256":"e0266013087a7cef6f77c04b0e8bd989e94a0ea6a4f956e0c775c3e101dce065","text_start":16605},"name":"E62720","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum Schedule D less investment interest amount","publisher_position":148,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":191,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":624,"page_start":552,"pdf_page":11,"printed_page":10,"text":"149 E62730 Alternative minimum Schedule D unrecaptured section 1250 gain","text_end":16690,"text_sha256":"b127f9881cd80e952eba67db0ee44585c5db04b52462d3da20bd6950e8088590","text_start":16618},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":633,"page_start":560,"pdf_page":11,"printed_page":10,"text":"149 E62730 A lternative minimum Schedule D unrecaptured section 1250 gain","text_end":16753,"text_sha256":"9e994e3bb4620cce4c5b4ad7d645ac89a4573815be8bd18c566e18e13835bb0a","text_start":16680},"name":"E62730","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum Schedule D unrecaptured section 1250 gain","publisher_position":149,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":192,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":675,"page_start":625,"pdf_page":11,"printed_page":10,"text":"150 E62740 Alternative minimum capital gain amount","text_end":16741,"text_sha256":"3f0e4f8610f28fef1b1059da930b02ae1b9e53a3a5c8104e02ae8ce0aa62db5f","text_start":16691},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":685,"page_start":634,"pdf_page":11,"printed_page":10,"text":"150 E62740 A lternative minimum capital gain amount","text_end":16805,"text_sha256":"8983a99dd7ced7fb7c64d16cb1caac795b6ecb3f93749f45deea3acb2dcc59c3","text_start":16754},"name":"E62740","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum capital gain amount","publisher_position":150,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":193,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":722,"page_start":676,"pdf_page":11,"printed_page":10,"text":"Form 8582 151 P65300 Total passive net income ","text_end":16788,"text_sha256":"18c73ccd9957b717d5cad787a63b63ae1450bf1929efe9a2b2a6cebcdb82d8ca","text_start":16742},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":733,"page_start":686,"pdf_page":11,"printed_page":10,"text":"Form 8582 151 P 65300 Total passive net income ","text_end":16853,"text_sha256":"5df06c3a1dc98670573115f414b7a42a2f68765ed730fb9e975bd6ef51ff377b","text_start":16806},"name":"P65300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive net income","publisher_position":151,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":194,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":754,"page_start":723,"pdf_page":11,"printed_page":10,"text":"152 P65400 Total passive losses","text_end":16820,"text_sha256":"55eab1e0a505311110697cecd36aea2ee3712b75b9ae79d346279ecf5c1b1e5c","text_start":16789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":766,"page_start":734,"pdf_page":11,"printed_page":10,"text":"152 P65400 T otal passive losses","text_end":16886,"text_sha256":"6c793fc771664f415b4b0170f15271ace694dcf94fe2f5b77f999ba84aa50530","text_start":16854},"name":"P65400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive losses","publisher_position":152,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":195,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":814,"page_start":755,"pdf_page":11,"printed_page":10,"text":"153 E68000 Total losses allowed from all passive activities","text_end":16880,"text_sha256":"eb2b6e50257d12b9c7f8dce236a0388c2b5d522d09343099dcc652a58c0f9760","text_start":16821},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":827,"page_start":767,"pdf_page":11,"printed_page":10,"text":"153 E68000 T otal losses allowed from all passive activities","text_end":16947,"text_sha256":"0076d8bc39224c5e88840043bee988c96f577dcb1f5d56d682c2ebb7546bb845","text_start":16887},"name":"E68000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total losses allowed from all passive activities","publisher_position":153,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":196,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":871,"page_start":815,"pdf_page":11,"printed_page":10,"text":"Form 8801 154 E82200 Carry forward of minimum tax credit","text_end":16937,"text_sha256":"59aaf65f7d9bda290af1799c886cea58b219ea0fd131e6782c487981a8978460","text_start":16881},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":885,"page_start":828,"pdf_page":11,"printed_page":10,"text":"Form 8801 154 E 82200 Carry forward of minimum tax credit","text_end":17005,"text_sha256":"7114e0e1beb6c8ddb0bb3090feafca9f7e34b1196d374bd258ead395a915c158","text_start":16948},"name":"E82200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Carry forward of minimum tax credit","publisher_position":154,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":197,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":913,"page_start":872,"pdf_page":11,"printed_page":10,"text":"Schedule J 155 T27800 Elected Farm Income","text_end":16979,"text_sha256":"be7feb33b6d1bb698b7471b0eb76d53828ff8fdd71ec3a873c5350a35e04fcb2","text_start":16938},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":928,"page_start":886,"pdf_page":11,"printed_page":10,"text":"Schedule J 155 T27800 E lected Farm Income","text_end":17048,"text_sha256":"c9f5371d024a6b00856e029ebce7efd258e90c37bf3b878e22de4dbd618f2529","text_start":17006},"name":"T27800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Elected Farm Income","publisher_position":155,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":198,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":961,"page_start":914,"pdf_page":11,"printed_page":10,"text":"156 S27860 Tentative Current and Prior-Year Tax","text_end":17027,"text_sha256":"88b7f1492af50b6ab6fb9e5b1515a91eb1f44c9cfcd9c6e496b9540140cfc578","text_start":16980},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":977,"page_start":929,"pdf_page":11,"printed_page":10,"text":"156 S27860 T entative Current and Prior-Year Tax","text_end":17097,"text_sha256":"f13162d3346d786e547e854b719133cad86374235ef58655b75113d136ed2782","text_start":17049},"name":"S27860","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tentative Current and Prior-Year Tax","publisher_position":156,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":199,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":994,"page_start":962,"pdf_page":11,"printed_page":10,"text":"157 P27895 Actual Prior-Year Tax","text_end":17060,"text_sha256":"76e1d2ada7d1619e48cc36189dd7d1b0675a5b9cda2ee63c7a528992b04a6393","text_start":17028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1011,"page_start":978,"pdf_page":11,"printed_page":10,"text":"157 P27895 A ctual Prior-Year Tax","text_end":17131,"text_sha256":"e4171a351ea18aef659363e6a5e321d54382cfbfc5532f04b470eedbc3bac085","text_start":17098},"name":"P27895","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Actual Prior-Year Tax","publisher_position":157,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":200,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1055,"page_start":995,"pdf_page":11,"printed_page":10,"text":"Form 8863 158 P87482 American Opportunity Qualified Expenses","text_end":17121,"text_sha256":"65ce2c35ea2c651a1777a474cfab02698be506018fd268503663d1b328df8e19","text_start":17061},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1073,"page_start":1012,"pdf_page":11,"printed_page":10,"text":"Form 8863 158 P 87482 American Opportunity Qualified Expenses","text_end":17193,"text_sha256":"291f4dc96e2a1af58ea03ffc7d0bee29690136da12c38562a3ae8feccc3ee8b9","text_start":17132},"name":"P87482","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"American Opportunity Qualified Expenses","publisher_position":158,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":201,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1094,"page_start":1056,"pdf_page":11,"printed_page":10,"text":"159 E87521 American Opportunity Credit","text_end":17160,"text_sha256":"66fbe1d0a2afb1dabca85e4e06815cc7cd9e492ab8498550ea5ed8faa77d3858","text_start":17122},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1113,"page_start":1074,"pdf_page":11,"printed_page":10,"text":"159 E87521 A merican Opportunity Credit","text_end":17233,"text_sha256":"c31dea021b1fb8ff758fc124873c7bb68799d02c8c7a1be6860f550ce7e497d0","text_start":17194},"name":"E87521","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"American Opportunity Credit","publisher_position":159,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":202,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1148,"page_start":1095,"pdf_page":11,"printed_page":10,"text":"160 E87530 Lifetime Learning Total Qualified Expenses","text_end":17214,"text_sha256":"d93b29ad481354864195c98bbfa19935d385701e17bb4f92b4dd6eefc74335b4","text_start":17161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1168,"page_start":1114,"pdf_page":11,"printed_page":10,"text":"160 E87530 Li fetime Learning Total Qualified Expenses","text_end":17288,"text_sha256":"d042b095d6c9b805cf0d86e25a046271deb88169d5962376618b0969bcabfe18","text_start":17234},"name":"E87530","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Lifetime Learning Total Qualified Expenses","publisher_position":160,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":203,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1184,"page_start":1149,"pdf_page":11,"printed_page":10,"text":"161 E87550 Lifetime Learning Credit","text_end":17250,"text_sha256":"74be56787cde243473bd5253de4762f94df31a11763b059c0598773b48d57887","text_start":17215},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1205,"page_start":1169,"pdf_page":11,"printed_page":10,"text":"161 E87550 Li fetime Learning Credit","text_end":17325,"text_sha256":"7c06009e6df97fb8f375ef343e2c2b6fa21ca97a4ef5c5b9fb856d43efccdf7a","text_start":17289},"name":"E87550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Lifetime Learning Credit","publisher_position":161,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":204,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1226,"page_start":1185,"pdf_page":11,"printed_page":10,"text":"Form 8888 162 P86421 Bond Purchase Amount","text_end":17292,"text_sha256":"1c2b0f09f1951160e78c576d8ee59ab6a183e80b66d47414c3709ebc973f89c7","text_start":17251},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1248,"page_start":1206,"pdf_page":11,"printed_page":10,"text":"Form 8888 162 P 86421 Bond Purchase Amount","text_end":17368,"text_sha256":"31952dc713df089ee582a7bace50bba79e99272644300f7feb558984e98dfb0a","text_start":17326},"name":"P86421","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Bond Purchase Amount","publisher_position":162,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":205,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1320,"page_start":1227,"pdf_page":11,"printed_page":10,"text":"Form 8960 163 E85050 Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","text_end":17386,"text_sha256":"397c4f3ddfe481b56785ca2125f8e8f1f95ec3355786e99e3b444370cf4c17b7","text_start":17293},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1343,"page_start":1249,"pdf_page":11,"printed_page":10,"text":"Form 8960 163 E 85050 Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","text_end":17463,"text_sha256":"bb78df6edcc7919f42e04f398ffa426c1836b5bb2070168808d9efb00f0c53ef","text_start":17369},"name":"E85050","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","publisher_position":163,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":206,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1383,"page_start":1321,"pdf_page":11,"printed_page":10,"text":"164 E85090 Total Net Gain or Loss from Disposition of Property","text_end":17449,"text_sha256":"705c9e4cb724c99ef271546ea78f83e2ad27363bdf949d876558514e8b011bd8","text_start":17387},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1407,"page_start":1344,"pdf_page":11,"printed_page":10,"text":"164 E85090 T otal Net Gain or Loss from Disposition of Property","text_end":17527,"text_sha256":"9e628010b94bb4b33247d774d7562678f0461a14df56a8054f4dc88d244b146c","text_start":17464},"name":"E85090","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Net Gain or Loss from Disposition of Property","publisher_position":164,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":207,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1418,"page_start":1384,"pdf_page":11,"printed_page":10,"text":"165 E85120 Total Investment Income","text_end":17484,"text_sha256":"b16ed39d6b7e1169732dcd0bbf5d2e0340dc2a2af8018fcb5dcf8b2c42b04457","text_start":17450},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1443,"page_start":1408,"pdf_page":11,"printed_page":10,"text":"165 E85120 T otal Investment Income","text_end":17563,"text_sha256":"08a6dcc6ce09f63f41d681df1a9779a955206f5cefe01f0b0917d922238e59f4","text_start":17528},"name":"E85120","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Investment Income","publisher_position":165,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":208,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1464,"page_start":1419,"pdf_page":11,"printed_page":10,"text":"166 E85180 Total Deductions and Modifications","text_end":17530,"text_sha256":"ef9ab37b928d5a8d1af456371c753cc753ad7dd9a919ab22dc00d9661583b013","text_start":17485},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1490,"page_start":1444,"pdf_page":11,"printed_page":10,"text":"166 E85180 T otal Deductions and Modifications","text_end":17610,"text_sha256":"d0810d5f09a52dcab3abce22b6829dd5984c9234136efefaae7ee89ff788fd2d","text_start":17564},"name":"E85180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Deductions and Modifications","publisher_position":166,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":209,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1534,"page_start":1465,"pdf_page":11,"printed_page":10,"text":"Form 8962 167 E85570 Dependents Modified Adjusted Gross Income Amount","text_end":17600,"text_sha256":"943aca30c20f5fb020ff080c7c43742bceebed742c39de678fd67ff9a0b33736","text_start":17531},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1561,"page_start":1491,"pdf_page":11,"printed_page":10,"text":"Form 8962 167 E 85570 Dependents Modified Adjusted Gross Income Amount","text_end":17681,"text_sha256":"32701ca71a4560d09f067877f6f3b79b04a0a0e27af16fffd95a619b8ee9a669","text_start":17611},"name":"E85570","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Dependents Modified Adjusted Gross Income Amount","publisher_position":167,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":210,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1584,"page_start":1535,"pdf_page":11,"printed_page":10,"text":"168 E85595 Annual Contribution Health Care Amount","text_end":17650,"text_sha256":"6018e18cd7a2e2470fa74e6a434f1e289dd47ce2729efae65a6414d129dc6168","text_start":17601},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1612,"page_start":1562,"pdf_page":11,"printed_page":10,"text":"168 E85595 A nnual Contribution Health Care Amount","text_end":17732,"text_sha256":"152389beefb4a57a1c1ed6d947d489ba80d3d2e34444ae7f57bd2cb66e13140d","text_start":17682},"name":"E85595","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Annual Contribution Health Care Amount","publisher_position":168,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":211,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1635,"page_start":1585,"pdf_page":11,"printed_page":10,"text":"169 E85600 Monthly Contribution Health Care Amount","text_end":17701,"text_sha256":"4dde94616b6352f9830a0a54c108b59b6ee419c3bd594eb6e94d54a43a86ff6e","text_start":17651},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1664,"page_start":1613,"pdf_page":11,"printed_page":10,"text":"169 E85600 M onthly Contribution Health Care Amount","text_end":17784,"text_sha256":"bc44b59cab85460cf9cb8813099019cf662c2e34404112d0a990d1b8ac4d157d","text_start":17733},"name":"E85600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Monthly Contribution Health Care Amount","publisher_position":169,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":212,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1678,"page_start":1636,"pdf_page":11,"printed_page":10,"text":"170 E85770 Total Premium Tax Credit Amount","text_end":17744,"text_sha256":"64606a7da5a48e73d24e89218ef70e9ae31fd68c89ccc0d8d3c54884760cfcbf","text_start":17702},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1708,"page_start":1665,"pdf_page":11,"printed_page":10,"text":"170 E85770 T otal Premium Tax Credit Amount","text_end":17828,"text_sha256":"793741882a24cee3341e5eb553e9c81fa8166f8b95ff5355390ebd0446cc47dc","text_start":17785},"name":"E85770","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Premium Tax Credit Amount","publisher_position":170,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":213,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1723,"page_start":1679,"pdf_page":11,"printed_page":10,"text":"171 E85775 Advance Premium Tax Credit Amount","text_end":17789,"text_sha256":"be57870e63e21ecc92ee3c7859b22f870e9c8a02fee6d6eadf1829dd0b3cfc7e","text_start":17745},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1754,"page_start":1709,"pdf_page":11,"printed_page":10,"text":"171 E85775 A dvance Premium Tax Credit Amount","text_end":17874,"text_sha256":"4ce5b76a95041589061900712ea4b6097f4b65db7603909770ef8ae95e62f041","text_start":17829},"name":"E85775","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Advance Premium Tax Credit Amount","publisher_position":171,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":214,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1779,"page_start":1724,"pdf_page":11,"printed_page":10,"text":"172 E85785 Excess Advance Payment of Premium Tax Credit","text_end":17845,"text_sha256":"682b60d37c6a051d28c738cbaadcaa27355e5a80cbbd6fac8b118e0c2b81aabd","text_start":17790},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1811,"page_start":1755,"pdf_page":11,"printed_page":10,"text":"172 E85785 E xcess Advance Payment of Premium Tax Credit","text_end":17931,"text_sha256":"8c7824521a2c4f39d248d9788e4864c655682ec4943e4af2413b8598c0779a89","text_start":17875},"name":"E85785","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Excess Advance Payment of Premium Tax Credit","publisher_position":172,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":215,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1818,"page_start":1780,"pdf_page":11,"printed_page":10,"text":"173 E85790 Repayment Limitation Amount","text_end":17884,"text_sha256":"e4e51fa05b88a4444696529e93b4af60cc38004e38683eacdf8775813eef7994","text_start":17846},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1851,"page_start":1812,"pdf_page":11,"printed_page":10,"text":"173 E85790 R epayment Limitation Amount","text_end":17971,"text_sha256":"b5717343afbdf2ffd509aa66483e329aa9f0299f74c8abc8949009e53cbe3f6d","text_start":17932},"name":"E85790","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Repayment Limitation Amount","publisher_position":173,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":216,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1853,"page_start":1819,"pdf_page":11,"printed_page":10,"text":"Misc Codes 174 RECID Return ID [1]","text_end":17919,"text_sha256":"c31ab3dc86007f00497fbcbe6a2300dae6ad9fd81d1cd737b9649e9f9935afe9","text_start":17885},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1887,"page_start":1852,"pdf_page":11,"printed_page":10,"text":"Misc Codes 174 RE CID Return ID [1]","text_end":18007,"text_sha256":"c8567954d56a543c67758acf509172bb52fac2f2c16a485af42ad10ea501a383","text_start":17972},"name":"RECID","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Return ID [1]","publisher_position":174,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":217,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1877,"page_start":1854,"pdf_page":11,"printed_page":10,"text":"175 S006 Decimal weight","text_end":17943,"text_sha256":"7c79365e7c8b795f4dab53c0bd2ba80b10d3f8e65097dd8280e677ebf43a3d77","text_start":17920},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1912,"page_start":1888,"pdf_page":11,"printed_page":10,"text":"175 S006 D ecimal weight","text_end":18032,"text_sha256":"be59a7d865776b309ea490873456224b74aa0ac325f30c52883f95b35bad43b1","text_start":18008},"name":"S006","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Decimal weight","publisher_position":175,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":218,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1899,"page_start":1878,"pdf_page":11,"printed_page":10,"text":"176 S008 Sample count","text_end":17965,"text_sha256":"821d80cd5cb84376b5b94f6e77402c96d9daf1cce9bb1f58ca0221951321c2c0","text_start":17944},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1935,"page_start":1913,"pdf_page":11,"printed_page":10,"text":"176 S008 S ample count","text_end":18055,"text_sha256":"a5e8730451f4fa4869912da3750da459fd10d0580567a20f2dc7128f6267d055","text_start":18033},"name":"S008","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Sample count","publisher_position":176,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":219,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1925,"page_start":1900,"pdf_page":11,"printed_page":10,"text":"177 S009 Population count","text_end":17991,"text_sha256":"e91ae26bc32c45d1a8ac73a3bcf09445274eefaf2a47c35f27838f072adeaaf4","text_start":17966},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1962,"page_start":1936,"pdf_page":11,"printed_page":10,"text":"177 S009 P opulation count","text_end":18082,"text_sha256":"9d6363dc4716f7e51d2a6d595c5d339f4d8faa5603f28e725e4b444fd64b4379","text_start":18056},"name":"S009","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Population count","publisher_position":177,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":220,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1947,"page_start":1926,"pdf_page":11,"printed_page":10,"text":"178 WSAMP Sample code","text_end":18013,"text_sha256":"c658f35e5c9eb6c0039ac9e4057240be468074bc93ca49b175c2ae9c4bc6cfae","text_start":17992},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1985,"page_start":1963,"pdf_page":11,"printed_page":10,"text":"178 WSAMP S ample code","text_end":18105,"text_sha256":"ede3f968711699b9778a4d39dabd97d3cca4bc4e532a1c8634b768af4fcbbd21","text_start":18083},"name":"WSAMP","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Sample code","publisher_position":178,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":221,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1974,"page_start":1948,"pdf_page":11,"printed_page":10,"text":"179 TXRT Marginal tax rate","text_end":18040,"text_sha256":"62f98fd3bdca207c721c1ddaeac8f9a6249fc3a102b323776336cf4832348708","text_start":18014},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":2013,"page_start":1986,"pdf_page":11,"printed_page":10,"text":"179 TXRT M arginal tax rate","text_end":18133,"text_sha256":"9b89661798b6580b6bbd987fc7a91d8bd36dc8ef58e537144e5c2cf7b688b96e","text_start":18106},"name":"TXRT","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Marginal tax rate","publisher_position":179,"publisher_signed_marker":false}],"corroborating_edition_tolerant_positions":[{"name":"P04470","position":32},{"name":"E04600","position":33},{"name":"E07180","position":42},{"name":"E06500","position":53},{"name":"E58950","position":141},{"name":"P60100","position":143},{"name":"P65300","position":151},{"name":"E82200","position":154},{"name":"P87482","position":158},{"name":"P86421","position":162},{"name":"E85050","position":163},{"name":"E85570","position":167},{"name":"RECID","position":174}],"delivered_header_width":222,"printed_order_matches_delivered_order":true},"not_selected_reasons":{"amount_field_outside_this_increment":"A publisher amount field that this bounded increment does not project. It stays source-only: no unit, sign or period claim is made about it here, and nothing downstream may treat its absence from the projection as a statement about its value.","code_field_not_an_amount":"The publisher's CORE RECORD LAYOUT lists this name under 'Code Fields'. It is a code, a count or an indicator, never a dollar amount.","misc_code_not_an_amount":"The publisher's own 'Misc Codes' block sits inside the amount-field list and holds a return identifier, a weight with implied decimals, two sample counts, a sample code and a marginal tax rate with an implied decimal point. Membership of the amount list is therefore not evidence that a column is money."},"notable_source_only":{"E01100":"Capital gain distributions reported on Form 1040. A capital-gains component this increment does not project, because no publisher statement composes it with E01000, P22250 or P23250.","E01400":"Taxable IRA distribution. Retirement-adjacent but a distinct publisher concept from pensions and annuities; not projected here.","E02000":"Schedule E net income or loss. Business-adjacent but a different publisher concept from Schedule C; not projected here.","E02400":"Gross Social Security benefits. Not projected in this increment.","E05800":"Income tax before credits: the publisher's own before-credits quantity, and the reason E06500 must not be read as one.","E06500":"Total income tax. A tax outcome of 2015 law applied to 2015 amounts. Aging it would not compute any later year's policy, and the archived SOI measure table binds this same code to both 'total income tax' and 'income tax before credits', a conflict a successor must settle. It stays source-year evidence.","E08800":"Income tax after credits (SOI). A tax outcome; source-only.","E10300":"Total tax liability (SOI). A tax outcome; source-only.","E22320":"Long term gain/loss from other forms on Schedule D. See E01100.","E22370":"Schedule D capital gain distributions. See E01100.","E25920":"Partnership total passive loss specifically, not a generic gross loss. The archived code path consumed this code without ever describing it; the label alone establishes no blank or negative decoder, so it stays source-only.","E26270":"Combined partnership and S corporation net income/loss. Not projected; the archived producer both sign-branched and AGI-fallback aged this code, which double-counted it.","E27200":"Farm rent net income or loss. A separate publisher row from the Schedule F farm result; not projected here.","E30400":"The record-layout row says 'primary taxpayer', but the publisher's own definition on the same booklet's detailed pages says the combined primary and secondary net earnings from self-employment *subject to Social Security tax*. The label and the definition disagree, and the definition governs. See the refused uncapped-SE derivation.","E30500":"The secondary taxpayer's capped amount. See E30400 and the refused uncapped-SE derivation."},"prefix_is_not_evidence":"No column's role is inferred from an E, P, S or T prefix or from membership of the amount-field list. RECID, S006, S008, S009, WSAMP and TXRT are inside that list and none is money; TXRT carries an implied decimal point."},"non_claims":{"aggregate_record_disaggregation":false,"amount_growth_or_restatement":false,"fit_or_calibration":false,"microdata_read_by_this_generator":false,"person_or_household_expansion":false,"policyengine_import":false,"release_eligibility":false,"target_year_frame_entry":false,"tax_outcome_projection":false,"weight_promotion":false},"projection":{"amounts_decoded":true,"column_count":13,"columns":[{"concept":"salaries_and_wages","concept_group":"wages","delivered_index":43,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":["multivariate_blurring","itemized_blurring_007","nonitemized_wage_noise"],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00200","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9732,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9697},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Wages carry two documented edits beyond the universal ones: multivariate blurring wherever at least two of wages, state and local income taxes and real estate taxes are nonzero, and up to 10 percent uniform random noise on nonitemized returns sampled at 0.07 percent. A wage value is therefore an edited amount, not one taxpayer's reported wage."],"publisher_label":"Salaries and wages","publisher_position":1,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"taxable_interest_received","concept_group":"interest","delivered_index":44,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00300","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9759,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9733},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Interest received","publisher_position":2,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"relationship_to_tax_exempt_interest","statement":"Neither booklet states whether E00300 and E00400 partition interest or overlap. They are retained as two independent columns and never summed here."}]},{"concept":"tax_exempt_interest_income","concept_group":"interest","delivered_index":45,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00400","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9795,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9760},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Tax-exempt interest income","publisher_position":3,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"relationship_to_taxable_interest","statement":"See E00300. No publisher statement partitions the two."}]},{"concept":"dividends_included_in_agi","concept_group":"dividends","delivered_index":46,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00600","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9830,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9796},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Dividends included in AGI","publisher_position":4,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"qualified_dividend_containment","statement":"Neither booklet states that E00650 is a part of E00600. The two are retained independently; no difference is computed."}]},{"concept":"qualified_dividends","concept_group":"dividends","delivered_index":47,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00650","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9859,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9831},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Qualified Dividends","publisher_position":5,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"qualified_dividend_containment","statement":"See E00600. Containment is not documented, so no ordinary dividend residual is derived."}]},{"concept":"schedule_c_net_profit_or_loss","concept_group":"business_income","delivered_index":50,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00900","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":9987,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9921},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["The record layout prints the (+/-) marker, so negative values are documented. This is a net Schedule C result, not gross receipts, and it is not self-employment earnings subject to Social Security tax; see the refused E30400/E30500 derivation."],"publisher_label":"Business or profession (Schedule C) net profit/loss (+/-)","publisher_position":8,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"net_capital_gain_or_loss","concept_group":"capital_gains","delivered_index":51,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01000","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10027,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":9988},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Marker-signed on the record layout row."],"publisher_label":"Net capital gain or loss (+/-)","publisher_position":9,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"No publisher statement composes E01000 out of P22250 and P23250, and the statutory capital loss limitation is not described in either booklet. The three columns are retained independently and never reconciled here."}]},{"concept":"total_pensions_and_annuities_received","concept_group":"pensions","delivered_index":55,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01500","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10208,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10161},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Total pensions and annuities received","publisher_position":13,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"taxable_share_relationship","statement":"The labels say 'received' and 'included in AGI' but neither booklet defines E01700 as a part of E01500. No taxable share is derived."}]},{"concept":"pensions_and_annuities_included_in_agi","concept_group":"pensions","delivered_index":56,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01700","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10257,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10209},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Pensions and annuities included in AGI","publisher_position":14,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"taxable_share_relationship","statement":"See E01500."}]},{"concept":"schedule_f_net_profit_or_loss","concept_group":"farm_income","delivered_index":58,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E02100","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10346,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10304},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Schedule F net profit or loss, marker-signed. Farm rent (E27200) is a separate publisher row and is not selected here."],"publisher_label":"Schedule F net profit/loss (+/-)","publisher_position":16,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"adjusted_gross_income_deficit","concept_group":"adjusted_gross_income","delivered_index":73,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00100","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1464,"page_start":1405,"pdf_page":7,"printed_page":6,"text":"AGI 31 E00100 Adjusted Gross Income (deficit) (AGI) (+/-)","text_end":11024,"text_sha256":"40277b800d8de41a73e6df8b5a317a17bf4434f5b70f0176d19b390cbe993012","text_start":10965},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Reported adjusted gross income as delivered. This is the publisher's own edited AGI field, not a quantity derived from the other projected columns and not a tax outcome recomputed under any year's law."],"publisher_label":"Adjusted Gross Income (deficit) (AGI) (+/-)","publisher_position":31,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"component_identity_with_the_projected_amounts","statement":"Neither booklet states that reported AGI equals any sum or difference of the twelve projected columns, and the publisher's rebalancing edit is not documented as preserving such an identity. AGI is retained as its own delivered column; nothing here computes, checks or repairs an accounting relationship between it and the components."},{"code":"deficit_sign_semantics","statement":"The record layout prints the (+/-) marker, so negative values are documented, but the booklets do not define which deductions a negative AGI reflects. The sign is preserved and never interpreted."}]},{"concept":"short_term_gains_less_losses_net_of_carryover","concept_group":"capital_gains","delivered_index":143,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"P22250","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14185,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14118},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["The record-layout row prints no (+/-) marker, but the publisher's own definition is 'all short-term capital gains less losses ... except for the short-term capital loss carryover', which is a netted quantity. The marker's absence is recorded verbatim rather than treated as a nonnegative domain.","E22250 does not exist in the delivered layout or in either booklet. It is never aliased onto this column."],"publisher_label":"Short-term gains less losses net of carryover","publisher_position":101,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":["p22250_definition"],"negative_values":"documented_by_publisher_definition_text","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"See E01000."}]},{"concept":"long_term_gains_less_losses_net_of_carryover","concept_group":"capital_gains","delivered_index":146,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"P23250","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14362,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14307},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["As P22250: netted by definition, unmarked on the layout row."],"publisher_label":"Long-term gains less losses net of carryover","publisher_position":104,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":["p23250_definition"],"negative_values":"documented_by_publisher_definition_text","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"See E01000."}]}],"concept_groups":["adjusted_gross_income","business_income","capital_gains","dividends","farm_income","interest","pensions","wages"],"grain":"source_return","period_semantics":"unresolved_raw_FLPDYR","row_policy":"every delivered main record is projected, including the 88,021 with no demographic row and the four disclosure aggregates","target_year_use":"refused_pending_independent_growth_decision"},"projection_edition":{"adds":["E00100"],"edition":"thirteen_field_reported_agi","not_derived":"E00100 is the delivered reported field. No AGI is computed, reconstructed from components, or reconciled against them.","parent_document_canonical_sha256":"5e4f5943624427376a13e656a6fedd378e2a0ff8276a90bd10bc5149f1291749","parent_document_sha256":"986f632b2edac960ad04b6dfde4b39063e4a09cf953b04cd473df25724ad369c","parent_generator_sha256":"c35b8b20342376d404427bb00f522b81e115e1f079ed53e81f923f9a7660b48b","parent_projected_fields":["E00200","E00300","E00400","E00600","E00650","E00900","E01000","E01500","E01700","E02100","P22250","P23250"],"preserves":"The twelve-field document, its artifact type microcosm.us.puf_2015_monetary_source_projection/3, its kernel and its store key are untouched. This edition is a separate document behind a separate artifact type.","supersedes_notable_source_only":{"E00100":{"upstream_statement":"Adjusted gross income (deficit). The reviewed growth slice already retains source-year AGI as provenance; projecting it here would create a second, competing source of the same quantity.","why_it_no_longer_applies":"The growth slice's puf_source_year_agi column is a copy of that transform's own input E00100, not an independent AGI source, and GrowthRecipe requires E00100 as a declared money field. Without this column there is no AGI on the projection path at all, so projecting it is the only source rather than a second competing one."}}},"raw_source_definition":{"bytes":9706,"relationship":"This document extends the reviewed raw source definition. It restates none of that document's pins as its own authority and never modifies it; the artifact binds both.","schema":"microcosm.us.puf_2015_raw_source_definition","schema_version":1,"sha256":"c2ac73aa86b0b210e9919fd551f26c469098e6bade91bf8b1e03e1cf91e308e4"},"refused_derivations":{"uncapped_self_employment_earnings":{"anchors":["e30400_definition","e30500_definition"],"reason":"E30400 is the combined primary and secondary net earnings from self-employment *subject to Social Security tax* — the lesser of net Schedule SE earnings and the maximum amount subject to the tax — and E30500 is the secondary taxpayer's own capped amount. Subtracting one from the other yields a capped primary residual, not uncapped self-employment earnings, and grossing the difference up cannot recover what the cap removed.","refused_inputs":["E30400","E30500"]}},"refused_fields":{"E22250":{"policy":"never_aliased_to_P22250","reason":"The archived producer's screened-field handling and several downstream discussions treat E22250 and P22250 as one column. The delivered header carries P22250 only, and neither booklet prints E22250 anywhere. Normalising one onto the other would be a guess, so a declaration or a delivered header naming E22250 is a refusal instead.","state":"absent_from_delivered_layout_and_from_both_retained_booklets"}},"route":"packaged","schema":"microcosm.us.puf_2015_monetary_source_projection","schema_version":1,"source_conflicts":{"record_count_includes_header":{"anchors":["record_count_intro","record_count_technical"],"statement":"The technical description says 207,696 CSV records with the first record containing field names, implying 207,695 data records. The accepted raw-byte audit observed 207,696 data records plus a header. Preserve the measured count without silently reconciling this wording conflict.","status":"documentary_wording_unresolved"},"rounding_five_dollar_tie":{"anchors":["rounding_grid"],"statement":"The nearest-ten-dollar rule does not state how a five-dollar tie is resolved. A delivered zero does not identify reported zero, absence, nonapplicability or another edit.","status":"documentary_rounding_tie_unresolved"}},"sources":{"demographic":{"bytes":2225050,"codec":"us-puf-2015-demographic-raw-v1","data_records":119675,"delivered_header_width":7,"git_blob_sha1":"46c63e94d94a3662672304b60008dbd514b10e1b","header_record_canonical_sha256":"856cadc0522d2da8c2be67ac7cf6b44b257d97e5e725f9e352ee9fa1a403ccdf","sha256":"2d275cc405a1d0b254855628eabefe6c6484570ab789b2777a037ea604a82e81","source_name":"puf_2015_demographic"},"main":{"bytes":126034649,"codec":"us-puf-2015-main-raw-v1","data_records":207696,"delivered_header_width":222,"git_blob_sha1":"a9fa4cd5b4b3f93abaf2a068cf709620c88ea33b","header_record_canonical_sha256":"3652a41e23a9f51e5e1ffe5fa7bcbb05c89f8b2ba53eec789dcad026789c98f2","sha256":"0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df","source_name":"puf_2015_main"}}} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_source_projection.json b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_source_projection.json new file mode 100644 index 000000000..39c075481 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_monetary_source_projection.json @@ -0,0 +1 @@ +{"anchors":{"aggregate_recid_footnote":{"booklet":"main_2022","needle":"[1] Records containing aggregated amounts are identified by RECID=999996, 999997, 999998, and 999999","needle_sha256":"2686c04e7238897d583f14f6a152502f55a7d15cf595c2c63dbb5924f2d72e24","page_end":2075,"page_start":1975,"pdf_page":11,"printed_page":10,"text":"[1] Records containing aggregated amounts are identified by RECID=999996, 999997, 999998, and 999999","text_end":18141,"text_sha256":"2686c04e7238897d583f14f6a152502f55a7d15cf595c2c63dbb5924f2d72e24","text_start":18041},"aggregate_records":{"booklet":"main_2022","needle":"Second, returns that contain one or more amount fields with values deeme d extremely","needle_sha256":"bf1d708a634bd26f9ccbf59a7b873b07e22e3f8374921efdf0d565afc57f664a","page_end":1227,"page_start":443,"pdf_page":4,"printed_page":3,"text":"Second, returns that contain one or more amount fields with values deeme d extremely \nlarge\n1 have been excluded from the microdata sample and are aggregated into one of four records2, \nidentified by RECID=999996 for returns reporting negative Adjusted Gross Income (AGI), \nRECID=999997 for returns reporting positive AGI between $0 and $10,000,000, RECID=999998 \nfor returns reporting positive AGI between $10,000,001 and $100,000,000, and RECID=999999 \nfor returns reporting positive AGI of $100,000,001 or more. The rules for identifying extremely \nlarge values are not applied to amount fields that are statutorily capped, subject to income limits, \nor calculated from other fields that are subject to these rules. Information regarding these aggregate \nreturns can be found in ","text_end":3883,"text_sha256":"e53f1ac33b87d6ba5d2274fc83cbc33b245918cfe88a33d33f74b97cbe0e853f","text_start":3099},"amount_field_width":{"booklet":"main_2022","needle":"12 digit amount fields","needle_sha256":"db574766fcc9683b21083244a370659b5b44104660f62c76615916974c819641","page_end":136,"page_start":114,"pdf_page":7,"printed_page":6,"text":"12 digit amount fields","text_end":9696,"text_sha256":"db574766fcc9683b21083244a370659b5b44104660f62c76615916974c819641","text_start":9674},"amount_fields_heading":{"booklet":"main_2022","needle":"Amount Fields and Large Codes","needle_sha256":"02d4caf81fa5527bb8094826e492f2f9d9c7ce880293f3b3d12cc20059d2070a","page_end":98,"page_start":69,"pdf_page":7,"printed_page":6,"text":"Amount Fields and Large Codes","text_end":9658,"text_sha256":"02d4caf81fa5527bb8094826e492f2f9d9c7ce880293f3b3d12cc20059d2070a","text_start":9629},"code_fields_heading":{"booklet":"main_2022","needle":"CORE RECORD LAYOUT","needle_sha256":"50320ab982369f7648e226b45f83bd6eb80a0d293664ea259ad2c21e77ce8496","page_end":72,"page_start":54,"pdf_page":6,"printed_page":5,"text":"CORE RECORD LAYOUT","text_end":8591,"text_sha256":"50320ab982369f7648e226b45f83bd6eb80a0d293664ea259ad2c21e77ce8496","text_start":8573},"csv_order":{"booklet":"main_2022","needle":"Field names have been ordered first","needle_sha256":"20a74ea4f09169699e5b2ece904448b24f7844529a683243a4e3c35ed4ebbb55","page_end":395,"page_start":256,"pdf_page":27,"printed_page":26,"text":"Field names have been ordered first \nby code fields followed by amount fields in the same order as presented on pages 5 through 10. \n \n ","text_end":47599,"text_sha256":"51c519bd5fe4634e29f8a84928ded60babb8ddcf09e49f81bb260f05cb95a7cc","text_start":47460},"demographic_join_section":{"booklet":"demographic_2022","needle":"This section describes the domain of the supplemental file and method for joining to the 2015","needle_sha256":"0de8bafb27d9fa0deb73a67039f545f2f0628edcb8cab89126c695f6fef31d5f","page_end":183,"page_start":90,"pdf_page":10,"printed_page":9,"text":"This section describes the domain of the supplemental file and method for joining to the 2015","text_end":10136,"text_sha256":"0de8bafb27d9fa0deb73a67039f545f2f0628edcb8cab89126c695f6fef31d5f","text_start":10043},"demographic_strata_restriction":{"booklet":"demographic_2022","needle":"found on the 2015 PUF for records in strata 7 through 13, or those returns","needle_sha256":"b6996405e87a53511dd51a9a5243f0d3d17d7ce41bdd2ce6fc12113a7a7b9425","page_end":411,"page_start":337,"pdf_page":10,"printed_page":9,"text":"found on the 2015 PUF for records in strata 7 through 13, or those returns","text_end":10364,"text_sha256":"b6996405e87a53511dd51a9a5243f0d3d17d7ce41bdd2ce6fc12113a7a7b9425","text_start":10290},"e30400_definition":{"booklet":"main_2022","needle":"E30400 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO","needle_sha256":"490778fc1ebf2d244613820d6a90cf2e2c8fe584ffb06d1019adaf9c4e898393","page_end":1554,"page_start":1166,"pdf_page":24,"printed_page":23,"text":"E30400 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO \nSOCIAL SECURITY TAX, PRIMARY AND SECONDARY TAXPAYERS \nThe lesser of Schedule SE, Section B, Part I line 6 (net Schedule SE earnings) or \nPart I line 9 (maximum amount subject to Social Security tax) on both the primary \nand secondary taxpayers’ Schedule SE. For Section A of Schedule SE, this is line \n4. \n \nE30500 NET EARNING","text_end":44200,"text_sha256":"8da0186ad3a92e40587ffcf02c926ebec043b808166092a208d46704d623de8d","text_start":43812},"e30500_definition":{"booklet":"main_2022","needle":"E30500 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO","needle_sha256":"2b9922cbc4a7f61482c7b6f8a931c82d6e88c8939551b0ed389617df99bbfad3","page_end":1914,"page_start":1536,"pdf_page":24,"printed_page":23,"text":"E30500 NET EARNINGS FROM SELF-EMPLOYMENT INCOME SUBJECT TO \nSOCIAL SECURITY TAX, SECONDARY TAXPAYER \nThe lesser of Schedule SE, Section B, Part I line 6 (net Schedule SE earnings) or \nPart I line 9 (maximum amount subject to Social Security tax) on the secondary \ntaxpayer’s Schedule SE. For Section A of Schedule SE, this is line 4. \n \n4 Refundable credits include the Making ","text_end":44560,"text_sha256":"4966f97612b401b41a49d40b82d9f879047c4586129fe93edd63e1a60b35cb2d","text_start":44182},"fiscal_year_conversion":{"booklet":"main_2022","needle":"First, fiscal year returns have been converted to reflect the most recent year-end Tax Year","needle_sha256":"e849f8821b08b5075693d85a87df27204351c92bcb1cef439e5b85374c2d622c","page_end":487,"page_start":256,"pdf_page":4,"printed_page":3,"text":"First, fiscal year returns have been converted to reflect the most recent year-end Tax Year \nand returns filed for years earlier than Tax Year 2012 have been removed from the file. \n \n Second, returns that contain one or more amo","text_end":3143,"text_sha256":"03f8559da89549befa950b5c13b0e1e7a39d3472728f4ba28e5f4c3797d67f7c","text_start":2912},"flpdyr_printed_range":{"booklet":"main_2022","needle":"8. FLPDYR Filing (Accounting) Period, Year:","needle_sha256":"d89025f42eade49e5b783e5893d339dbde3aadd1a6136bf18f6a50d9cc38e9b9","page_end":633,"page_start":450,"pdf_page":14,"printed_page":13,"text":"8. FLPDYR Filing (Accounting) Period, Year: \n \nTax Year ........................................................................ 2011-2014 \n \n \n9. FLPDMO Filing (Accounting) Period, M","text_end":24008,"text_sha256":"923c432965a0c6d3713033c8fe8e88b9858ecb098de0cb7944bd1942aded2da0","text_start":23825},"itemized_blurring_007":{"booklet":"main_2022","needle":"Multivariate blurring has been applied to itemized tax returns with nonzero values","needle_sha256":"e13235e26ef16aba4d8992fb9bc1c6e5c6fd84a301c79de030b28fea319e12d1","page_end":1157,"page_start":835,"pdf_page":5,"printed_page":4,"text":"Multivariate blurring has been applied to itemized tax returns with nonzero values in \nat least two of the following fields: wages and salaries, state and local income taxes, \nand real estate taxes. Blurring of these fields is handled in the same manner as those \nrecords sampled at a rate greater than 0.07 percent. \n","text_end":6762,"text_sha256":"a13012f370078a41fd9ec84e21b7ea2c1590a42be1dd9027a7727acaad1caf19","text_start":6440},"multivariate_blurring":{"booklet":"main_2022","needle":"Multivariate blurring has been applied to these returns with nonzero values in at least","needle_sha256":"247d89d50a53cb8d3aca2d7d68bbf614ea0d161f22d4292b3734837ab5dbbe62","page_end":2544,"page_start":2197,"pdf_page":4,"printed_page":3,"text":"Multivariate blurring has been applied to these returns with nonzero values in at least \ntwo of the following fields: wages and salaries, state and local income taxes, and real \nestate taxes. Prior to blurring, these returns are grouped into one of 10 categories \nbased on their filing status and the number of dependents, and then further group","text_end":5200,"text_sha256":"4d2b0c5f244c64ea4a52f0b0aab3b22de9a05e1345edbd4742995106176f84ea","text_start":4853},"national_estimates_only":{"booklet":"main_2022","needle":"2015 Public Use Tax File is only appropriate for national level estima tes.","needle_sha256":"753825a8a47b9f7378dc3007206a4eea4a5ca35f84744d921230339b2ab68883","page_end":625,"page_start":550,"pdf_page":3,"printed_page":2,"text":"2015 Public Use Tax File is only appropriate for national level estima tes.","text_end":2133,"text_sha256":"753825a8a47b9f7378dc3007206a4eea4a5ca35f84744d921230339b2ab68883","text_start":2058},"nonitemized_wage_noise":{"booklet":"main_2022","needle":"For nonitemized tax returns, up to 10 percent of random noise, drawn from a uniform","needle_sha256":"9ceea058079776ba675b63c5194573294e651ef6699d218667eedd1c749e75d6","page_end":1462,"page_start":1159,"pdf_page":5,"printed_page":4,"text":"For nonitemized tax returns, up to 10 percent of random noise, drawn from a uniform \ndistribution, has been a dded to wages and salaries. The random noise has been \nconstrained as to not disqualify previously qualifying Earned Income Tax Credit \nrecipients or Retirement Saver’s Credit recipients. \n \n","text_end":7067,"text_sha256":"b51a7c2443aaeea041f2bbbd108e7571a39d8d65222c0cd082397eb239650a33","text_start":6764},"not_one_taxpayer":{"booklet":"main_2022","needle":"Since individual records in this file may or may not contain data from just one tax return","needle_sha256":"5f651403aea3e17b13254dfb1d4f6b48001ea3007a5dc40e36ff73d2869ee319","page_end":2913,"page_start":2538,"pdf_page":5,"printed_page":4,"text":"Since individual records in this file may or may not contain data from just one tax return--\nand never contain the full item content of any one tax return-- we request that researchers make \nthese facts clear in publishing their findings. In particular, they should refrain from using any \nlanguage that would imply that they had access to individual taxpayers' records. \n ","text_end":8518,"text_sha256":"99af06c9990f3dbb468724f54de3b711fd6d4a9e78b9c83672f5829db76b7ecb","text_start":8143},"p22250_definition":{"booklet":"main_2022","needle":"P22250 SHORT-TERM GAINS LESS LOSSES NET OF CARRYOVER","needle_sha256":"9ff26c5fade441cfb71aa8e9472922bbdf19a61b20955a053962097c76bc69c5","page_end":702,"page_start":450,"pdf_page":24,"printed_page":23,"text":"P22250 SHORT-TERM GAINS LESS LOSSES NET OF CARRYOVER \nAll short-term capital gains less losses shown in column (h) of Schedule D except \nfor the short-term capital loss carryover. \n \nP23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYO","text_end":43348,"text_sha256":"93098c0bfddb1298db9019e2a47657d95fd943c7cd5f6a76a239efc948518189","text_start":43096},"p23250_definition":{"booklet":"main_2022","needle":"P23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYOVER","needle_sha256":"3d19ab3284dd523bee3cb36f38f7abb88419fbfc3038b819a66f05a8d068b1bd","page_end":895,"page_start":643,"pdf_page":24,"printed_page":23,"text":"P23250 LONG-TERM GAINS LESS LOSSES NET OF CARRYOVER \nAll long-term capital gains less losses shown in column (h) of Schedule D except \nfor the long-term capital loss carryover \n \nP25350 TOTAL RENTS AND ROYALTIES RECEIVED \n The sum of lines ","text_end":43541,"text_sha256":"419b64e90741764c0ed815a165821259e5860b27c7ef1fad217cc2a1f47d69a1","text_start":43289},"rebalancing":{"booklet":"main_2022","needle":"Finally, all records in the file are rebalanced to ensure accounting accuracy","needle_sha256":"b8255f0567486c82414832afad9cc3304cb7e294b39a430579976675f8164306","page_end":2572,"page_start":2405,"pdf_page":5,"printed_page":4,"text":"Finally, all records in the file are rebalanced to ensure accounting accuracy after the above \ndisclosure procedures are applied. \n \nSince individual records in this f","text_end":8177,"text_sha256":"4248cde7dd22d71756c2400db4837550b96aa519df2b7b6f9c6be71225e94347","text_start":8010},"record_count_intro":{"booklet":"main_2022","needle":"The Internal Revenue Service 2015 Public Use Tax File, which contains 207,696 records","needle_sha256":"27a686d1fbf3e6850267f7616c1e1dd87baa0af8ea0cc740a88a7052c1809add","page_end":155,"page_start":70,"pdf_page":3,"printed_page":2,"text":"The Internal Revenue Service 2015 Public Use Tax File, which contains 207,696 records","text_end":1663,"text_sha256":"27a686d1fbf3e6850267f7616c1e1dd87baa0af8ea0cc740a88a7052c1809add","text_start":1578},"record_count_technical":{"booklet":"main_2022","needle":"This comma separated file contains 207,696","needle_sha256":"7a30ba3ee0b6719fbdf0ccfdf9105407da36683a09d7876be92b62bbd6cda930","page_end":282,"page_start":150,"pdf_page":27,"printed_page":26,"text":"This comma separated file contains 207,696 \nrecords where the first record consists of only field names. Field names have been orde","text_end":47486,"text_sha256":"5ea7fb638fd255ff77436d39fd3f806ee64265ea2824f875e035c976fd83916e","text_start":47354},"rounding_grid":{"booklet":"main_2022","needle":"Ninth, for all records on the file, all amount fields have been rounded.","needle_sha256":"d9ab4ab8efe86cf0c235da1904e664c3a643cdfd2e10e15259a4d16d824d6a38","page_end":2447,"page_start":1955,"pdf_page":5,"printed_page":4,"text":"Ninth, for all records on the file, all amount fields have been rounded. Amounts, in absolute \nvalues, above $100,000 are rounded to the four most significant digits from left to right(e.g., \n$228,867 = $228,900 and $1,158,235 = $1,158,000). Amounts between $10,000 and $100,000 are \nrounded to the nearest $100. Amounts between $5 and $10,000 are rounded to the nearest $10. \nNonzero amounts less than $5 are set to $2, with sign retained. \n \n Finally, all records in the file are rebal","text_end":8052,"text_sha256":"eb56b16079f3e2bf633b7d72cecf8e20ca6e7837c7fc5570790129dcee517531","text_start":7560},"s006_definition":{"booklet":"main_2022","needle":"S006 DECIMAL WEIGHT","needle_sha256":"981896b68f40bde5639176999f2f2f856aca3cfb9b1ac4f967fe9e83c0830c81","page_end":1237,"page_start":958,"pdf_page":25,"printed_page":24,"text":"S006 DECIMAL WEIGHT \nThis field was derived by dividing the computer population count of returns in a \nsample stratum by the number of sample returns for that stratum (carried to two \ndecimal places). The decimal place is implied. Estimates derived from each \nreturn record mus","text_end":46079,"text_sha256":"92641e3f7dfc09690b3aed4c110767d01a15819f82ff04fee4fda546584c06bf","text_start":45800},"txrt_implied_decimal":{"booklet":"main_2022","needle":"that this field uses an implied decimal point (28.0 percent is represented by 280).","needle_sha256":"ef23bfa821ae8f4f3aa57420ae2ed24b987d6ea93780917e1288a70ce8f43be6","page_end":407,"page_start":324,"pdf_page":26,"printed_page":25,"text":"that this field uses an implied decimal point (28.0 percent is represented by 280).","text_end":46825,"text_sha256":"ef23bfa821ae8f4f3aa57420ae2ed24b987d6ea93780917e1288a70ce8f43be6","text_start":46742}},"authority":"irs_soi_2015_public_use_file_retained_booklets","booklets":{"demographic_2022":{"edition":"September 2022","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":345247,"pdf_pages":13,"pdf_sha256":"7bb623c7c7631e30eb620f0a522d74bff59487c1778d0b2023b2536cead5f38b","role":"primary_layout_authority","span_digest":"sha256 of the span encoded as UTF-8","text_characters":19352,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"d2074c3a2e2dabb6d25da4a9a62b8d9302e001cc5d86939edc0bdd09719d8c9a","title":"2015 Public Use Tax Demographic File"},"main_2022":{"edition":"September 2022","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":23417059,"pdf_pages":88,"pdf_sha256":"dfb2a6c85f98cba95624ece7c2bd239a0fd09457ca2498777c8242208ff7295b","role":"primary_layout_authority","span_digest":"sha256 of the span encoded as UTF-8","text_characters":93621,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"c2446b589f60592f337ed45f0d82406593fddafd16ec45d06440c07c35418765","title":"2015 Public Use Tax File General Description Booklet"},"main_2023":{"edition":"Updated February 2023","offset_unit":"unicode_codepoints_into_the_pinned_text","pdf_bytes":23413549,"pdf_pages":88,"pdf_sha256":"93a1d863b5a4af21b624714fb2bb3f9d30509defdd8f2c052c2c5ca4d9cce35c","role":"corroborating_edition","span_digest":"sha256 of the span encoded as UTF-8","text_characters":93783,"text_extraction":"pypdf 6.11.0, strict, page texts joined with U+000C","text_sha256":"c7f84a1ec266d341b713ed1251c7056b6636e6a8ac23ef1124f21fe367f34353","title":"2015 Public Use Tax File General Description Booklet"}},"generation":{"generator":"tools/build_us_puf_source_layout.py","generator_sha256":"c35b8b20342376d404427bb00f522b81e115e1f079ed53e81f923f9a7660b48b","reads":["the three pinned booklet text extractions","the packaged raw source definition"],"reads_microdata":false},"layout":{"demographic":{"code_fields":7,"columns":[{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":115,"page_start":86,"pdf_page":6,"printed_page":5,"text":"1. AGEDP1 Age of Dependent 1 ","text_end":3806,"text_sha256":"209cf4c864128785757d242779538f0ee5b0bb935cca291d020ab7ed7efade3c","text_start":3777},"delivered_index":0,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":116,"page_start":104,"pdf_page":5,"printed_page":4,"text":" 1. AGEDP11 ","text_end":3224,"text_sha256":"e032518be848a70629a50c3b886430b0ec8b200475c4fa8b469d698f7794e478","text_start":3212},"name":"AGEDP1","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 1","publisher_position":1,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":1182,"page_start":1153,"pdf_page":6,"printed_page":5,"text":"2. AGEDP2 Age of Dependent 2 ","text_end":4873,"text_sha256":"52d4f54ef51fd158513cb53774a9b8c98d3c8b1e6334729bd22791f4937e6d5d","text_start":4844},"delivered_index":1,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":129,"page_start":117,"pdf_page":5,"printed_page":4,"text":" 2. AGEDP21 ","text_end":3237,"text_sha256":"586f75b3d4b4702611f91b89a26b212f0a62b5286740e55d440cb7644e897425","text_start":3225},"name":"AGEDP2","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 2","publisher_position":2,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":95,"page_start":66,"pdf_page":7,"printed_page":6,"text":"3. AGEDP3 Age of Dependent 3 ","text_end":6054,"text_sha256":"01751c44a83a96e86a4fcd33696ba5de01f4c9cae9e12a4090d433b747001ae1","text_start":6025},"delivered_index":2,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":"1","page_end":142,"page_start":130,"pdf_page":5,"printed_page":4,"text":" 3. AGEDP31 ","text_end":3250,"text_sha256":"75e66f62320bbcd1968a85db6cae7a52578303cdfab10cb7266b2e68f0e047d7","text_start":3238},"name":"AGEDP3","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age of Dependent 3","publisher_position":3,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":1231,"page_start":1192,"pdf_page":7,"printed_page":6,"text":"4. AGERANGE Age range of Primary Filer ","text_end":7190,"text_sha256":"0d896e896bb5e35eed7e1cb2c70237c01d924333de5aef052e20aab5d5bbc042","text_start":7151},"delivered_index":3,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":156,"page_start":143,"pdf_page":5,"printed_page":4,"text":" 4. AGERANGE ","text_end":3264,"text_sha256":"137b0c36ae39e9423e67b23a641f6775455ff711e295633a910035d7ef7478ec","text_start":3251},"name":"AGERANGE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Age range of Primary Filer","publisher_position":4,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":110,"page_start":64,"pdf_page":8,"printed_page":7,"text":"5. EARNSPLIT Earnings Split on Joint Returns ","text_end":8608,"text_sha256":"9e8326c12d2f58e09f9ce9d83568569dcc6468731e7dd34f58715992c19f2070","text_start":8562},"delivered_index":4,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":171,"page_start":157,"pdf_page":5,"printed_page":4,"text":" 5. EARNSPLIT ","text_end":3279,"text_sha256":"cd6115b551a6da5a2b21c7191327ea923514aee715de0b30a2e61f26c16d6f25","text_start":3265},"name":"EARNSPLIT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Earnings Split on Joint Returns","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":674,"page_start":640,"pdf_page":8,"printed_page":7,"text":"6. GENDER Gender of Primary filer ","text_end":9172,"text_sha256":"2a358c87a94b63cd540a29b76cc359a67de90819cfd3694430894b3df43ce631","text_start":9138},"delivered_index":5,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":183,"page_start":172,"pdf_page":5,"printed_page":4,"text":" 6. GENDER ","text_end":3291,"text_sha256":"af2cf4870ae8abfc99f88c9e3c90cc71c2d8b54eb9cba902fdfdb0e719b63670","text_start":3280},"name":"GENDER","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Gender of Primary filer","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"demographic_2022","page_end":911,"page_start":890,"pdf_page":8,"printed_page":7,"text":"7. RECID RETURN ID ","text_end":9409,"text_sha256":"1415d1f01fe6e062d13b8eeb929909980bc0b9da00becda6f1b24b3570fc8144","text_start":9388},"delivered_index":6,"label_state":"resolved_from_code_definitions","layout_row":{"booklet":"demographic_2022","footnote_markers":null,"page_end":194,"page_start":184,"pdf_page":5,"printed_page":4,"text":" 7. RECID ","text_end":3302,"text_sha256":"98dfcfd8c77226b0297f30f25fde9283b73b6ac87ef268d4f3de27ac109a7f01","text_start":3292},"name":"RECID","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"RETURN ID","publisher_position":7,"publisher_signed_marker":false}],"delivered_header_width":7,"printed_order_matches_delivered_order":true},"main":{"amount_or_large_code_fields":179,"code_fields":43,"columns":[{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":119,"page_start":75,"pdf_page":12,"printed_page":11,"text":"1. AGIR1 Adjusted Gross Income (AGI) Range: ","text_end":18261,"text_sha256":"f304d466d6ad08a9d463641c9b4667edae19aeb36eeca325da90db983145f1a4","text_start":18217},"delivered_index":0,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":102,"page_start":92,"pdf_page":6,"printed_page":5,"text":" 1. AGIR1 ","text_end":8621,"text_sha256":"ef23b0e0e7abf66606a29cda9d566a187f525c1373ed6f09fcb38398b5ca6023","text_start":8611},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":102,"page_start":92,"pdf_page":6,"printed_page":5,"text":" 1. AGIR1 ","text_end":8643,"text_sha256":"ef23b0e0e7abf66606a29cda9d566a187f525c1373ed6f09fcb38398b5ca6023","text_start":8633},"name":"AGIR1","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Adjusted Gross Income (AGI) Range:","publisher_position":1,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":490,"page_start":449,"pdf_page":13,"printed_page":12,"text":"2. CLAIM8965 Health Coverage Exemptions ","text_end":22012,"text_sha256":"377cee7d29bf4e186ead2daccd837bd50f9d7d4fd7b15fe22d4f1508c7fe49e5","text_start":21971},"delivered_index":1,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":120,"page_start":103,"pdf_page":6,"printed_page":5,"text":" 2. CLAIM89651,2 ","text_end":8639,"text_sha256":"34d92e2e3e846dd2917000322e0f7e05378045aebf1c9a2ef2982a07eb4be909","text_start":8622},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":120,"page_start":103,"pdf_page":6,"printed_page":5,"text":" 2. CLAIM89651,2 ","text_end":8661,"text_sha256":"34d92e2e3e846dd2917000322e0f7e05378045aebf1c9a2ef2982a07eb4be909","text_start":8644},"name":"CLAIM8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Health Coverage Exemptions","publisher_position":2,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":674,"page_start":639,"pdf_page":13,"printed_page":12,"text":"3. DSI Dependent Status Indicator: ","text_end":22196,"text_sha256":"907528bcc8c57c09dfb71bdf79228e3d3de69adf38663789ce8903ae3967a424","text_start":22161},"delivered_index":2,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":129,"page_start":121,"pdf_page":6,"printed_page":5,"text":" 3. DSI ","text_end":8648,"text_sha256":"cc9c81fed4dabdf7a1e6074ac8ea0aa291945cf834f127affadbec903e56d53f","text_start":8640},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":129,"page_start":121,"pdf_page":6,"printed_page":5,"text":" 3. DSI ","text_end":8670,"text_sha256":"cc9c81fed4dabdf7a1e6074ac8ea0aa291945cf834f127affadbec903e56d53f","text_start":8662},"name":"DSI","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Dependent Status Indicator:","publisher_position":3,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":949,"page_start":912,"pdf_page":13,"printed_page":12,"text":"4. EFI Electronic Filing Indicator: ","text_end":22471,"text_sha256":"a4467e5ca8e46f72746fb6491f40d66f94edcfaaaa123c4f0d071416f9c6ed70","text_start":22434},"delivered_index":3,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":138,"page_start":130,"pdf_page":6,"printed_page":5,"text":" 4. EFI ","text_end":8657,"text_sha256":"224488a72c014cd6368c5de18c7f75b3e313f13015e9c509ad995ba198c049d3","text_start":8649},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":138,"page_start":130,"pdf_page":6,"printed_page":5,"text":" 4. EFI ","text_end":8679,"text_sha256":"224488a72c014cd6368c5de18c7f75b3e313f13015e9c509ad995ba198c049d3","text_start":8671},"name":"EFI","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Electronic Filing Indicator:","publisher_position":4,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1163,"page_start":1129,"pdf_page":13,"printed_page":12,"text":"5. EIC Earned Income Credit Code: ","text_end":22685,"text_sha256":"9ef3462adfb166c47c84ebd57ffeeaf95f226274ce146d37f9964e2bd7b3545c","text_start":22651},"delivered_index":4,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":147,"page_start":139,"pdf_page":6,"printed_page":5,"text":" 5. EIC ","text_end":8666,"text_sha256":"ddfc450e354ad0e909bc48204bbdd4c82ca9d20981f5252575e9af54c88a2724","text_start":8658},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":147,"page_start":139,"pdf_page":6,"printed_page":5,"text":" 5. EIC ","text_end":8688,"text_sha256":"ddfc450e354ad0e909bc48204bbdd4c82ca9d20981f5252575e9af54c88a2724","text_start":8680},"name":"EIC","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Earned Income Credit Code:","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1594,"page_start":1542,"pdf_page":13,"printed_page":12,"text":"6. ELECT Presidential Election Campaign Fund Boxes: ","text_end":23116,"text_sha256":"e9e30b1db453e79532cef891c8990253a1909c6cc8323e6238e21a28ee71dfa2","text_start":23064},"delivered_index":5,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":158,"page_start":148,"pdf_page":6,"printed_page":5,"text":" 6. ELECT ","text_end":8677,"text_sha256":"7c5188295524711beef5fab70c2dd0960d52793a9da2647e2a1f20c362fd7a53","text_start":8667},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":158,"page_start":148,"pdf_page":6,"printed_page":5,"text":" 6. ELECT ","text_end":8699,"text_sha256":"7c5188295524711beef5fab70c2dd0960d52793a9da2647e2a1f20c362fd7a53","text_start":8689},"name":"ELECT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Presidential Election Campaign Fund Boxes:","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":85,"page_start":53,"pdf_page":14,"printed_page":13,"text":"7. FDED Form of Deduction Code: ","text_end":23460,"text_sha256":"c328a37c4c3f209b45630cdb9df3e0fcb9ab9eae39088e5d510ac1720ac068df","text_start":23428},"delivered_index":6,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":168,"page_start":159,"pdf_page":6,"printed_page":5,"text":" 7. FDED ","text_end":8687,"text_sha256":"e8e6d023b0c3a10a18cf3f5fb54a65342299f83be75b3378ab3f974ae93f8d1c","text_start":8678},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":168,"page_start":159,"pdf_page":6,"printed_page":5,"text":" 7. FDED ","text_end":8709,"text_sha256":"e8e6d023b0c3a10a18cf3f5fb54a65342299f83be75b3378ab3f974ae93f8d1c","text_start":8700},"name":"FDED","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form of Deduction Code:","publisher_position":7,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":494,"page_start":450,"pdf_page":14,"printed_page":13,"text":"8. FLPDYR Filing (Accounting) Period, Year: ","text_end":23869,"text_sha256":"e655ae86dedfa56d7356d721981cf206e43e290f2ab9d821f5ed75dc08447de0","text_start":23825},"delivered_index":7,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":180,"page_start":169,"pdf_page":6,"printed_page":5,"text":" 8. FLPDYR ","text_end":8699,"text_sha256":"2d43a4bd7b53098a69fad6d88ff71c93eac7f4209274a802d99fbb04102a83ce","text_start":8688},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":180,"page_start":169,"pdf_page":6,"printed_page":5,"text":" 8. FLPDYR ","text_end":8721,"text_sha256":"2d43a4bd7b53098a69fad6d88ff71c93eac7f4209274a802d99fbb04102a83ce","text_start":8710},"name":"FLPDYR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Filing (Accounting) Period, Year:","publisher_position":8,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":639,"page_start":594,"pdf_page":14,"printed_page":13,"text":"9. FLPDMO Filing (Accounting) Period, Month: ","text_end":24014,"text_sha256":"1052efbe06ec372bda396ff6873b18f310663c9000a439a954ffb0e62a190da3","text_start":23969},"delivered_index":8,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":192,"page_start":181,"pdf_page":6,"printed_page":5,"text":" 9. FLPDMO ","text_end":8711,"text_sha256":"21b628067e0e0cae93f82779561150bde7ed78afb74d69e455ed1e3186bd3fa4","text_start":8700},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":192,"page_start":181,"pdf_page":6,"printed_page":5,"text":" 9. FLPDMO ","text_end":8733,"text_sha256":"21b628067e0e0cae93f82779561150bde7ed78afb74d69e455ed1e3186bd3fa4","text_start":8722},"name":"FLPDMO","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Filing (Accounting) Period, Month:","publisher_position":9,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":793,"page_start":732,"pdf_page":14,"printed_page":13,"text":"10. F2441 Form 2441, Child Care Credit Qualified Individual: ","text_end":24168,"text_sha256":"b152765931e89ba3fb8448e01cfa70bbb8aa71b29e701486f78b2ce1ef17bee6","text_start":24107},"delivered_index":9,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":203,"page_start":193,"pdf_page":6,"printed_page":5,"text":"10. F2441 ","text_end":8722,"text_sha256":"01c52f4b2fe0d9d5165a17189ab4e5282b5b18d69ff2cad7e18d8c8a4f372c53","text_start":8712},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":203,"page_start":193,"pdf_page":6,"printed_page":5,"text":"10. F2441 ","text_end":8744,"text_sha256":"01c52f4b2fe0d9d5165a17189ab4e5282b5b18d69ff2cad7e18d8c8a4f372c53","text_start":8734},"name":"F2441","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 2441, Child Care Credit Qualified Individual:","publisher_position":10,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1016,"page_start":970,"pdf_page":14,"printed_page":13,"text":"11. F3800 Form 3800, General Business credit: ","text_end":24391,"text_sha256":"14e338b0441e0cb6c76e480bc1ba0b4e483c372dd5733f009574a1c8445d453d","text_start":24345},"delivered_index":10,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":214,"page_start":204,"pdf_page":6,"printed_page":5,"text":"11. F3800 ","text_end":8733,"text_sha256":"bb537dbb5a7aade7fc2c214184c697b080473fffe36d511244ebfffac3ba3778","text_start":8723},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":214,"page_start":204,"pdf_page":6,"printed_page":5,"text":"11. F3800 ","text_end":8755,"text_sha256":"bb537dbb5a7aade7fc2c214184c697b080473fffe36d511244ebfffac3ba3778","text_start":8745},"name":"F3800","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 3800, General Business credit:","publisher_position":11,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1252,"page_start":1201,"pdf_page":14,"printed_page":13,"text":" 12. F6251 Form 6251, Alternative Minimum Tax ","text_end":24627,"text_sha256":"83056f2f4b5d89d70af9812d722a74a903c11bc8171f4c288bd65372211ad4db","text_start":24576},"delivered_index":11,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":225,"page_start":215,"pdf_page":6,"printed_page":5,"text":"12. F6251 ","text_end":8744,"text_sha256":"07948b9784d84be63148935b012474f06a2f3c41b4fc9bef606b53b0600ef979","text_start":8734},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":225,"page_start":215,"pdf_page":6,"printed_page":5,"text":"12. F6251 ","text_end":8766,"text_sha256":"07948b9784d84be63148935b012474f06a2f3c41b4fc9bef606b53b0600ef979","text_start":8756},"name":"F6251","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 6251, Alternative Minimum Tax","publisher_position":12,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1489,"page_start":1433,"pdf_page":14,"printed_page":13,"text":"13. F8582 Form 8582, Passive Activity Loss Limitation: ","text_end":24864,"text_sha256":"f4fdb0f6d622efb5553da40f994ff677a5510b0cc317825441200248a6469f9e","text_start":24808},"delivered_index":12,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":237,"page_start":226,"pdf_page":6,"printed_page":5,"text":"13. F8582 ","text_end":8756,"text_sha256":"68a09f329357da168ad01490dbee20c1658f4f77e77bd10c7da496cc6b2fd5e5","text_start":8745},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":237,"page_start":226,"pdf_page":6,"printed_page":5,"text":"13. F8582 ","text_end":8778,"text_sha256":"68a09f329357da168ad01490dbee20c1658f4f77e77bd10c7da496cc6b2fd5e5","text_start":8767},"name":"F8582","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8582, Passive Activity Loss Limitation:","publisher_position":13,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":108,"page_start":53,"pdf_page":15,"printed_page":14,"text":"14. F8606 Form 8606, Nondeductible IRA Contributions: ","text_end":25160,"text_sha256":"dc8cb5749d573bb72e1c2b70d860134f5d815688cf28f497bb3511beba6f5cf8","text_start":25105},"delivered_index":13,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":248,"page_start":238,"pdf_page":6,"printed_page":5,"text":"14. F8606 ","text_end":8767,"text_sha256":"b0339bb6ca69ad94d567242f788c12cacef4870df79eba42789992fcc9aa4ef6","text_start":8757},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":248,"page_start":238,"pdf_page":6,"printed_page":5,"text":"14. F8606 ","text_end":8789,"text_sha256":"b0339bb6ca69ad94d567242f788c12cacef4870df79eba42789992fcc9aa4ef6","text_start":8779},"name":"F8606","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8606, Nondeductible IRA Contributions:","publisher_position":14,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":352,"page_start":290,"pdf_page":15,"printed_page":14,"text":"15. F8829 Form 8829, Expenses for business use of your Home: ","text_end":25404,"text_sha256":"9fd3c1b619f18c307090369c2e6ee66db0acc06fb5162eceb37d2bd5f4f3b2e0","text_start":25342},"delivered_index":14,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":259,"page_start":249,"pdf_page":6,"printed_page":5,"text":"15. F8829 ","text_end":8778,"text_sha256":"b0d9f7bfd115bf296b265117053937e4c8ce62bb4cdef9f8ab444767000b47ae","text_start":8768},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":259,"page_start":249,"pdf_page":6,"printed_page":5,"text":"15. F8829 ","text_end":8800,"text_sha256":"b0d9f7bfd115bf296b265117053937e4c8ce62bb4cdef9f8ab444767000b47ae","text_start":8790},"name":"F8829","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8829, Expenses for business use of your Home:","publisher_position":15,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":605,"page_start":534,"pdf_page":15,"printed_page":14,"text":"16. F8867 Form 8867, Paid Preparer’s Earned Income Credit Checklist ","text_end":25657,"text_sha256":"e1145d516646e92197631ec15785a0e97e8c2524882913c6ef91e728f48b17c3","text_start":25586},"delivered_index":15,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":260,"pdf_page":6,"printed_page":5,"text":"16. F8867 ","text_end":8789,"text_sha256":"61b533fadde404dbbeaed16ed60c54263faa32f9485e48cc4bb832d4ff20644e","text_start":8779},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":270,"page_start":260,"pdf_page":6,"printed_page":5,"text":"16. F8867 ","text_end":8811,"text_sha256":"61b533fadde404dbbeaed16ed60c54263faa32f9485e48cc4bb832d4ff20644e","text_start":8801},"name":"F8867","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8867, Paid Preparer’s Earned Income Credit Checklist","publisher_position":16,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":861,"page_start":793,"pdf_page":15,"printed_page":14,"text":"17. F8949 Form 8949, Sales and Other Dispositions of Capital Assets ","text_end":25913,"text_sha256":"e6f94330f8238e110113e378c222dd640b75cfede2fccd561f9851e9bc4f8d69","text_start":25845},"delivered_index":16,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":281,"page_start":271,"pdf_page":6,"printed_page":5,"text":"17. F8949 ","text_end":8800,"text_sha256":"ee402923ac3783fddb2f269f3a7910872dc6ed0fff422e5909e5fe5c704f4790","text_start":8790},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":281,"page_start":271,"pdf_page":6,"printed_page":5,"text":"17. F8949 ","text_end":8822,"text_sha256":"ee402923ac3783fddb2f269f3a7910872dc6ed0fff422e5909e5fe5c704f4790","text_start":8812},"name":"F8949","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8949, Sales and Other Dispositions of Capital Assets","publisher_position":17,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1094,"page_start":1049,"pdf_page":15,"printed_page":14,"text":"18. F8959 Form 8959, Additional Medicare Tax ","text_end":26146,"text_sha256":"568a604f5d031c5444be004ad8b02bb9f0946ad6f82b7673c05f5f8c953637ba","text_start":26101},"delivered_index":17,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":292,"page_start":282,"pdf_page":6,"printed_page":5,"text":"18. F8959 ","text_end":8811,"text_sha256":"3e61f77e7e3bdaae2ea51f63f7db2c68e0baeff67d48380adbbb3d504e485729","text_start":8801},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":292,"page_start":282,"pdf_page":6,"printed_page":5,"text":"18. F8959 ","text_end":8833,"text_sha256":"3e61f77e7e3bdaae2ea51f63f7db2c68e0baeff67d48380adbbb3d504e485729","text_start":8823},"name":"F8959","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8959, Additional Medicare Tax","publisher_position":18,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1330,"page_start":1282,"pdf_page":15,"printed_page":14,"text":"19. F8960 Form 8960, Net Investment Income Tax ","text_end":26382,"text_sha256":"bc4fce44984c18f262036bbe816dda8b4ef1df87b54fd8cb5ec175846ede6c70","text_start":26334},"delivered_index":18,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":303,"page_start":293,"pdf_page":6,"printed_page":5,"text":"19. F8960 ","text_end":8822,"text_sha256":"ee66926934ed3a726a77dc689b9b6ab98ca6aa9c05c13d438c723ba2119ee189","text_start":8812},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":303,"page_start":293,"pdf_page":6,"printed_page":5,"text":"19. F8960 ","text_end":8844,"text_sha256":"ee66926934ed3a726a77dc689b9b6ab98ca6aa9c05c13d438c723ba2119ee189","text_start":8834},"name":"F8960","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8960, Net Investment Income Tax","publisher_position":19,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1559,"page_start":1518,"pdf_page":15,"printed_page":14,"text":"20. F8962 Form 8962, Premium Tax Credit ","text_end":26611,"text_sha256":"961f04e8544622ebc45ffd98d69321f5bef6ee2fecc12a29e0da6e685173fe60","text_start":26570},"delivered_index":19,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":314,"page_start":304,"pdf_page":6,"printed_page":5,"text":"20. F8962 ","text_end":8833,"text_sha256":"2f63031c7b761dcc11e5828103e54bff5dfe3dc61cfb9bc882e41d4099db5a8e","text_start":8823},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":314,"page_start":304,"pdf_page":6,"printed_page":5,"text":"20. F8962 ","text_end":8855,"text_sha256":"2f63031c7b761dcc11e5828103e54bff5dfe3dc61cfb9bc882e41d4099db5a8e","text_start":8845},"name":"F8962","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8962, Premium Tax Credit","publisher_position":20,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":104,"page_start":55,"pdf_page":16,"printed_page":15,"text":"21. F8965 Form 8965, Health Coverage Exemptions ","text_end":26905,"text_sha256":"ae896abb0ad3155680f67895ba158bfa6231ca868f603f0766874e0b82dd5ef6","text_start":26856},"delivered_index":20,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":325,"page_start":315,"pdf_page":6,"printed_page":5,"text":"21. F8965 ","text_end":8844,"text_sha256":"73855028c4832b198c2180bdf2470509b5d4651da2c9cb0f7e59b8d46aed2503","text_start":8834},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":325,"page_start":315,"pdf_page":6,"printed_page":5,"text":"21. F8965 ","text_end":8866,"text_sha256":"73855028c4832b198c2180bdf2470509b5d4651da2c9cb0f7e59b8d46aed2503","text_start":8856},"name":"F8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form 8965, Health Coverage Exemptions","publisher_position":21,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":373,"page_start":292,"pdf_page":16,"printed_page":15,"text":"22. IE Itemized Deductions Election Indicator (Taxpayer elects to claim itemized ","text_end":27174,"text_sha256":"89bd2433f9d9626758da01462b21f40b8b2bd4718aa51d4a14e4f6e6b2752ba9","text_start":27093},"delivered_index":21,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":334,"page_start":326,"pdf_page":6,"printed_page":5,"text":"22. IE ","text_end":8853,"text_sha256":"a057fca32e6df91862eb45cced31eb61b4e244507f8ad327a386d9c49f1707e0","text_start":8845},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":334,"page_start":326,"pdf_page":6,"printed_page":5,"text":"22. IE ","text_end":8875,"text_sha256":"a057fca32e6df91862eb45cced31eb61b4e244507f8ad327a386d9c49f1707e0","text_start":8867},"name":"IE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Itemized Deductions Election Indicator (Taxpayer elects to claim itemized","publisher_position":22,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":703,"page_start":669,"pdf_page":16,"printed_page":15,"text":"23. MARS Marital (Filing) Status: ","text_end":27504,"text_sha256":"f6e3a499c660e5d7a98a143fe4112646260b5c1f4b044f1b9cbb579526a1e6ff","text_start":27470},"delivered_index":22,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":344,"page_start":335,"pdf_page":6,"printed_page":5,"text":"23. MARS ","text_end":8863,"text_sha256":"6ab746b927b75d8ca8116a16cf94f76d543c1a6b07d870bec86a67c3cbc6fa3a","text_start":8854},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":344,"page_start":335,"pdf_page":6,"printed_page":5,"text":"23. MARS ","text_end":8885,"text_sha256":"6ab746b927b75d8ca8116a16cf94f76d543c1a6b07d870bec86a67c3cbc6fa3a","text_start":8876},"name":"MARS","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Marital (Filing) Status:","publisher_position":23,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1480,"page_start":1403,"pdf_page":16,"printed_page":15,"text":"24. MIDR Married Filing Separately Itemized Deductions Requirement Indicator ","text_end":28281,"text_sha256":"54f50b4c999632fb255c8e4e089bac30d227f5c5264a1843b4dfaba4a0b2818c","text_start":28204},"delivered_index":23,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":354,"page_start":345,"pdf_page":6,"printed_page":5,"text":"24. MIDR ","text_end":8873,"text_sha256":"715abc6dfd87c8db8911ce5d31132613e16507d82be608ce30d84d3bd488792c","text_start":8864},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":354,"page_start":345,"pdf_page":6,"printed_page":5,"text":"24. MIDR ","text_end":8895,"text_sha256":"715abc6dfd87c8db8911ce5d31132613e16507d82be608ce30d84d3bd488792c","text_start":8886},"name":"MIDR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Married Filing Separately Itemized Deductions Requirement Indicator","publisher_position":24,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1726,"page_start":1669,"pdf_page":16,"printed_page":15,"text":"25. N24 Number of Children for Child Tax Credit ","text_end":28527,"text_sha256":"d7fa0439ae8cc3bca825a9e9cd326a09f2d63d5a14514af5daed980f2f1c4de3","text_start":28470},"delivered_index":24,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":362,"page_start":355,"pdf_page":6,"printed_page":5,"text":"25. N24","text_end":8881,"text_sha256":"64ee893dbe036d6f8e92fe63eb8e981dd69c7dfc4673a298c463c9b5bfd10491","text_start":8874},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":362,"page_start":355,"pdf_page":6,"printed_page":5,"text":"25. N24","text_end":8903,"text_sha256":"64ee893dbe036d6f8e92fe63eb8e981dd69c7dfc4673a298c463c9b5bfd10491","text_start":8896},"name":"N24","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Children for Child Tax Credit","publisher_position":25,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1851,"page_start":1782,"pdf_page":16,"printed_page":15,"text":"26. N25 Number of Qualified Students for Lifetime Learning Credit ","text_end":28652,"text_sha256":"71b80f1051188dd397ca82a77fd2ea48d31ed257cd027dac4502c2787edd1a9b","text_start":28583},"delivered_index":25,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":375,"page_start":366,"pdf_page":6,"printed_page":5,"text":"26. N251 ","text_end":8894,"text_sha256":"e67dbf65f66e23d7eeb486fcc555a5f28db5b6825f70dc148518fc19e2f443e4","text_start":8885},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":375,"page_start":366,"pdf_page":6,"printed_page":5,"text":"26. N251 ","text_end":8916,"text_sha256":"e67dbf65f66e23d7eeb486fcc555a5f28db5b6825f70dc148518fc19e2f443e4","text_start":8907},"name":"N25","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Qualified Students for Lifetime Learning Credit","publisher_position":26,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":134,"page_start":57,"pdf_page":17,"printed_page":16,"text":"27. N30 Number of Qualified Students for American Opportunity Credit ","text_end":28842,"text_sha256":"c42aead4d3c5742c32259a4ac711e3656e85576230e16f9006b6bcdb23d08634","text_start":28765},"delivered_index":26,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":386,"page_start":376,"pdf_page":6,"printed_page":5,"text":"27. N301 ","text_end":8905,"text_sha256":"271fb00168a07968994777a34e3403bc21928b7dc3ee7ac9bd5aee54c8c57c92","text_start":8895},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":386,"page_start":376,"pdf_page":6,"printed_page":5,"text":"27. N301 ","text_end":8927,"text_sha256":"271fb00168a07968994777a34e3403bc21928b7dc3ee7ac9bd5aee54c8c57c92","text_start":8917},"name":"N30","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Qualified Students for American Opportunity Credit","publisher_position":27,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":216,"page_start":193,"pdf_page":17,"printed_page":16,"text":"28. PREP Tax Preparer: ","text_end":28924,"text_sha256":"8a34bb702a90c77cad2ea1dd34526f242dd73867c82b5e0c651fc91ca3ed0fed","text_start":28901},"delivered_index":27,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":397,"page_start":387,"pdf_page":6,"printed_page":5,"text":"28. PREP ","text_end":8916,"text_sha256":"d1c5c0755db5e7466e609a50cd025069ef2545510b72edf16083dd3bc731f81f","text_start":8906},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":397,"page_start":387,"pdf_page":6,"printed_page":5,"text":"28. PREP ","text_end":8938,"text_sha256":"d1c5c0755db5e7466e609a50cd025069ef2545510b72edf16083dd3bc731f81f","text_start":8928},"name":"PREP","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Tax Preparer:","publisher_position":28,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":816,"page_start":744,"pdf_page":17,"printed_page":16,"text":"29. PREMNTHS Number of months enrolled in Health Insurance Marketplace: ","text_end":29524,"text_sha256":"32994b013e555ac88bc339cd4170dafe046092d35360e0e1b0e528a29aed8c93","text_start":29452},"delivered_index":28,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":411,"page_start":398,"pdf_page":6,"printed_page":5,"text":"29. PREMNTHS ","text_end":8930,"text_sha256":"73de2a8a9136440ff88b507a4e71d51c50ff10be496b23b168c5218d7609500d","text_start":8917},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":411,"page_start":398,"pdf_page":6,"printed_page":5,"text":"29. PREMNTHS ","text_end":8952,"text_sha256":"73de2a8a9136440ff88b507a4e71d51c50ff10be496b23b168c5218d7609500d","text_start":8939},"name":"PREMNTHS","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of months enrolled in Health Insurance Marketplace:","publisher_position":29,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":905,"page_start":873,"pdf_page":17,"printed_page":16,"text":"30. SCHB Schedule B Indicator: ","text_end":29613,"text_sha256":"3480701404d3bd7153400771e38fbfdbf2ef3192d494f5ef90d6fa48ae4c3ac2","text_start":29581},"delivered_index":29,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":421,"page_start":412,"pdf_page":6,"printed_page":5,"text":"30. SCHB ","text_end":8940,"text_sha256":"81699bc01628f8f813147f62b9ee4b0de71234bcdd3cae4b5b903119ffd1573f","text_start":8931},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":421,"page_start":412,"pdf_page":6,"printed_page":5,"text":"30. SCHB ","text_end":8962,"text_sha256":"81699bc01628f8f813147f62b9ee4b0de71234bcdd3cae4b5b903119ffd1573f","text_start":8953},"name":"SCHB","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule B Indicator:","publisher_position":30,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1134,"page_start":1097,"pdf_page":17,"printed_page":16,"text":"31. SCHCF Schedule C or F Indicator: ","text_end":29842,"text_sha256":"c6b2101f4c86a5908e5952f55a9cb68f4ca3bb6b5a95b22334e868458ee60023","text_start":29805},"delivered_index":30,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":432,"page_start":422,"pdf_page":6,"printed_page":5,"text":"31. SCHCF ","text_end":8951,"text_sha256":"c8d3cc667c96dd120043a8f2800ee14845c10eef8e9e444ec3c7423935891c4a","text_start":8941},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":432,"page_start":422,"pdf_page":6,"printed_page":5,"text":"31. SCHCF ","text_end":8973,"text_sha256":"c8d3cc667c96dd120043a8f2800ee14845c10eef8e9e444ec3c7423935891c4a","text_start":8963},"name":"SCHCF","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule C or F Indicator:","publisher_position":31,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1590,"page_start":1559,"pdf_page":17,"printed_page":16,"text":"32. SCHE Schedule E Indicator: ","text_end":30298,"text_sha256":"919c31e69c49c54b2210be9213c7e1bc005c84e1dd9d035df2914d6a1f54475c","text_start":30267},"delivered_index":31,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":442,"page_start":433,"pdf_page":6,"printed_page":5,"text":"32. SCHE ","text_end":8961,"text_sha256":"a0c8f93f41c9d9c5e2044a9f75b05f73d114be91707f44c59f9f1f825256fa3f","text_start":8952},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":442,"page_start":433,"pdf_page":6,"printed_page":5,"text":"32. SCHE ","text_end":8983,"text_sha256":"a0c8f93f41c9d9c5e2044a9f75b05f73d114be91707f44c59f9f1f825256fa3f","text_start":8974},"name":"SCHE","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Schedule E Indicator:","publisher_position":32,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":79,"page_start":53,"pdf_page":18,"printed_page":17,"text":"33. TFORM Form of Return: ","text_end":30562,"text_sha256":"3f48ca202e984c862e8b008783e2f9b0dd2c765af5d1d4d0cc0826e3080bd15d","text_start":30536},"delivered_index":32,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":453,"page_start":443,"pdf_page":6,"printed_page":5,"text":"33. TFORM ","text_end":8972,"text_sha256":"43c0167976d05afcca47000325ec120354c485d1b028898e6eaf970f792a9cc8","text_start":8962},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":453,"page_start":443,"pdf_page":6,"printed_page":5,"text":"33. TFORM ","text_end":8994,"text_sha256":"43c0167976d05afcca47000325ec120354c485d1b028898e6eaf970f792a9cc8","text_start":8984},"name":"TFORM","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Form of Return:","publisher_position":33,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":432,"page_start":411,"pdf_page":18,"printed_page":17,"text":"34. TXST TAX STATUS: ","text_end":30915,"text_sha256":"1cc17672f79629fcb8faf909e2b10b4bd4bb21a5b183477d16f1f646eca8266b","text_start":30894},"delivered_index":33,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":463,"page_start":454,"pdf_page":6,"printed_page":5,"text":"34. TXST ","text_end":8982,"text_sha256":"a66590af6e4892e80d9a4cf549a5d06848b59fed91d548988c5b86f63237778e","text_start":8973},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":463,"page_start":454,"pdf_page":6,"printed_page":5,"text":"34. TXST ","text_end":9004,"text_sha256":"a66590af6e4892e80d9a4cf549a5d06848b59fed91d548988c5b86f63237778e","text_start":8995},"name":"TXST","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"TAX STATUS:","publisher_position":34,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1724,"page_start":1685,"pdf_page":18,"printed_page":17,"text":"35. XFPT Primary Taxpayer Exemption: ","text_end":32207,"text_sha256":"71b564ede2584f8fbd3452d53da0111f388951420b3e1346de10ff8bc68d3d3a","text_start":32168},"delivered_index":34,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":473,"page_start":464,"pdf_page":6,"printed_page":5,"text":"35. XFPT ","text_end":8992,"text_sha256":"d80624a20d2b2d40220541a1ee9a182ba47d57ff21453ca0a4e7ce9c4f574269","text_start":8983},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":473,"page_start":464,"pdf_page":6,"printed_page":5,"text":"35. XFPT ","text_end":9014,"text_sha256":"d80624a20d2b2d40220541a1ee9a182ba47d57ff21453ca0a4e7ce9c4f574269","text_start":9005},"name":"XFPT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Primary Taxpayer Exemption:","publisher_position":35,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1944,"page_start":1903,"pdf_page":18,"printed_page":17,"text":"36. XFST Secondary Taxpayer Exemption: ","text_end":32427,"text_sha256":"694ace59a6e0e0e1af6c5ead4b5bae1d33a608bb072d25e269660cf4c20d443c","text_start":32386},"delivered_index":35,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":483,"page_start":474,"pdf_page":6,"printed_page":5,"text":"36. XFST ","text_end":9002,"text_sha256":"01dc6c79a08a86010c3a477707ce8d180aa7e9f06850e41c97ae15df54bda59b","text_start":8993},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":483,"page_start":474,"pdf_page":6,"printed_page":5,"text":"36. XFST ","text_end":9024,"text_sha256":"01dc6c79a08a86010c3a477707ce8d180aa7e9f06850e41c97ae15df54bda59b","text_start":9015},"name":"XFST","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Secondary Taxpayer Exemption:","publisher_position":36,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":2175,"page_start":2125,"pdf_page":18,"printed_page":17,"text":"37. XOCAH Exemptions for Children Living at Home: ","text_end":32658,"text_sha256":"183eb94b7c211eb2d7c65a2a299f485007bd23be5d922f6a92c31558e988f106","text_start":32608},"delivered_index":36,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":495,"page_start":484,"pdf_page":6,"printed_page":5,"text":"37. XOCAH1 ","text_end":9014,"text_sha256":"74da3cf13cd316e9f3e5ef5479cfc61384d91777a65b487867565257ac96fc66","text_start":9003},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":null,"page_end":493,"page_start":484,"pdf_page":6,"printed_page":5,"text":"37. XOCAH","text_end":9034,"text_sha256":"9f339784357264867577084eeff2324f32af7e6a78bfbb835195eb0156b92ced","text_start":9025},"name":"XOCAH","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Children Living at Home:","publisher_position":37,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":114,"page_start":55,"pdf_page":19,"printed_page":18,"text":"38. XOCAWH Exemptions for Children Living Away from Home: ","text_end":32976,"text_sha256":"6eb94eb248ac20032207633ec9fef8dede4d355e1ad0ce109cbd1b3688e9a96f","text_start":32917},"delivered_index":37,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":508,"page_start":496,"pdf_page":6,"printed_page":5,"text":"38. XOCAWH1 ","text_end":9027,"text_sha256":"70998a2ceb5b54883dc74cdfa3556083f867466ef55d4b278ff2ff8fb61a9f38","text_start":9015},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":509,"page_start":497,"pdf_page":6,"printed_page":5,"text":"38. XOCAWH1 ","text_end":9050,"text_sha256":"70998a2ceb5b54883dc74cdfa3556083f867466ef55d4b278ff2ff8fb61a9f38","text_start":9038},"name":"XOCAWH","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Children Living Away from Home:","publisher_position":38,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":378,"page_start":333,"pdf_page":19,"printed_page":18,"text":"39. XOODEP Exemptions for Other Dependents: ","text_end":33240,"text_sha256":"8428645125fbf4e150038bebb8d4132fafd9d844954bc3e424715f3d3778fff6","text_start":33195},"delivered_index":38,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":521,"page_start":509,"pdf_page":6,"printed_page":5,"text":"39. XOODEP1 ","text_end":9040,"text_sha256":"bc8e1545ee428c93ea46ef63d2043dcbbc3632a21be60b9da409dbed80b06c1d","text_start":9028},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":522,"page_start":510,"pdf_page":6,"printed_page":5,"text":"39. XOODEP1 ","text_end":9063,"text_sha256":"bc8e1545ee428c93ea46ef63d2043dcbbc3632a21be60b9da409dbed80b06c1d","text_start":9051},"name":"XOODEP","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Other Dependents:","publisher_position":39,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":680,"page_start":613,"pdf_page":19,"printed_page":18,"text":"40. XOPAR Exemptions for Parents Living at Home or Away from Home: ","text_end":33542,"text_sha256":"60838bc09ccfa6c113ba042622d48900789c3c806d7b5b0b5c24bf33b274bcf5","text_start":33475},"delivered_index":39,"layout_row":{"booklet":"main_2022","footnote_markers":"1","page_end":533,"page_start":522,"pdf_page":6,"printed_page":5,"text":"40. XOPAR1 ","text_end":9052,"text_sha256":"2134b12224986f1b7a019090eff091f29c2707dd253d98d17d50bb3c2fef28cd","text_start":9041},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1","page_end":534,"page_start":523,"pdf_page":6,"printed_page":5,"text":"40. XOPAR1 ","text_end":9075,"text_sha256":"2134b12224986f1b7a019090eff091f29c2707dd253d98d17d50bb3c2fef28cd","text_start":9064},"name":"XOPAR","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Exemptions for Parents Living at Home or Away from Home:","publisher_position":40,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":955,"page_start":927,"pdf_page":19,"printed_page":18,"text":"41. XTOT Total Exemptions: ","text_end":33817,"text_sha256":"161e71174e6137bfe5573edcc93198e876df78d5a4f1ea0a4e0a59201519de39","text_start":33789},"delivered_index":40,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":546,"page_start":534,"pdf_page":6,"printed_page":5,"text":"41. XTOT1,2 ","text_end":9065,"text_sha256":"b4edcb7ff97942f842ab84fdf1655dbbb9b6db60529a8b7cc35a83c74b086890","text_start":9053},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":547,"page_start":535,"pdf_page":6,"printed_page":5,"text":"41. XTOT1,2 ","text_end":9088,"text_sha256":"b4edcb7ff97942f842ab84fdf1655dbbb9b6db60529a8b7cc35a83c74b086890","text_start":9076},"name":"XTOT","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Total Exemptions:","publisher_position":41,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1195,"page_start":1145,"pdf_page":19,"printed_page":18,"text":"42. XTOT8962 Number of Exemptions on Form 8962: ","text_end":34057,"text_sha256":"6e46fb37b1b3c3b9b1908649a4a0078c12a5712b627ad22942e14a0a8c165331","text_start":34007},"delivered_index":41,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":563,"page_start":547,"pdf_page":6,"printed_page":5,"text":"42. XTOT89621,2 ","text_end":9082,"text_sha256":"4de83366114efa6b6f2cb99bb131fb8dbebb8462de9010a91c17d6250848c87b","text_start":9066},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":564,"page_start":548,"pdf_page":6,"printed_page":5,"text":"42. XTOT89621,2 ","text_end":9105,"text_sha256":"4de83366114efa6b6f2cb99bb131fb8dbebb8462de9010a91c17d6250848c87b","text_start":9089},"name":"XTOT8962","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Exemptions on Form 8962:","publisher_position":42,"publisher_signed_marker":false},{"admission":"not_selected","code_definition_row":{"booklet":"main_2022","page_end":1351,"page_start":1301,"pdf_page":19,"printed_page":18,"text":"43. XTOT8965 Number of Exemptions on Form 8965: ","text_end":34213,"text_sha256":"f774c483511496dacf032950cc61f8a8dd14a92fe65ae958eccc3c674776f0de","text_start":34163},"delivered_index":42,"layout_row":{"booklet":"main_2022","footnote_markers":"1,2","page_end":580,"page_start":564,"pdf_page":6,"printed_page":5,"text":"43. XTOT89651,2 ","text_end":9099,"text_sha256":"59ed964dd25a600a3006f29fd696db60520b9091defce95a7df7c0f9676b12b5","text_start":9083},"layout_row_corroboration":{"booklet":"main_2023","footnote_markers":"1,2","page_end":581,"page_start":565,"pdf_page":6,"printed_page":5,"text":"43. XTOT89651,2 ","text_end":9122,"text_sha256":"59ed964dd25a600a3006f29fd696db60520b9091defce95a7df7c0f9676b12b5","text_start":9106},"name":"XTOT8965","not_selected_reason":"code_field_not_an_amount","publisher_group":"code_field","publisher_label":"Number of Exemptions on Form 8965:","publisher_position":43,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":43,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9732,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9697},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9755,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9720},"name":"E00200","publisher_group":"amount_or_large_code","publisher_label":"Salaries and wages","publisher_position":1,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":44,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9759,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9733},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9782,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9756},"name":"E00300","publisher_group":"amount_or_large_code","publisher_label":"Interest received","publisher_position":2,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":45,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9795,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9760},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9818,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9783},"name":"E00400","publisher_group":"amount_or_large_code","publisher_label":"Tax-exempt interest income","publisher_position":3,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":46,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9830,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9796},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9853,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9819},"name":"E00600","publisher_group":"amount_or_large_code","publisher_label":"Dividends included in AGI","publisher_position":4,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":47,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9859,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9831},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9882,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9854},"name":"E00650","publisher_group":"amount_or_large_code","publisher_label":"Qualified Dividends","publisher_position":5,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":48,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":333,"page_start":300,"pdf_page":7,"printed_page":6,"text":"6 E00700 State income tax refunds","text_end":9893,"text_sha256":"f2b052b44e74bdf5f4870a1e46127391246bd3ee2edabe260c3388208dae7ab6","text_start":9860},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":333,"page_start":300,"pdf_page":7,"printed_page":6,"text":"6 E00700 State income tax refunds","text_end":9916,"text_sha256":"f2b052b44e74bdf5f4870a1e46127391246bd3ee2edabe260c3388208dae7ab6","text_start":9883},"name":"E00700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"State income tax refunds","publisher_position":6,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":49,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":360,"page_start":334,"pdf_page":7,"printed_page":6,"text":"7 E00800 Alimony received ","text_end":9920,"text_sha256":"95a7d728cb5f15e14fc8a10efecf4caee43cc2534ee505c5dab4f7d20faea4ea","text_start":9894},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":360,"page_start":334,"pdf_page":7,"printed_page":6,"text":"7 E00800 Alimony received ","text_end":9943,"text_sha256":"95a7d728cb5f15e14fc8a10efecf4caee43cc2534ee505c5dab4f7d20faea4ea","text_start":9917},"name":"E00800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alimony received","publisher_position":7,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":50,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":9987,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9921},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":10010,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9944},"name":"E00900","publisher_group":"amount_or_large_code","publisher_label":"Business or profession (Schedule C) net profit/loss (+/-)","publisher_position":8,"publisher_signed_marker":true},{"admission":"monetary_source_projection","delivered_index":51,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10027,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":9988},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10050,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":10011},"name":"E01000","publisher_group":"amount_or_large_code","publisher_label":"Net capital gain or loss (+/-)","publisher_position":9,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":52,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":526,"page_start":468,"pdf_page":7,"printed_page":6,"text":"10 E01100 Capital gain distributions reported on Form 1040","text_end":10086,"text_sha256":"c87bd839a618d48c0471a13631800dd39d82a9cd514d60bb6e693c6c1b7d3b90","text_start":10028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":526,"page_start":468,"pdf_page":7,"printed_page":6,"text":"10 E01100 Capital gain distributions reported on Form 1040","text_end":10109,"text_sha256":"c87bd839a618d48c0471a13631800dd39d82a9cd514d60bb6e693c6c1b7d3b90","text_start":10051},"name":"E01100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Capital gain distributions reported on Form 1040","publisher_position":10,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":53,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":565,"page_start":527,"pdf_page":7,"printed_page":6,"text":"11 E01200 Other gains (or loss) (+/-)","text_end":10125,"text_sha256":"642ca8e20a5621d511d4b6a77204b7f45ceab14cf33c85cce7ea54874daa28d3","text_start":10087},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":565,"page_start":527,"pdf_page":7,"printed_page":6,"text":"11 E01200 Other gains (or loss) (+/-)","text_end":10148,"text_sha256":"642ca8e20a5621d511d4b6a77204b7f45ceab14cf33c85cce7ea54874daa28d3","text_start":10110},"name":"E01200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other gains (or loss) (+/-)","publisher_position":11,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":54,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":600,"page_start":566,"pdf_page":7,"printed_page":6,"text":"12 E01400 Taxable IRA distribution","text_end":10160,"text_sha256":"310cb5b974160d517c423872da0cc305c6f9ce0beadc81aeca21971240b310b7","text_start":10126},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":600,"page_start":566,"pdf_page":7,"printed_page":6,"text":"12 E01400 Taxable IRA distribution","text_end":10183,"text_sha256":"310cb5b974160d517c423872da0cc305c6f9ce0beadc81aeca21971240b310b7","text_start":10149},"name":"E01400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Taxable IRA distribution","publisher_position":12,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":55,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10208,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10231,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10184},"name":"E01500","publisher_group":"amount_or_large_code","publisher_label":"Total pensions and annuities received","publisher_position":13,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":56,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10257,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10209},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10280,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10232},"name":"E01700","publisher_group":"amount_or_large_code","publisher_label":"Pensions and annuities included in AGI","publisher_position":14,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":57,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":743,"page_start":698,"pdf_page":7,"printed_page":6,"text":"15 E02000 Schedule E net income or loss (+/-)","text_end":10303,"text_sha256":"529435b1456ddff50c1d6d5cb2a84462492cbe3bcb2e87b571711a5c9cfe17f0","text_start":10258},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":743,"page_start":698,"pdf_page":7,"printed_page":6,"text":"15 E02000 Schedule E net income or loss (+/-)","text_end":10326,"text_sha256":"529435b1456ddff50c1d6d5cb2a84462492cbe3bcb2e87b571711a5c9cfe17f0","text_start":10281},"name":"E02000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule E net income or loss (+/-)","publisher_position":15,"publisher_signed_marker":true},{"admission":"monetary_source_projection","delivered_index":58,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10346,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10304},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10369,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10327},"name":"E02100","publisher_group":"amount_or_large_code","publisher_label":"Schedule F net profit/loss (+/-)","publisher_position":16,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":59,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":829,"page_start":787,"pdf_page":7,"printed_page":6,"text":"17 E02300 Unemployment compensation in AGI","text_end":10389,"text_sha256":"49fa031dafe5db53dac70ad0a8551a55d659d559c106f315d005fc8331f25199","text_start":10347},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":829,"page_start":787,"pdf_page":7,"printed_page":6,"text":"17 E02300 Unemployment compensation in AGI","text_end":10412,"text_sha256":"49fa031dafe5db53dac70ad0a8551a55d659d559c106f315d005fc8331f25199","text_start":10370},"name":"E02300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unemployment compensation in AGI","publisher_position":17,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":60,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":7,"printed_page":6,"text":"18 E02400 Gross Social Security benefits","text_end":10430,"text_sha256":"93ac4847542fa632ac7da30fa7a66bcc905a0a9f440f0eebb343a0fe20ace190","text_start":10390},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":7,"printed_page":6,"text":"18 E02400 Gross Social Security benefits","text_end":10453,"text_sha256":"93ac4847542fa632ac7da30fa7a66bcc905a0a9f440f0eebb343a0fe20ace190","text_start":10413},"name":"E02400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Gross Social Security benefits","publisher_position":18,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":61,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":912,"page_start":871,"pdf_page":7,"printed_page":6,"text":"19 E02500 Social Security benefits in AGI","text_end":10472,"text_sha256":"b2e9a9fd99b7b17ca804b24a09c86a6d62570404ec46b49900d0594139775f13","text_start":10431},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":912,"page_start":871,"pdf_page":7,"printed_page":6,"text":"19 E02500 Social Security benefits in AGI","text_end":10495,"text_sha256":"b2e9a9fd99b7b17ca804b24a09c86a6d62570404ec46b49900d0594139775f13","text_start":10454},"name":"E02500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Social Security benefits in AGI","publisher_position":19,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":62,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":994,"page_start":913,"pdf_page":7,"printed_page":6,"text":"Statutory 20 E03150 Total deductible individual retirement account (IRA) payments","text_end":10554,"text_sha256":"3ed3ece015295a7a5b0b3046629af4c012b0eeb5f6066ea87c963ae66e9b9c81","text_start":10473},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":994,"page_start":913,"pdf_page":7,"printed_page":6,"text":"Statutory 20 E03150 Total deductible individual retirement account (IRA) payments","text_end":10577,"text_sha256":"3ed3ece015295a7a5b0b3046629af4c012b0eeb5f6066ea87c963ae66e9b9c81","text_start":10496},"name":"E03150","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total deductible individual retirement account (IRA) payments","publisher_position":20,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":63,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1050,"page_start":995,"pdf_page":7,"printed_page":6,"text":"Adjustments 21 E03210 Student Loan Interest Deduction","text_end":10610,"text_sha256":"41e03c2ee377762c689e57a03f67ba4e2e031f3688f2ac02de62da16666b3885","text_start":10555},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1050,"page_start":995,"pdf_page":7,"printed_page":6,"text":"Adjustments 21 E03210 Student Loan Interest Deduction","text_end":10633,"text_sha256":"41e03c2ee377762c689e57a03f67ba4e2e031f3688f2ac02de62da16666b3885","text_start":10578},"name":"E03210","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Student Loan Interest Deduction","publisher_position":21,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":64,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1078,"page_start":1051,"pdf_page":7,"printed_page":6,"text":"22 E03220 Educator Expenses","text_end":10638,"text_sha256":"cb43a76c4c81b7ef03bf093a6cf492bb595b81ecf74804838adcf955aa699afa","text_start":10611},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1078,"page_start":1051,"pdf_page":7,"printed_page":6,"text":"22 E03220 Educator Expenses","text_end":10661,"text_sha256":"cb43a76c4c81b7ef03bf093a6cf492bb595b81ecf74804838adcf955aa699afa","text_start":10634},"name":"E03220","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Educator Expenses","publisher_position":22,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":65,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1115,"page_start":1079,"pdf_page":7,"printed_page":6,"text":"23 E03230 Tuition and Fees Deduction","text_end":10675,"text_sha256":"af3ba4332089496cb164d163ee3428b90f57b8301a6b2cb6af793224d24d0f0e","text_start":10639},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1115,"page_start":1079,"pdf_page":7,"printed_page":6,"text":"23 E03230 Tuition and Fees Deduction","text_end":10698,"text_sha256":"af3ba4332089496cb164d163ee3428b90f57b8301a6b2cb6af793224d24d0f0e","text_start":10662},"name":"E03230","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tuition and Fees Deduction","publisher_position":23,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":66,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1159,"page_start":1116,"pdf_page":7,"printed_page":6,"text":"24 E03260 Deduction for self-employment tax","text_end":10719,"text_sha256":"1697766007704455dbf9bef8986c2bf0975e5a71a0c87e9bd1e34c9b698db1e6","text_start":10676},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1159,"page_start":1116,"pdf_page":7,"printed_page":6,"text":"24 E03260 Deduction for self-employment tax","text_end":10742,"text_sha256":"1697766007704455dbf9bef8986c2bf0975e5a71a0c87e9bd1e34c9b698db1e6","text_start":10699},"name":"E03260","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Deduction for self-employment tax","publisher_position":24,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":67,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1210,"page_start":1160,"pdf_page":7,"printed_page":6,"text":"25 E03270 Self-employed health insurance deduction","text_end":10770,"text_sha256":"cf87ff838847798de4b572b71e86d7da07ff3090a60bf6fa4e616452d17412f8","text_start":10720},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1210,"page_start":1160,"pdf_page":7,"printed_page":6,"text":"25 E03270 Self-employed health insurance deduction","text_end":10793,"text_sha256":"cf87ff838847798de4b572b71e86d7da07ff3090a60bf6fa4e616452d17412f8","text_start":10743},"name":"E03270","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employed health insurance deduction","publisher_position":25,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":68,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1261,"page_start":1211,"pdf_page":7,"printed_page":6,"text":"26 E03240 Domestic Production Activities deduction","text_end":10821,"text_sha256":"0e8f3a8c01b8b891ffac6474d8cdd32ac5d56aecf8a7caca58959a71c05d0989","text_start":10771},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1261,"page_start":1211,"pdf_page":7,"printed_page":6,"text":"26 E03240 Domestic Production Activities deduction","text_end":10844,"text_sha256":"0e8f3a8c01b8b891ffac6474d8cdd32ac5d56aecf8a7caca58959a71c05d0989","text_start":10794},"name":"E03240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Domestic Production Activities deduction","publisher_position":26,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":69,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1304,"page_start":1262,"pdf_page":7,"printed_page":6,"text":"27 E03290 Health Savings Account deduction","text_end":10864,"text_sha256":"12be5316231352f334c79873cf0ebef86e8fdcc81687b2656ba7866290550838","text_start":10822},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1304,"page_start":1262,"pdf_page":7,"printed_page":6,"text":"27 E03290 Health Savings Account deduction","text_end":10887,"text_sha256":"12be5316231352f334c79873cf0ebef86e8fdcc81687b2656ba7866290550838","text_start":10845},"name":"E03290","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Health Savings Account deduction","publisher_position":27,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":70,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1342,"page_start":1305,"pdf_page":7,"printed_page":6,"text":"28 E03300 Payments to KEOGH accounts ","text_end":10902,"text_sha256":"7dae2808be86e6f0bbf5855f0fee2a388e485d2b1a99dcb6406c859b3f8536ad","text_start":10865},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1342,"page_start":1305,"pdf_page":7,"printed_page":6,"text":"28 E03300 Payments to KEOGH accounts ","text_end":10925,"text_sha256":"7dae2808be86e6f0bbf5855f0fee2a388e485d2b1a99dcb6406c859b3f8536ad","text_start":10888},"name":"E03300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Payments to KEOGH accounts","publisher_position":28,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":71,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1380,"page_start":1343,"pdf_page":7,"printed_page":6,"text":"29 E03400 Forfeited interest penalty ","text_end":10940,"text_sha256":"867401e61c00cab7dde702e81c565e3f757bdbcbf2af32477834dfdc0157a4da","text_start":10903},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1380,"page_start":1343,"pdf_page":7,"printed_page":6,"text":"29 E03400 Forfeited interest penalty ","text_end":10963,"text_sha256":"867401e61c00cab7dde702e81c565e3f757bdbcbf2af32477834dfdc0157a4da","text_start":10926},"name":"E03400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Forfeited interest penalty","publisher_position":29,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":72,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1404,"page_start":1381,"pdf_page":7,"printed_page":6,"text":"30 E03500 Alimony paid ","text_end":10964,"text_sha256":"a242a0ae85f5b2e7bc3d0bc4c8c4265435beb5299fc68f7a05f1b06108a9615c","text_start":10941},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1404,"page_start":1381,"pdf_page":7,"printed_page":6,"text":"30 E03500 Alimony paid ","text_end":10987,"text_sha256":"a242a0ae85f5b2e7bc3d0bc4c8c4265435beb5299fc68f7a05f1b06108a9615c","text_start":10964},"name":"E03500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alimony paid","publisher_position":30,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":73,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1464,"page_start":1405,"pdf_page":7,"printed_page":6,"text":"AGI 31 E00100 Adjusted Gross Income (deficit) (AGI) (+/-)","text_end":11024,"text_sha256":"40277b800d8de41a73e6df8b5a317a17bf4434f5b70f0176d19b390cbe993012","text_start":10965},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1464,"page_start":1405,"pdf_page":7,"printed_page":6,"text":"AGI 31 E00100 Adjusted Gross Income (deficit) (AGI) (+/-)","text_end":11047,"text_sha256":"40277b800d8de41a73e6df8b5a317a17bf4434f5b70f0176d19b390cbe993012","text_start":10988},"name":"E00100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Adjusted Gross Income (deficit) (AGI) (+/-)","publisher_position":31,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":74,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":113,"page_start":59,"pdf_page":8,"printed_page":7,"text":"Tax 32 P04470 Total deductions (standard or itemized) ","text_end":11138,"text_sha256":"7217c3a84ba6199a4568c02ecfe0b4e7f6595ddb3750a5747ed673b2fcecf75a","text_start":11084},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":114,"page_start":59,"pdf_page":8,"printed_page":7,"text":"Tax 32 P 04470 Total deductions (standard or itemized) ","text_end":11162,"text_sha256":"e64b39c4f1d50937158869f0ab4f13c5e99d5fd0f04b534cba45e6b011587889","text_start":11107},"name":"P04470","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total deductions (standard or itemized)","publisher_position":32,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":75,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":152,"page_start":114,"pdf_page":8,"printed_page":7,"text":"Computation 33 E04600 Exemption amount","text_end":11177,"text_sha256":"10be05859eaeec4a0842c04c01138f0c5d68d521ec82d36102844cf3fa652c6a","text_start":11139},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":154,"page_start":115,"pdf_page":8,"printed_page":7,"text":"Computation 33 E 04600 Exemption amount","text_end":11202,"text_sha256":"4a96690a9d89c712a55dd39d78f8a14a62ef3ef74d00fabc87b793ab57c50a9c","text_start":11163},"name":"E04600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Exemption amount","publisher_position":33,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":76,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":177,"page_start":153,"pdf_page":8,"printed_page":7,"text":"34 E04800 Taxable income","text_end":11202,"text_sha256":"225956f7e91356f9f90468a4395e1ed1451117a10725a5582a366af9ebb73665","text_start":11178},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":180,"page_start":155,"pdf_page":8,"printed_page":7,"text":"34 E04800 T axable income","text_end":11228,"text_sha256":"a6647ad88a4fb78304bb79035e3acbc2255212638c8a5c9f4ed9c7e08d1abd50","text_start":11203},"name":"E04800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Taxable income","publisher_position":34,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":77,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":209,"page_start":178,"pdf_page":8,"printed_page":7,"text":"35 E05100 Tax on taxable income","text_end":11234,"text_sha256":"f07c9259b17f51670ba7efd80c7b97ab1ecaec58f086b2684c76ec05bd23e7c2","text_start":11203},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":213,"page_start":181,"pdf_page":8,"printed_page":7,"text":"35 E05100 T ax on taxable income","text_end":11261,"text_sha256":"ba3d243665c0afd6cf245154c47ff83a499abb58a5acd53f88cc2fec45506c10","text_start":11229},"name":"E05100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax on taxable income","publisher_position":35,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":78,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":240,"page_start":210,"pdf_page":8,"printed_page":7,"text":"36 E05200 Computed regular tax","text_end":11265,"text_sha256":"7fe4593d18d6f23f85a0d57e4cc326a7d703a8df348bac50c6ca0ca94d33141e","text_start":11235},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":245,"page_start":214,"pdf_page":8,"printed_page":7,"text":"36 E05200 C omputed regular tax","text_end":11293,"text_sha256":"610ccde83a35567b71be3f7b134f6c2b7fd49a5002144b9fdb9adab087a18071","text_start":11262},"name":"E05200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Computed regular tax","publisher_position":36,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":79,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":276,"page_start":241,"pdf_page":8,"printed_page":7,"text":"37 E05800 Income tax before credits","text_end":11301,"text_sha256":"d6c09afb0eb5859b5f2303f4c57a3b439ab09c3d50b55e30fdd42f95cbbfe1f3","text_start":11266},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":282,"page_start":246,"pdf_page":8,"printed_page":7,"text":"37 E05800 I ncome tax before credits","text_end":11330,"text_sha256":"5cfba19950c904e1b2ed68779913fa3dca9a195d55aaed231f3f23a340f5a83f","text_start":11294},"name":"E05800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income tax before credits","publisher_position":37,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":80,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":308,"page_start":277,"pdf_page":8,"printed_page":7,"text":"38 E06000 Income subject to tax","text_end":11333,"text_sha256":"3aab3e844780e493265bfee48edef4d92c8e523e3af3bae76a3024c5de49bdde","text_start":11302},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":315,"page_start":283,"pdf_page":8,"printed_page":7,"text":"38 E06000 I ncome subject to tax","text_end":11363,"text_sha256":"3d72d60a6987087644a188050422acb8ca44ee4741dc53049113416d0a64f2fb","text_start":11331},"name":"E06000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income subject to tax","publisher_position":38,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":81,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":336,"page_start":309,"pdf_page":8,"printed_page":7,"text":"39 E06200 Marginal tax base","text_end":11361,"text_sha256":"cc9cabdcad93c863ec588b36ec7398ea2799fee802d92d209fc351647ec06854","text_start":11334},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":344,"page_start":316,"pdf_page":8,"printed_page":7,"text":"39 E06200 M arginal tax base","text_end":11392,"text_sha256":"05bba14fbadd5949179ca9ecdb80a849e9d9dc81f66ff24c932ec24b8b55a410","text_start":11364},"name":"E06200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Marginal tax base","publisher_position":39,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":82,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":378,"page_start":337,"pdf_page":8,"printed_page":7,"text":"40 E06300 Tax generated (tax rate tables)","text_end":11403,"text_sha256":"2d912e7ead6d8b12b0e1576cf402112dec8ed1f00ddb95d70eb96115a1e46727","text_start":11362},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":387,"page_start":345,"pdf_page":8,"printed_page":7,"text":"40 E06300 T ax generated (tax rate tables)","text_end":11435,"text_sha256":"3b2e76a22d70602f18562832d61d4c5386a4504648490bef9a7f964bd4c2861c","text_start":11393},"name":"E06300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax generated (tax rate tables)","publisher_position":40,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":83,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":412,"page_start":379,"pdf_page":8,"printed_page":7,"text":"41 E09600 Alternative minimum tax","text_end":11437,"text_sha256":"21de44978c2d3b1c06219dce1f35457e2d63c70380b2c3b508730e5a9b729920","text_start":11404},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":422,"page_start":388,"pdf_page":8,"printed_page":7,"text":"41 E09600 A lternative minimum tax","text_end":11470,"text_sha256":"4f1df3f395a937de11095cc2726d9adc9fdda85de4436a28ff4bdcdd1e23615e","text_start":11436},"name":"E09600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum tax","publisher_position":41,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":84,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":462,"page_start":413,"pdf_page":8,"printed_page":7,"text":"Credits 42 E07180 Child and Dependent Care Credit","text_end":11487,"text_sha256":"c7f8b0a16eed143191bdc26d8673a4fa7f0bb26b77163f4725e7b1a6fef829c3","text_start":11438},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":473,"page_start":423,"pdf_page":8,"printed_page":7,"text":"Credits 42 E 07180 Child and Dependent Care Credit","text_end":11521,"text_sha256":"6c7fb4017550d824a8580a9f58783f80834bfcd18c87eb7221495274eb83c03b","text_start":11471},"name":"E07180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Child and Dependent Care Credit","publisher_position":42,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":85,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":499,"page_start":463,"pdf_page":8,"printed_page":7,"text":"43 E07200 Elderly or Disabled Credit","text_end":11524,"text_sha256":"86a25fa2ff94fabc225ba5db603f961c7ddfc77a505a105cac63779c8a1bbeb2","text_start":11488},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":511,"page_start":474,"pdf_page":8,"printed_page":7,"text":"43 E07200 E lderly or Disabled Credit","text_end":11559,"text_sha256":"dbd830a3a7c0fd4c8518a9c65fa2aa01bbee66f26e05021e536fb7e785238121","text_start":11522},"name":"E07200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Elderly or Disabled Credit","publisher_position":43,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":86,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":528,"page_start":500,"pdf_page":8,"printed_page":7,"text":"44 E07220 Child Tax Credit","text_end":11553,"text_sha256":"4665ac1c4be769c23bd4e874cf7ac0be0e3fc159c3eb99cd45755c75788fec10","text_start":11525},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":541,"page_start":512,"pdf_page":8,"printed_page":7,"text":"44 E07220 C hild Tax Credit","text_end":11589,"text_sha256":"2e8c10e2c4b8c72efcb18d8b1b8ac1371dce6ad2cb64dabd8c0f01f7f54e06d7","text_start":11560},"name":"E07220","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Child Tax Credit","publisher_position":44,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":87,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":556,"page_start":529,"pdf_page":8,"printed_page":7,"text":"45 E07230 Education Credits","text_end":11581,"text_sha256":"d22ce5cc8537272d49db695f282bc049e59da6d3f7ba71f8a2d659cda3cec245","text_start":11554},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":570,"page_start":542,"pdf_page":8,"printed_page":7,"text":"45 E07230 E ducation Credits","text_end":11618,"text_sha256":"cf53661d3150bb55482e3f4bf94cd39b1634b576f99f2056af72ffe888463a01","text_start":11590},"name":"E07230","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Education Credits","publisher_position":45,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":88,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":592,"page_start":557,"pdf_page":8,"printed_page":7,"text":"46 E07240 Retirement Savings Credit","text_end":11617,"text_sha256":"ed0581e66c4d315b00544c6c5870f50d59b78866d11ba70ab7b8f1c806530828","text_start":11582},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":607,"page_start":571,"pdf_page":8,"printed_page":7,"text":"46 E07240 R etirement Savings Credit","text_end":11655,"text_sha256":"01c2172fb6749206e21e3ac233baedc6f036f9c658a26495018ba37eabada65e","text_start":11619},"name":"E07240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Retirement Savings Credit","publisher_position":46,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":89,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":628,"page_start":593,"pdf_page":8,"printed_page":7,"text":"47 E07260 Residential Energy Credit","text_end":11653,"text_sha256":"e4b21e879dead9af53a59d70b1ad032f1df39ab3c0f0a58779d932d7b0303ad1","text_start":11618},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":644,"page_start":608,"pdf_page":8,"printed_page":7,"text":"47 E07260 R esidential Energy Credit","text_end":11692,"text_sha256":"5bfb63e57d318883c240ab16740b02956147a641a5af9e1053af50bc180e1ab2","text_start":11656},"name":"E07260","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Residential Energy Credit","publisher_position":47,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":90,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":657,"page_start":629,"pdf_page":8,"printed_page":7,"text":"48 E07300 Foreign Tax Credit","text_end":11682,"text_sha256":"b811c7f90feb7afd9cdd277dee25be672d7e35627f734a106464c6fb1c3a4c9a","text_start":11654},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":674,"page_start":645,"pdf_page":8,"printed_page":7,"text":"48 E07300 F oreign Tax Credit","text_end":11722,"text_sha256":"b995a12a2eb3a80e72a1c84879e2785fb79b77a2facd459e532a0d96dd4c81e0","text_start":11693},"name":"E07300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Foreign Tax Credit","publisher_position":48,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":91,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":691,"page_start":658,"pdf_page":8,"printed_page":7,"text":"49 E07400 General Business Credit","text_end":11716,"text_sha256":"ce11f898797d8cfb0d3c67d6a3fe1786a7bf4c6deb9c35ab962a1bb568694e44","text_start":11683},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":709,"page_start":675,"pdf_page":8,"printed_page":7,"text":"49 E07400 G eneral Business Credit","text_end":11757,"text_sha256":"18b5a70022d438fb26656e16d8df7f40b5ce6a4dda85d539cf47abeb304c0185","text_start":11723},"name":"E07400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"General Business Credit","publisher_position":49,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":92,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":735,"page_start":692,"pdf_page":8,"printed_page":7,"text":"50 E07600 Credit for prior year minimum tax","text_end":11760,"text_sha256":"5bda2c85cf5a6668471b1a32d74e1647ca9faa346522bad6acc959bdb8575e94","text_start":11717},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":754,"page_start":710,"pdf_page":8,"printed_page":7,"text":"50 E07600 C redit for prior year minimum tax","text_end":11802,"text_sha256":"879f9c80102582a44633fb593718e762162ad1efa2e827282931eb169033d5cb","text_start":11758},"name":"E07600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit for prior year minimum tax","publisher_position":50,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":93,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":763,"page_start":736,"pdf_page":8,"printed_page":7,"text":"51 P08000 Other Tax Credits","text_end":11788,"text_sha256":"a06e7b7c20d74977127cbbe60401c09eae0a6d6fd1108537e90f6978f0e10699","text_start":11761},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":783,"page_start":755,"pdf_page":8,"printed_page":7,"text":"51 P08000 O ther Tax Credits","text_end":11831,"text_sha256":"bdcb3d710589efc74e919875a86a1529492ef024d550db7993e73d78248b2490","text_start":11803},"name":"P08000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other Tax Credits","publisher_position":51,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":94,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":796,"page_start":764,"pdf_page":8,"printed_page":7,"text":"52 E07150 Total tax credit (SOI)","text_end":11821,"text_sha256":"2bdd809fb4a2b5fad127d86f1919a4eb45e4a13e88553e5849d1b42691793dc4","text_start":11789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":817,"page_start":784,"pdf_page":8,"printed_page":7,"text":"52 E07150 T otal tax credit (SOI)","text_end":11865,"text_sha256":"7ed6338920c3da222745d61d65971c36f37303971c4a98ab432942f3e2405b75","text_start":11832},"name":"E07150","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax credit (SOI)","publisher_position":52,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":95,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":829,"page_start":797,"pdf_page":8,"printed_page":7,"text":"Taxes 53 E06500 Total income tax","text_end":11854,"text_sha256":"1469e4a861ea96162067edf32f7c0f8c7ae68a498e754ee9f14ec96fd48560d5","text_start":11822},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":851,"page_start":818,"pdf_page":8,"printed_page":7,"text":"Taxes 53 E 06500 Total income tax","text_end":11899,"text_sha256":"0762def63b6cd9e122bc748d2727e061ddc698c4d84aff1d7abfd16f273c5570","text_start":11866},"name":"E06500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total income tax","publisher_position":53,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":96,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":870,"page_start":830,"pdf_page":8,"printed_page":7,"text":"54 E08800 Income Tax after Credits (SOI)","text_end":11895,"text_sha256":"a9a103dc7547c727ba65a7d6b64450ed88d4885943ae06015eccc657f399f6aa","text_start":11855},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":893,"page_start":852,"pdf_page":8,"printed_page":7,"text":"54 E08800 I ncome Tax after Credits (SOI)","text_end":11941,"text_sha256":"bfe3a1c2c853909de7ad99d58a24f82482cd65ded6de5062ebe6d6754a1bf57a","text_start":11900},"name":"E08800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income Tax after Credits (SOI)","publisher_position":54,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":97,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":900,"page_start":871,"pdf_page":8,"printed_page":7,"text":"55 E09400 Self-employment tax","text_end":11925,"text_sha256":"7aae3feca6c81e9898b38d2470e6d52d320ca078a5a3f5d7bdcda51081702022","text_start":11896},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":924,"page_start":894,"pdf_page":8,"printed_page":7,"text":"55 E09400 S elf-employment tax","text_end":11972,"text_sha256":"cf8699e596032b9fd4764f46b334bb2ea3503dfc55f97310705e534ff943c397","text_start":11942},"name":"E09400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment tax","publisher_position":55,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":98,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":926,"page_start":901,"pdf_page":8,"printed_page":7,"text":"56 E09700 Recapture taxes","text_end":11951,"text_sha256":"95614e6f4cb3a76ceffb9e17d6d31eee5be0df29a4b8650bda98e8a996a56dd7","text_start":11926},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":951,"page_start":925,"pdf_page":8,"printed_page":7,"text":"56 E09700 R ecapture taxes","text_end":11999,"text_sha256":"d69e68dc57d670091cc988385d97b58e46d749157bab78600ef07f44e5bb3461","text_start":11973},"name":"E09700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Recapture taxes","publisher_position":56,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":99,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":966,"page_start":927,"pdf_page":8,"printed_page":7,"text":"57 E09730 Total Additional Medicare Tax","text_end":11991,"text_sha256":"76e56e65c3ea9269de1ac7c4f517f6a6e3e7a7190e817728f9c87f00be8900bd","text_start":11952},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":992,"page_start":952,"pdf_page":8,"printed_page":7,"text":"57 E09730 T otal Additional Medicare Tax","text_end":12040,"text_sha256":"fc1bdab89bd58d4992f505a48ba4bb50fe3489533304c7f3d0093864b5f4dd0d","text_start":12000},"name":"E09730","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Additional Medicare Tax","publisher_position":57,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":100,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1002,"page_start":967,"pdf_page":8,"printed_page":7,"text":"58 E09740 Net Investment Income Tax","text_end":12027,"text_sha256":"ae779721c98e842e8f3c73eb8c052ea9437311f2fcf4a78596d78b6aedbcf0f5","text_start":11992},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1029,"page_start":993,"pdf_page":8,"printed_page":7,"text":"58 E09740 N et Investment Income Tax","text_end":12077,"text_sha256":"2eea0be3d95ca8e28846e33d87213963674c000be2dcea9164ac84fb03b1f02c","text_start":12041},"name":"E09740","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Investment Income Tax","publisher_position":58,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":101,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1058,"page_start":1003,"pdf_page":8,"printed_page":7,"text":"59 E09750 Health Care Individual Responsibility Payment","text_end":12083,"text_sha256":"2fc634d9369fdfd2e26fcf87d86b9cad2a1d3f19045f84a538757332d437de2f","text_start":12028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1086,"page_start":1030,"pdf_page":8,"printed_page":7,"text":"59 E09750 H ealth Care Individual Responsibility Payment","text_end":12134,"text_sha256":"81052da246ea49aaf486e06f39b37d3c4e8953091e45fc44079beb76b8553f33","text_start":12078},"name":"E09750","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Health Care Individual Responsibility Payment","publisher_position":59,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":102,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1102,"page_start":1059,"pdf_page":8,"printed_page":7,"text":"60 E09800 Social security tax on tip income","text_end":12127,"text_sha256":"dfe787c463d2f18754a247ff0e9f2e2afa79f7e7a5ce8e1cc157053c1ba85559","text_start":12084},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1131,"page_start":1087,"pdf_page":8,"printed_page":7,"text":"60 E09800 S ocial security tax on tip income","text_end":12179,"text_sha256":"758076146bbb9e4271a02412e0002e077dc923ef78436fd97950230e6f833d18","text_start":12135},"name":"E09800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Social security tax on tip income","publisher_position":60,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":103,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1131,"page_start":1103,"pdf_page":8,"printed_page":7,"text":"61 E09900 Penalty tax on IRA","text_end":12156,"text_sha256":"28bac09e2a8048c28bf53b7c47990cf81509b535662409223b7a9427ee34c031","text_start":12128},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1161,"page_start":1132,"pdf_page":8,"printed_page":7,"text":"61 E09900 P enalty tax on IRA","text_end":12209,"text_sha256":"b99dbce1d89eba9969a0306947a1c7457d14bd6b57851c24248c391017932c7a","text_start":12180},"name":"E09900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Penalty tax on IRA","publisher_position":61,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":104,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1167,"page_start":1132,"pdf_page":8,"printed_page":7,"text":"62 E10300 Total tax liability (SOI)","text_end":12192,"text_sha256":"e556f48e984fbab4c1a608f63717e57b75e7d6987ad01672a5e5864ba6ac373c","text_start":12157},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1198,"page_start":1162,"pdf_page":8,"printed_page":7,"text":"62 E10300 T otal tax liability (SOI)","text_end":12246,"text_sha256":"5c4be64fc37f325dc96809408c1baab97b0537551ae9d110e7c183bfbbc4918e","text_start":12210},"name":"E10300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax liability (SOI)","publisher_position":62,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":105,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":95,"page_start":57,"pdf_page":9,"printed_page":8,"text":"Payments 63 E10700 Income tax withheld","text_end":12288,"text_sha256":"2ec1675834a00e512a58c9381864b776b714ed4373cdddae32895d985a0a7c3a","text_start":12250},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":95,"page_start":57,"pdf_page":9,"printed_page":8,"text":"Payments 63 E10700 Income tax withheld","text_end":12342,"text_sha256":"2ec1675834a00e512a58c9381864b776b714ed4373cdddae32895d985a0a7c3a","text_start":12304},"name":"E10700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Income tax withheld","publisher_position":63,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":106,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":130,"page_start":96,"pdf_page":9,"printed_page":8,"text":"64 E10900 Estimated tax payments","text_end":12323,"text_sha256":"75627724cfa477e23bbd43e0a7debdf4c4b6b25cb76fbc60d6105f235643da60","text_start":12289},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":130,"page_start":96,"pdf_page":9,"printed_page":8,"text":"64 E10900 Estimated tax payments","text_end":12377,"text_sha256":"75627724cfa477e23bbd43e0a7debdf4c4b6b25cb76fbc60d6105f235643da60","text_start":12343},"name":"E10900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Estimated tax payments","publisher_position":64,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":107,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":179,"page_start":131,"pdf_page":9,"printed_page":8,"text":"65 E10960 Refundable American Opportunity Credit","text_end":12372,"text_sha256":"ab0ab6d30d1af12adea7720e88748dc063811593b58152ac9789909ff1bd8eed","text_start":12324},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":179,"page_start":131,"pdf_page":9,"printed_page":8,"text":"65 E10960 Refundable American Opportunity Credit","text_end":12426,"text_sha256":"ab0ab6d30d1af12adea7720e88748dc063811593b58152ac9789909ff1bd8eed","text_start":12378},"name":"E10960","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Refundable American Opportunity Credit","publisher_position":65,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":108,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":234,"page_start":180,"pdf_page":9,"printed_page":8,"text":"66 E59560 Earned income for earned income credit (EIC)","text_end":12427,"text_sha256":"fe390c9b179c68068bf3cc5e920b5a89129d372f2c7fb81634d68d0f73a85ddd","text_start":12373},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":234,"page_start":180,"pdf_page":9,"printed_page":8,"text":"66 E59560 Earned income for earned income credit (EIC)","text_end":12481,"text_sha256":"fe390c9b179c68068bf3cc5e920b5a89129d372f2c7fb81634d68d0f73a85ddd","text_start":12427},"name":"E59560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Earned income for earned income credit (EIC)","publisher_position":66,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":109,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":289,"page_start":235,"pdf_page":9,"printed_page":8,"text":"67 E59680 EIC used to offset income tax before credits","text_end":12482,"text_sha256":"2f948c4d5be959a7aab4abc2107d24427f1bc1df7f202cb8781f99d684e1b262","text_start":12428},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":289,"page_start":235,"pdf_page":9,"printed_page":8,"text":"67 E59680 EIC used to offset income tax before credits","text_end":12536,"text_sha256":"2f948c4d5be959a7aab4abc2107d24427f1bc1df7f202cb8781f99d684e1b262","text_start":12482},"name":"E59680","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC used to offset income tax before credits","publisher_position":67,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":110,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":353,"page_start":290,"pdf_page":9,"printed_page":8,"text":"68 E59700 EIC used to offset all other taxes except advance EIC","text_end":12546,"text_sha256":"31b02c1b8ebc8097080e693556f9ccebaabf07ec1a8acf72213baa96372a9989","text_start":12483},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":353,"page_start":290,"pdf_page":9,"printed_page":8,"text":"68 E59700 EIC used to offset all other taxes except advance EIC","text_end":12600,"text_sha256":"31b02c1b8ebc8097080e693556f9ccebaabf07ec1a8acf72213baa96372a9989","text_start":12537},"name":"E59700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC used to offset all other taxes except advance EIC","publisher_position":68,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":111,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":386,"page_start":354,"pdf_page":9,"printed_page":8,"text":"69 E59720 EIC refundable portion","text_end":12579,"text_sha256":"061911af9638aa1707e023d2655cda4c06437b76a4a2b8ae7900156c8fbb8747","text_start":12547},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":386,"page_start":354,"pdf_page":9,"printed_page":8,"text":"69 E59720 EIC refundable portion","text_end":12633,"text_sha256":"061911af9638aa1707e023d2655cda4c06437b76a4a2b8ae7900156c8fbb8747","text_start":12601},"name":"E59720","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"EIC refundable portion","publisher_position":69,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":112,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":437,"page_start":387,"pdf_page":9,"printed_page":8,"text":"70 E11550 Refundable prior year minimum tax credit","text_end":12630,"text_sha256":"92a84d3f42a9d9ec071ea4571d7caaff0fb80d1132fad1fa7a2380f85425f553","text_start":12580},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":437,"page_start":387,"pdf_page":9,"printed_page":8,"text":"70 E11550 Refundable prior year minimum tax credit","text_end":12684,"text_sha256":"92a84d3f42a9d9ec071ea4571d7caaff0fb80d1132fad1fa7a2380f85425f553","text_start":12634},"name":"E11550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Refundable prior year minimum tax credit","publisher_position":70,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":113,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":470,"page_start":438,"pdf_page":9,"printed_page":8,"text":"71 E11560 Net Premium Tax Credit","text_end":12663,"text_sha256":"8f25388839d6cc3c3cd2c1d95d23eda2b98c4baaa0843245f346eb045aee4f0a","text_start":12631},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":470,"page_start":438,"pdf_page":9,"printed_page":8,"text":"71 E11560 Net Premium Tax Credit","text_end":12717,"text_sha256":"8f25388839d6cc3c3cd2c1d95d23eda2b98c4baaa0843245f346eb045aee4f0a","text_start":12685},"name":"E11560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit","publisher_position":71,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":114,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":544,"page_start":471,"pdf_page":9,"printed_page":8,"text":"72 E11561 Net Premium Tax Credit Used to Offset Income Tax Before Credits","text_end":12737,"text_sha256":"adc352b7a716e01f12838eddd579ba35172879a827f6d7479e2535d8202fa6a1","text_start":12664},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":544,"page_start":471,"pdf_page":9,"printed_page":8,"text":"72 E11561 Net Premium Tax Credit Used to Offset Income Tax Before Credits","text_end":12791,"text_sha256":"adc352b7a716e01f12838eddd579ba35172879a827f6d7479e2535d8202fa6a1","text_start":12718},"name":"E11561","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Used to Offset Income Tax Before Credits","publisher_position":72,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":115,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":608,"page_start":545,"pdf_page":9,"printed_page":8,"text":"73 E11562 Net Premium Tax Credit Used to Offset All Other Taxes","text_end":12801,"text_sha256":"c991a6bbefc24b16715685110d277b081078fa19cb554264f7b516fcfd7183d6","text_start":12738},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":608,"page_start":545,"pdf_page":9,"printed_page":8,"text":"73 E11562 Net Premium Tax Credit Used to Offset All Other Taxes","text_end":12855,"text_sha256":"c991a6bbefc24b16715685110d277b081078fa19cb554264f7b516fcfd7183d6","text_start":12792},"name":"E11562","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Used to Offset All Other Taxes","publisher_position":73,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":116,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":660,"page_start":609,"pdf_page":9,"printed_page":8,"text":"74 E11563 Net Premium Tax Credit Refundable Portion","text_end":12853,"text_sha256":"39936472c1503f60a139eddfe93bcce36d03d664fad3a30b23323daab3917ade","text_start":12802},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":660,"page_start":609,"pdf_page":9,"printed_page":8,"text":"74 E11563 Net Premium Tax Credit Refundable Portion","text_end":12907,"text_sha256":"39936472c1503f60a139eddfe93bcce36d03d664fad3a30b23323daab3917ade","text_start":12856},"name":"E11563","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net Premium Tax Credit Refundable Portion","publisher_position":74,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":117,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":700,"page_start":661,"pdf_page":9,"printed_page":8,"text":"75 E11070 Additional Child Tax Credit","text_end":12893,"text_sha256":"040dc313e83f019d1a7fdd86ac6fa9fa1dd4ea6e3bcd92c3ae66de5e09a557af","text_start":12854},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":700,"page_start":661,"pdf_page":9,"printed_page":8,"text":"75 E11070 Additional Child Tax Credit","text_end":12947,"text_sha256":"040dc313e83f019d1a7fdd86ac6fa9fa1dd4ea6e3bcd92c3ae66de5e09a557af","text_start":12908},"name":"E11070","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Additional Child Tax Credit","publisher_position":75,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":118,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":761,"page_start":701,"pdf_page":9,"printed_page":8,"text":"76 E11100 Amount paid with Form 4868 (Request for Extension)","text_end":12954,"text_sha256":"91a1c51438810477329c58b1633cdb8948a4dfe77239a39790e26f4be94d7914","text_start":12894},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":761,"page_start":701,"pdf_page":9,"printed_page":8,"text":"76 E11100 Amount paid with Form 4868 (Request for Extension)","text_end":13008,"text_sha256":"91a1c51438810477329c58b1633cdb8948a4dfe77239a39790e26f4be94d7914","text_start":12948},"name":"E11100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Amount paid with Form 4868 (Request for Extension)","publisher_position":76,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":119,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":788,"page_start":762,"pdf_page":9,"printed_page":8,"text":"77 E11200 Excess FICA/RRTA","text_end":12981,"text_sha256":"0c24759a396f0f273edc9dde02cc4562d41b1a8ac24615d03ba23c7c77feec8f","text_start":12955},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":788,"page_start":762,"pdf_page":9,"printed_page":8,"text":"77 E11200 Excess FICA/RRTA","text_end":13035,"text_sha256":"0c24759a396f0f273edc9dde02cc4562d41b1a8ac24615d03ba23c7c77feec8f","text_start":13009},"name":"E11200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Excess FICA/RRTA","publisher_position":77,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":120,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":847,"page_start":789,"pdf_page":9,"printed_page":8,"text":"78 E11300 Credit for federal tax on special fuels and oils","text_end":13040,"text_sha256":"7304b8c8e2a6223db69f4a0fc11b59de90d57ff9ed998f9ea24ba2d444191e00","text_start":12982},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":847,"page_start":789,"pdf_page":9,"printed_page":8,"text":"78 E11300 Credit for federal tax on special fuels and oils","text_end":13094,"text_sha256":"7304b8c8e2a6223db69f4a0fc11b59de90d57ff9ed998f9ea24ba2d444191e00","text_start":13036},"name":"E11300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit for federal tax on special fuels and oils","publisher_position":78,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":121,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":893,"page_start":848,"pdf_page":9,"printed_page":8,"text":"79 E11400 Regulated investment company credit","text_end":13086,"text_sha256":"929c13813a8b909fefaf29e06a1ea8d87ae6a798b760e6b05d09707656cd30ea","text_start":13041},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":893,"page_start":848,"pdf_page":9,"printed_page":8,"text":"79 E11400 Regulated investment company credit","text_end":13140,"text_sha256":"929c13813a8b909fefaf29e06a1ea8d87ae6a798b760e6b05d09707656cd30ea","text_start":13095},"name":"E11400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Regulated investment company credit","publisher_position":79,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":122,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":969,"page_start":894,"pdf_page":9,"printed_page":8,"text":"80 E11601 Total Refundable Credits Used to Offset Income Tax Before Credits","text_end":13162,"text_sha256":"901e4b62dd84f28c11773d525adce98423ad11db209d0acb64ee54f3f38e0f48","text_start":13087},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":969,"page_start":894,"pdf_page":9,"printed_page":8,"text":"80 E11601 Total Refundable Credits Used to Offset Income Tax Before Credits","text_end":13216,"text_sha256":"901e4b62dd84f28c11773d525adce98423ad11db209d0acb64ee54f3f38e0f48","text_start":13141},"name":"E11601","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits Used to Offset Income Tax Before Credits","publisher_position":80,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":123,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1035,"page_start":970,"pdf_page":9,"printed_page":8,"text":"81 E11602 Total Refundable Credits Used to Offset All Other Taxes","text_end":13228,"text_sha256":"990e11754e4558b12c74042a2dfab2766c5fa2a2f68f0e09ee8c30ff127b3c3b","text_start":13163},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1035,"page_start":970,"pdf_page":9,"printed_page":8,"text":"81 E11602 Total Refundable Credits Used to Offset All Other Taxes","text_end":13282,"text_sha256":"990e11754e4558b12c74042a2dfab2766c5fa2a2f68f0e09ee8c30ff127b3c3b","text_start":13217},"name":"E11602","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits Used to Offset All Other Taxes","publisher_position":81,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":124,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1087,"page_start":1036,"pdf_page":9,"printed_page":8,"text":"82 E11603 Total Refundable Credits refundable parts","text_end":13280,"text_sha256":"4c5413d9f359b29220824f2fd019c847b16b38d48e4f05aa8b2c232c7e8e06d4","text_start":13229},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1087,"page_start":1036,"pdf_page":9,"printed_page":8,"text":"82 E11603 Total Refundable Credits refundable parts","text_end":13334,"text_sha256":"4c5413d9f359b29220824f2fd019c847b16b38d48e4f05aa8b2c232c7e8e06d4","text_start":13283},"name":"E11603","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Refundable Credits refundable parts","publisher_position":82,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":125,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1122,"page_start":1088,"pdf_page":9,"printed_page":8,"text":"83 E10605 Total tax payments (SOI)","text_end":13315,"text_sha256":"bafa4982b2bc104cd05f8e4a8ace72c10c10caa7bd404d5d12093d7fdae3f6f6","text_start":13281},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1122,"page_start":1088,"pdf_page":9,"printed_page":8,"text":"83 E10605 Total tax payments (SOI)","text_end":13369,"text_sha256":"bafa4982b2bc104cd05f8e4a8ace72c10c10caa7bd404d5d12093d7fdae3f6f6","text_start":13335},"name":"E10605","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total tax payments (SOI)","publisher_position":83,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":126,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1176,"page_start":1123,"pdf_page":9,"printed_page":8,"text":"Refund etc. 84 E11900 Balance due (overpayment) (+/-)","text_end":13369,"text_sha256":"e1fa87d21b6f5be5368d265edc33a4e2782145385b7b5d37d7f7846e6ab5da48","text_start":13316},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1176,"page_start":1123,"pdf_page":9,"printed_page":8,"text":"Refund etc. 84 E11900 Balance due (overpayment) (+/-)","text_end":13423,"text_sha256":"e1fa87d21b6f5be5368d265edc33a4e2782145385b7b5d37d7f7846e6ab5da48","text_start":13370},"name":"E11900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Balance due (overpayment) (+/-)","publisher_position":84,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":127,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1199,"page_start":1177,"pdf_page":9,"printed_page":8,"text":"85 E12000 Credit elect","text_end":13392,"text_sha256":"5d78d040c8635ffc9f4169045377df4dd3f633b65c9bc2faafaf74b60ba327b2","text_start":13370},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1199,"page_start":1177,"pdf_page":9,"printed_page":8,"text":"85 E12000 Credit elect","text_end":13446,"text_sha256":"5d78d040c8635ffc9f4169045377df4dd3f633b65c9bc2faafaf74b60ba327b2","text_start":13424},"name":"E12000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Credit elect","publisher_position":85,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":128,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1245,"page_start":1200,"pdf_page":9,"printed_page":8,"text":"86 E12200 Predetermined estimated tax penalty","text_end":13438,"text_sha256":"d9ae52686557efdf00899962c7ced85b0a7c7b1456476672166687a6160f0b06","text_start":13393},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1245,"page_start":1200,"pdf_page":9,"printed_page":8,"text":"86 E12200 Predetermined estimated tax penalty","text_end":13492,"text_sha256":"d9ae52686557efdf00899962c7ced85b0a7c7b1456476672166687a6160f0b06","text_start":13447},"name":"E12200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Predetermined estimated tax penalty","publisher_position":86,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":129,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1328,"page_start":1246,"pdf_page":9,"printed_page":8,"text":"Schedule A 87 E17500 Medical and dental expenses subject to reduction by AGI limit","text_end":13521,"text_sha256":"e32903c1132bc1d3e5b64f301ff0bf4d3286af26c821cf59d501505bd45cbd32","text_start":13439},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1328,"page_start":1246,"pdf_page":9,"printed_page":8,"text":"Schedule A 87 E17500 Medical and dental expenses subject to reduction by AGI limit","text_end":13575,"text_sha256":"e32903c1132bc1d3e5b64f301ff0bf4d3286af26c821cf59d501505bd45cbd32","text_start":13493},"name":"E17500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Medical and dental expenses subject to reduction by AGI limit","publisher_position":87,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":130,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1360,"page_start":1329,"pdf_page":9,"printed_page":8,"text":"88 E18400 State and local taxes","text_end":13553,"text_sha256":"ddfa187258de45c2b29771f40d3c5ca194d94230e598b9b177faea049aa6b4aa","text_start":13522},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1360,"page_start":1329,"pdf_page":9,"printed_page":8,"text":"88 E18400 State and local taxes","text_end":13607,"text_sha256":"ddfa187258de45c2b29771f40d3c5ca194d94230e598b9b177faea049aa6b4aa","text_start":13576},"name":"E18400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"State and local taxes","publisher_position":88,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":131,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1397,"page_start":1361,"pdf_page":9,"printed_page":8,"text":"89 E18500 Real estate tax deductions","text_end":13590,"text_sha256":"8f00348204122a4830340660f171bb3c3821fdec2c22cc97d136d8ab05f41eb3","text_start":13554},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1397,"page_start":1361,"pdf_page":9,"printed_page":8,"text":"89 E18500 Real estate tax deductions","text_end":13644,"text_sha256":"8f00348204122a4830340660f171bb3c3821fdec2c22cc97d136d8ab05f41eb3","text_start":13608},"name":"E18500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Real estate tax deductions","publisher_position":89,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":132,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1437,"page_start":1398,"pdf_page":9,"printed_page":8,"text":"90 E19200 Total interest paid deduction","text_end":13630,"text_sha256":"439821e914f73fc45684f88a5cf2b8a33b108e9f661823070941548f3936f8bc","text_start":13591},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1437,"page_start":1398,"pdf_page":9,"printed_page":8,"text":"90 E19200 Total interest paid deduction","text_end":13684,"text_sha256":"439821e914f73fc45684f88a5cf2b8a33b108e9f661823070941548f3936f8bc","text_start":13645},"name":"E19200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total interest paid deduction","publisher_position":90,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":133,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1485,"page_start":1438,"pdf_page":9,"printed_page":8,"text":"91 E19550 Qualified Mortgage Insurance Premiums","text_end":13678,"text_sha256":"b40c4937f620b500331b1761429127aab74734aa246b5d5984372ca90a9aa383","text_start":13631},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1485,"page_start":1438,"pdf_page":9,"printed_page":8,"text":"91 E19550 Qualified Mortgage Insurance Premiums","text_end":13732,"text_sha256":"b40c4937f620b500331b1761429127aab74734aa246b5d5984372ca90a9aa383","text_start":13685},"name":"E19550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Qualified Mortgage Insurance Premiums","publisher_position":91,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":134,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1514,"page_start":1486,"pdf_page":9,"printed_page":8,"text":"92 E19800 Cash contributions","text_end":13707,"text_sha256":"1f645a012922ff0adaff19fb9bcc239e615bd77859877cfc6095f33bef086592","text_start":13679},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1514,"page_start":1486,"pdf_page":9,"printed_page":8,"text":"92 E19800 Cash contributions","text_end":13761,"text_sha256":"1f645a012922ff0adaff19fb9bcc239e615bd77859877cfc6095f33bef086592","text_start":13733},"name":"E19800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Cash contributions","publisher_position":92,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":135,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1554,"page_start":1515,"pdf_page":9,"printed_page":8,"text":"93 E20100 Other than cash contributions","text_end":13747,"text_sha256":"7b2ae0eda7289cdc24da53278d55944fd14e28383ae760756379c879c1902fca","text_start":13708},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1554,"page_start":1515,"pdf_page":9,"printed_page":8,"text":"93 E20100 Other than cash contributions","text_end":13801,"text_sha256":"7b2ae0eda7289cdc24da53278d55944fd14e28383ae760756379c879c1902fca","text_start":13762},"name":"E20100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Other than cash contributions","publisher_position":93,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":136,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1595,"page_start":1555,"pdf_page":9,"printed_page":8,"text":"94 E19700 Contributions deduction, total","text_end":13788,"text_sha256":"751df3b920c087fb69355fae1da856a1078970fd840324bc79f855dca29327a2","text_start":13748},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1595,"page_start":1555,"pdf_page":9,"printed_page":8,"text":"94 E19700 Contributions deduction, total","text_end":13842,"text_sha256":"751df3b920c087fb69355fae1da856a1078970fd840324bc79f855dca29327a2","text_start":13802},"name":"E19700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Contributions deduction, total","publisher_position":94,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":137,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1644,"page_start":1596,"pdf_page":9,"printed_page":8,"text":"95 E20550 Unreimbursed employee business expense","text_end":13837,"text_sha256":"3280ed6476b27be4243fbc9e564c1e5e9912ead010448d59581d39d72e6ded2b","text_start":13789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1644,"page_start":1596,"pdf_page":9,"printed_page":8,"text":"95 E20550 Unreimbursed employee business expense","text_end":13891,"text_sha256":"3280ed6476b27be4243fbc9e564c1e5e9912ead010448d59581d39d72e6ded2b","text_start":13843},"name":"E20550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unreimbursed employee business expense","publisher_position":95,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":138,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1674,"page_start":1645,"pdf_page":9,"printed_page":8,"text":"96 E20600 Tax preparation fee","text_end":13867,"text_sha256":"9a90312e099c6324a38e35937d5d21a9a9d8e2a8c72fd9529ea7cf5dddcb61d0","text_start":13838},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1674,"page_start":1645,"pdf_page":9,"printed_page":8,"text":"96 E20600 Tax preparation fee","text_end":13921,"text_sha256":"9a90312e099c6324a38e35937d5d21a9a9d8e2a8c72fd9529ea7cf5dddcb61d0","text_start":13892},"name":"E20600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tax preparation fee","publisher_position":96,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":139,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1742,"page_start":1675,"pdf_page":9,"printed_page":8,"text":"97 E20400 Miscellaneous deductions subject to AGI limitation, total","text_end":13935,"text_sha256":"db6224d05b7e4e7320dc041033f9b221b17b11bf1ddf91d6927a2c7a425982c1","text_start":13868},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1742,"page_start":1675,"pdf_page":9,"printed_page":8,"text":"97 E20400 Miscellaneous deductions subject to AGI limitation, total","text_end":13989,"text_sha256":"db6224d05b7e4e7320dc041033f9b221b17b11bf1ddf91d6927a2c7a425982c1","text_start":13922},"name":"E20400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Miscellaneous deductions subject to AGI limitation, total","publisher_position":97,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":140,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1789,"page_start":1743,"pdf_page":9,"printed_page":8,"text":"98 E20800 Net limited miscellaneous deductions","text_end":13982,"text_sha256":"b5735f97598efaaa6630b2f6d6f570a491bcdf4ab9769974b9a123467f7b67f0","text_start":13936},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1789,"page_start":1743,"pdf_page":9,"printed_page":8,"text":"98 E20800 Net limited miscellaneous deductions","text_end":14036,"text_sha256":"b5735f97598efaaa6630b2f6d6f570a491bcdf4ab9769974b9a123467f7b67f0","text_start":13990},"name":"E20800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net limited miscellaneous deductions","publisher_position":98,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":141,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1826,"page_start":1790,"pdf_page":9,"printed_page":8,"text":"99 E20500 Net casualty or theft loss","text_end":14019,"text_sha256":"fb66a5a3fcc72d2e64d8b5e5e226a4a12e1d1da8c5d00026a7273d3d181893d8","text_start":13983},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1826,"page_start":1790,"pdf_page":9,"printed_page":8,"text":"99 E20500 Net casualty or theft loss","text_end":14073,"text_sha256":"fb66a5a3fcc72d2e64d8b5e5e226a4a12e1d1da8c5d00026a7273d3d181893d8","text_start":14037},"name":"E20500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net casualty or theft loss","publisher_position":99,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":142,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1867,"page_start":1827,"pdf_page":9,"printed_page":8,"text":"100 E21040 Itemized deduction limitation","text_end":14060,"text_sha256":"dae3528bdda2a0b790242911e4b9cc1d18936bfb2fc3bd29ccfca74c4274a052","text_start":14020},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1867,"page_start":1827,"pdf_page":9,"printed_page":8,"text":"100 E21040 Itemized deduction limitation","text_end":14114,"text_sha256":"dae3528bdda2a0b790242911e4b9cc1d18936bfb2fc3bd29ccfca74c4274a052","text_start":14074},"name":"E21040","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Itemized deduction limitation","publisher_position":100,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":143,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14185,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14118},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14239,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14172},"name":"P22250","publisher_group":"amount_or_large_code","publisher_label":"Short-term gains less losses net of carryover","publisher_position":101,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":144,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":195,"page_start":125,"pdf_page":10,"printed_page":9,"text":"102 E22320 Long term gain/loss from other forms on Schedule D","text_end":14256,"text_sha256":"089560a8b3a740a645cdfec1838d2eb8006906fbd8932a94e1d91e8ad3381ba8","text_start":14186},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":195,"page_start":125,"pdf_page":10,"printed_page":9,"text":"102 E22320 Long term gain/loss from other forms on Schedule D","text_end":14310,"text_sha256":"089560a8b3a740a645cdfec1838d2eb8006906fbd8932a94e1d91e8ad3381ba8","text_start":14240},"name":"E22320","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Long term gain/loss from other forms on Schedule D","publisher_position":102,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":145,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":245,"page_start":196,"pdf_page":10,"printed_page":9,"text":"103 E22370 Schedule D capital gain distributions ","text_end":14306,"text_sha256":"9e3c59a530d0730058c9fba5bb7e99e090e8c389c585fa49f360d2107fd1f7a9","text_start":14257},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":245,"page_start":196,"pdf_page":10,"printed_page":9,"text":"103 E22370 Schedule D capital gain distributions ","text_end":14360,"text_sha256":"9e3c59a530d0730058c9fba5bb7e99e090e8c389c585fa49f360d2107fd1f7a9","text_start":14311},"name":"E22370","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D capital gain distributions","publisher_position":103,"publisher_signed_marker":false},{"admission":"monetary_source_projection","delivered_index":146,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14362,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14307},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14416,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14361},"name":"P23250","publisher_group":"amount_or_large_code","publisher_label":"Long-term gains less losses net of carryover","publisher_position":104,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":147,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":343,"page_start":302,"pdf_page":10,"printed_page":9,"text":"105 E24515 Unrecaptured Section 1250 gain","text_end":14404,"text_sha256":"b3530cde2dfbb1b4c6fb5a0b51e5313493dd594709a0cc3970a76bab97453461","text_start":14363},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":343,"page_start":302,"pdf_page":10,"printed_page":9,"text":"105 E24515 Unrecaptured Section 1250 gain","text_end":14458,"text_sha256":"b3530cde2dfbb1b4c6fb5a0b51e5313493dd594709a0cc3970a76bab97453461","text_start":14417},"name":"E24515","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Unrecaptured Section 1250 gain","publisher_position":105,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":148,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":391,"page_start":344,"pdf_page":10,"printed_page":9,"text":"106 E24516 Capital gain less investment expense","text_end":14452,"text_sha256":"9a4e3aff9c97e63b0d70dbe3b562faaef715367aee826d47dc278ece6dc903a3","text_start":14405},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":391,"page_start":344,"pdf_page":10,"printed_page":9,"text":"106 E24516 Capital gain less investment expense","text_end":14506,"text_sha256":"9a4e3aff9c97e63b0d70dbe3b562faaef715367aee826d47dc278ece6dc903a3","text_start":14459},"name":"E24516","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Capital gain less investment expense","publisher_position":106,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":149,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":424,"page_start":392,"pdf_page":10,"printed_page":9,"text":"107 E24518 28% Rate Gain or Loss","text_end":14485,"text_sha256":"6f1db8c750d7ce65f18c9f43563781a4116fa6ee8871e18c9310618d6324d75b","text_start":14453},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":424,"page_start":392,"pdf_page":10,"printed_page":9,"text":"107 E24518 28% Rate Gain or Loss","text_end":14539,"text_sha256":"6f1db8c750d7ce65f18c9f43563781a4116fa6ee8871e18c9310618d6324d75b","text_start":14507},"name":"E24518","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"28% Rate Gain or Loss","publisher_position":107,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":150,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":454,"page_start":425,"pdf_page":10,"printed_page":9,"text":"108 E24560 Non-Schedule D tax","text_end":14515,"text_sha256":"fa4bc43dd1207b390e04fa7d4e76787dc25fb318077540f034fcc3ce0d286465","text_start":14486},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":454,"page_start":425,"pdf_page":10,"printed_page":9,"text":"108 E24560 Non-Schedule D tax","text_end":14569,"text_sha256":"fa4bc43dd1207b390e04fa7d4e76787dc25fb318077540f034fcc3ce0d286465","text_start":14540},"name":"E24560","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Non-Schedule D tax","publisher_position":108,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":151,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":498,"page_start":455,"pdf_page":10,"printed_page":9,"text":"109 E24598 Schedule D 15 percent tax amount","text_end":14559,"text_sha256":"f74d6810780d586c9b08e9aa21b3e88c868528245a7761950f06ef70f4e4b7fd","text_start":14516},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":498,"page_start":455,"pdf_page":10,"printed_page":9,"text":"109 E24598 Schedule D 15 percent tax amount","text_end":14613,"text_sha256":"f74d6810780d586c9b08e9aa21b3e88c868528245a7761950f06ef70f4e4b7fd","text_start":14570},"name":"E24598","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 15 percent tax amount","publisher_position":109,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":152,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":542,"page_start":499,"pdf_page":10,"printed_page":9,"text":"110 E24615 Schedule D 25 percent tax amount","text_end":14603,"text_sha256":"2c70781b25a472c4b0f60ddd2020b6ccb429c0f6d3f9164e31315f973c902380","text_start":14560},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":542,"page_start":499,"pdf_page":10,"printed_page":9,"text":"110 E24615 Schedule D 25 percent tax amount","text_end":14657,"text_sha256":"2c70781b25a472c4b0f60ddd2020b6ccb429c0f6d3f9164e31315f973c902380","text_start":14614},"name":"E24615","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 25 percent tax amount","publisher_position":110,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":153,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":586,"page_start":543,"pdf_page":10,"printed_page":9,"text":"111 E24570 Schedule D 28 percent tax amount","text_end":14647,"text_sha256":"92ebc9580c0fb81e2158e5b24e0893662c90255b3170bae820bc60ad7254fd0e","text_start":14604},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":586,"page_start":543,"pdf_page":10,"printed_page":9,"text":"111 E24570 Schedule D 28 percent tax amount","text_end":14701,"text_sha256":"92ebc9580c0fb81e2158e5b24e0893662c90255b3170bae820bc60ad7254fd0e","text_start":14658},"name":"E24570","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Schedule D 28 percent tax amount","publisher_position":111,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":154,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":643,"page_start":587,"pdf_page":10,"printed_page":9,"text":"Schedule E 112 P25350 Total rents and royalties received","text_end":14704,"text_sha256":"b640f21795387c7dcee9b8b5eacf67e4afa2f4ea930fb4d85da852d634834ea5","text_start":14648},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":643,"page_start":587,"pdf_page":10,"printed_page":9,"text":"Schedule E 112 P25350 Total rents and royalties received","text_end":14758,"text_sha256":"b640f21795387c7dcee9b8b5eacf67e4afa2f4ea930fb4d85da852d634834ea5","text_start":14702},"name":"P25350","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total rents and royalties received","publisher_position":112,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":155,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":686,"page_start":644,"pdf_page":10,"printed_page":9,"text":"113 P25380 Rent/royalty interest expenses ","text_end":14747,"text_sha256":"682d74494bd432d1d5f49e4db91b96eed7a631d7c57717b5173c9069a7922f3a","text_start":14705},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":686,"page_start":644,"pdf_page":10,"printed_page":9,"text":"113 P25380 Rent/royalty interest expenses ","text_end":14801,"text_sha256":"682d74494bd432d1d5f49e4db91b96eed7a631d7c57717b5173c9069a7922f3a","text_start":14759},"name":"P25380","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty interest expenses","publisher_position":113,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":156,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":746,"page_start":687,"pdf_page":10,"printed_page":9,"text":"114 E25550 Total Depreciation and Depletion of all Property","text_end":14807,"text_sha256":"0ed56e992c2784b17db201b27c6061ca7d997a5bb32696270268b882c7b05e83","text_start":14748},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":746,"page_start":687,"pdf_page":10,"printed_page":9,"text":"114 E25550 Total Depreciation and Depletion of all Property","text_end":14861,"text_sha256":"0ed56e992c2784b17db201b27c6061ca7d997a5bb32696270268b882c7b05e83","text_start":14802},"name":"E25550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Depreciation and Depletion of all Property","publisher_position":114,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":157,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":797,"page_start":747,"pdf_page":10,"printed_page":9,"text":"115 P25700 Rent/royalty net income or loss (+/-) ","text_end":14858,"text_sha256":"bd63446a758ed0811529a953dd89e11eac58807a32a2837aa8c4022735a8b0b3","text_start":14808},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":797,"page_start":747,"pdf_page":10,"printed_page":9,"text":"115 P25700 Rent/royalty net income or loss (+/-) ","text_end":14912,"text_sha256":"bd63446a758ed0811529a953dd89e11eac58807a32a2837aa8c4022735a8b0b3","text_start":14862},"name":"P25700","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net income or loss (+/-)","publisher_position":115,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":158,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":831,"page_start":798,"pdf_page":10,"printed_page":9,"text":"116 E25820 Deductible rental loss","text_end":14892,"text_sha256":"9cb63df4b98225bfd3cb3e46ece699649bc9702fabb83f5777386461ce79d11b","text_start":14859},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":831,"page_start":798,"pdf_page":10,"printed_page":9,"text":"116 E25820 Deductible rental loss","text_end":14946,"text_sha256":"9cb63df4b98225bfd3cb3e46ece699649bc9702fabb83f5777386461ce79d11b","text_start":14913},"name":"E25820","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Deductible rental loss","publisher_position":116,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":159,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":866,"page_start":832,"pdf_page":10,"printed_page":9,"text":"117 E25850 Rent/royalty net income","text_end":14927,"text_sha256":"fa94a810b3593755108189021145c382351b0447afa5cfb9594bde8deda4a2e5","text_start":14893},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":866,"page_start":832,"pdf_page":10,"printed_page":9,"text":"117 E25850 Rent/royalty net income","text_end":14981,"text_sha256":"fa94a810b3593755108189021145c382351b0447afa5cfb9594bde8deda4a2e5","text_start":14947},"name":"E25850","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net income","publisher_position":117,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":160,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":899,"page_start":867,"pdf_page":10,"printed_page":9,"text":"118 E25860 Rent/royalty net loss","text_end":14960,"text_sha256":"6bde54987abb7b45eda76d5e9647fcf750298d941f9919bf63a0604a34b19d48","text_start":14928},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":899,"page_start":867,"pdf_page":10,"printed_page":9,"text":"118 E25860 Rent/royalty net loss","text_end":15014,"text_sha256":"6bde54987abb7b45eda76d5e9647fcf750298d941f9919bf63a0604a34b19d48","text_start":14982},"name":"E25860","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Rent/royalty net loss","publisher_position":118,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":161,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":944,"page_start":900,"pdf_page":10,"printed_page":9,"text":"Partnerships 119 E25940 Total passive income","text_end":15005,"text_sha256":"3db4b0dd75ddecf1bffa7e10d35822db4e1a6f86d1a1758db9ddacfb3a4b71ed","text_start":14961},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":944,"page_start":900,"pdf_page":10,"printed_page":9,"text":"Partnerships 119 E25940 Total passive income","text_end":15059,"text_sha256":"3db4b0dd75ddecf1bffa7e10d35822db4e1a6f86d1a1758db9ddacfb3a4b71ed","text_start":15015},"name":"E25940","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive income","publisher_position":119,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":162,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":980,"page_start":945,"pdf_page":10,"printed_page":9,"text":"120 E25980 Total non-passive income","text_end":15041,"text_sha256":"75021dd4dd31369b59b66d904b46f5b62ac6b0693fe6254531a197859051f047","text_start":15006},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":980,"page_start":945,"pdf_page":10,"printed_page":9,"text":"120 E25980 Total non-passive income","text_end":15095,"text_sha256":"75021dd4dd31369b59b66d904b46f5b62ac6b0693fe6254531a197859051f047","text_start":15060},"name":"E25980","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive income","publisher_position":120,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":163,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1010,"page_start":981,"pdf_page":10,"printed_page":9,"text":"121 E25920 Total passive loss","text_end":15071,"text_sha256":"0870768b18a3633a2023c645a78aa15aa9dd80b8032b7b31505c004329ee5e33","text_start":15042},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1010,"page_start":981,"pdf_page":10,"printed_page":9,"text":"121 E25920 Total passive loss","text_end":15125,"text_sha256":"0870768b18a3633a2023c645a78aa15aa9dd80b8032b7b31505c004329ee5e33","text_start":15096},"name":"E25920","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive loss","publisher_position":121,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":164,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1045,"page_start":1011,"pdf_page":10,"printed_page":9,"text":"122 E25960 Total non-passive loss ","text_end":15106,"text_sha256":"95bd8d56b53fd5087246f2535647572b80f35c81a3d1cfceeb37f02ef313d90f","text_start":15072},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1045,"page_start":1011,"pdf_page":10,"printed_page":9,"text":"122 E25960 Total non-passive loss ","text_end":15160,"text_sha256":"95bd8d56b53fd5087246f2535647572b80f35c81a3d1cfceeb37f02ef313d90f","text_start":15126},"name":"E25960","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive loss","publisher_position":122,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":165,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1099,"page_start":1046,"pdf_page":10,"printed_page":9,"text":"123 E26110 Partnership Section 179 expense deduction ","text_end":15160,"text_sha256":"8aab34c08db2c37001058efb2db7ed3bb4c768551e0f9ab9a90c8b6fa5aee250","text_start":15107},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1099,"page_start":1046,"pdf_page":10,"printed_page":9,"text":"123 E26110 Partnership Section 179 expense deduction ","text_end":15214,"text_sha256":"8aab34c08db2c37001058efb2db7ed3bb4c768551e0f9ab9a90c8b6fa5aee250","text_start":15161},"name":"E26110","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Partnership Section 179 expense deduction","publisher_position":123,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":166,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1137,"page_start":1100,"pdf_page":10,"printed_page":9,"text":"Small 124 E26170 Total passive income","text_end":15198,"text_sha256":"851d2824b95073f372ea07f402e7fb4f2f8bda41f8384e25a3282f88dce713e6","text_start":15161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1137,"page_start":1100,"pdf_page":10,"printed_page":9,"text":"Small 124 E26170 Total passive income","text_end":15252,"text_sha256":"851d2824b95073f372ea07f402e7fb4f2f8bda41f8384e25a3282f88dce713e6","text_start":15215},"name":"E26170","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive income","publisher_position":124,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":167,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1182,"page_start":1138,"pdf_page":10,"printed_page":9,"text":"Business 125 E26190 Total non-passive income","text_end":15243,"text_sha256":"747af42f61dced71b614799a3f5682ba4506b1d09b36957568d755e91b3aed49","text_start":15199},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1182,"page_start":1138,"pdf_page":10,"printed_page":9,"text":"Business 125 E26190 Total non-passive income","text_end":15297,"text_sha256":"747af42f61dced71b614799a3f5682ba4506b1d09b36957568d755e91b3aed49","text_start":15253},"name":"E26190","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive income","publisher_position":125,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":168,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1217,"page_start":1183,"pdf_page":10,"printed_page":9,"text":"Corp 126 E26160 Total passive loss","text_end":15278,"text_sha256":"1a777ab5ce50cf58d0422059c263de66fe7e1f724a26f9bc8ee14ec253b38066","text_start":15244},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1217,"page_start":1183,"pdf_page":10,"printed_page":9,"text":"Corp 126 E26160 Total passive loss","text_end":15332,"text_sha256":"1a777ab5ce50cf58d0422059c263de66fe7e1f724a26f9bc8ee14ec253b38066","text_start":15298},"name":"E26160","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive loss","publisher_position":126,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":169,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1252,"page_start":1218,"pdf_page":10,"printed_page":9,"text":"127 E26180 Total non-passive loss ","text_end":15313,"text_sha256":"048170ad209b195d4da910c9bc6eff93d10996098925e7d9c457ce4b07b04f36","text_start":15279},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1252,"page_start":1218,"pdf_page":10,"printed_page":9,"text":"127 E26180 Total non-passive loss ","text_end":15367,"text_sha256":"048170ad209b195d4da910c9bc6eff93d10996098925e7d9c457ce4b07b04f36","text_start":15333},"name":"E26180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total non-passive loss","publisher_position":127,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":170,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1324,"page_start":1253,"pdf_page":10,"printed_page":9,"text":"128 E26270 Combined partnership and S corporation net income/loss (+/-)","text_end":15385,"text_sha256":"31a1191bd5b7096b5d5e1f25b7c1bf70c014437c5ced1571411caf4321790779","text_start":15314},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1324,"page_start":1253,"pdf_page":10,"printed_page":9,"text":"128 E26270 Combined partnership and S corporation net income/loss (+/-)","text_end":15439,"text_sha256":"31a1191bd5b7096b5d5e1f25b7c1bf70c014437c5ced1571411caf4321790779","text_start":15368},"name":"E26270","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Combined partnership and S corporation net income/loss (+/-)","publisher_position":128,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":171,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1373,"page_start":1325,"pdf_page":10,"printed_page":9,"text":"129 E26100 S-corp. Section 179 expense deduction","text_end":15434,"text_sha256":"64da1fc754e9d2ddc722d9593e9f7457b8db6d95893ac8d2178120731f51903b","text_start":15386},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1373,"page_start":1325,"pdf_page":10,"printed_page":9,"text":"129 E26100 S-corp. Section 179 expense deduction","text_end":15488,"text_sha256":"64da1fc754e9d2ddc722d9593e9f7457b8db6d95893ac8d2178120731f51903b","text_start":15440},"name":"E26100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"S-corp. Section 179 expense deduction","publisher_position":129,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":172,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1413,"page_start":1374,"pdf_page":10,"printed_page":9,"text":"Estate or Trust 130 E26390 Total income","text_end":15474,"text_sha256":"2f0898262ee6425d4ee7d4033a22819627afbeec6ecd4a00fa73b3c0761161f6","text_start":15435},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1413,"page_start":1374,"pdf_page":10,"printed_page":9,"text":"Estate or Trust 130 E26390 Total income","text_end":15528,"text_sha256":"2f0898262ee6425d4ee7d4033a22819627afbeec6ecd4a00fa73b3c0761161f6","text_start":15489},"name":"E26390","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total income","publisher_position":130,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":173,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1435,"page_start":1414,"pdf_page":10,"printed_page":9,"text":"131 E26400 Total loss","text_end":15496,"text_sha256":"0dd69be8e4d8a255afd523dabd6c4dd2c9a4ce1145510ef6e5eb16f598679d8f","text_start":15475},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1435,"page_start":1414,"pdf_page":10,"printed_page":9,"text":"131 E26400 Total loss","text_end":15550,"text_sha256":"0dd69be8e4d8a255afd523dabd6c4dd2c9a4ce1145510ef6e5eb16f598679d8f","text_start":15529},"name":"E26400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total loss","publisher_position":131,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":174,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1493,"page_start":1436,"pdf_page":10,"printed_page":9,"text":"Farm Rental 132 E27200 Farm rent net income or loss (+/-)","text_end":15554,"text_sha256":"4d10ccb6bb7947db33cf75041f47bd5ce5139849f0e1a67fc1adb951537346db","text_start":15497},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1493,"page_start":1436,"pdf_page":10,"printed_page":9,"text":"Farm Rental 132 E27200 Farm rent net income or loss (+/-)","text_end":15608,"text_sha256":"4d10ccb6bb7947db33cf75041f47bd5ce5139849f0e1a67fc1adb951537346db","text_start":15551},"name":"E27200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Farm rent net income or loss (+/-)","publisher_position":132,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":175,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1575,"page_start":1494,"pdf_page":10,"printed_page":9,"text":"Schedule SE 133 E30400 Self-employment income subject to SS tax, primary taxpayer","text_end":15636,"text_sha256":"0bb1fa0b3d54c49628e277295abce0b052e2fae074957df9386626e1158c1b97","text_start":15555},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1575,"page_start":1494,"pdf_page":10,"printed_page":9,"text":"Schedule SE 133 E30400 Self-employment income subject to SS tax, primary taxpayer","text_end":15690,"text_sha256":"0bb1fa0b3d54c49628e277295abce0b052e2fae074957df9386626e1158c1b97","text_start":15609},"name":"E30400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment income subject to SS tax, primary taxpayer","publisher_position":133,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":176,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1647,"page_start":1576,"pdf_page":10,"printed_page":9,"text":"134 E30500 Self-employment income subject to SS tax, secondary taxpayer","text_end":15708,"text_sha256":"f03ef146af52bdb8e74f0e445fd123bb3398b9633eb21e61263dd8b86b9ca9dd","text_start":15637},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1647,"page_start":1576,"pdf_page":10,"printed_page":9,"text":"134 E30500 Self-employment income subject to SS tax, secondary taxpayer","text_end":15762,"text_sha256":"f03ef146af52bdb8e74f0e445fd123bb3398b9633eb21e61263dd8b86b9ca9dd","text_start":15691},"name":"E30500","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Self-employment income subject to SS tax, secondary taxpayer","publisher_position":134,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":177,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1701,"page_start":1648,"pdf_page":10,"printed_page":9,"text":"Form 2441 135 E32800 Qualifying individuals' Expenses","text_end":15762,"text_sha256":"a96204e1552bf8811d3c3ea05f88afad8fc1e452c183b45ceed6504ec1be5b1e","text_start":15709},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1701,"page_start":1648,"pdf_page":10,"printed_page":9,"text":"Form 2441 135 E32800 Qualifying individuals' Expenses","text_end":15816,"text_sha256":"a96204e1552bf8811d3c3ea05f88afad8fc1e452c183b45ceed6504ec1be5b1e","text_start":15763},"name":"E32800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Qualifying individuals' Expenses","publisher_position":135,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":178,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1746,"page_start":1702,"pdf_page":10,"printed_page":9,"text":"136 E33000 Expenses limited to earned income","text_end":15807,"text_sha256":"c179dcba5a5bfab5d99c5ca6fe984c9469bfacff81b0fa37ee43481843393703","text_start":15763},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1746,"page_start":1702,"pdf_page":10,"printed_page":9,"text":"136 E33000 Expenses limited to earned income","text_end":15861,"text_sha256":"c179dcba5a5bfab5d99c5ca6fe984c9469bfacff81b0fa37ee43481843393703","text_start":15817},"name":"E33000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Expenses limited to earned income","publisher_position":136,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":179,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1829,"page_start":1747,"pdf_page":10,"printed_page":9,"text":"Form 3800 137 E53240 Work Opportunity - Jobs (included in general business credit)","text_end":15890,"text_sha256":"04ffe18138bd89fc8e2ed9b6a255a172545bd7d10f8f1ee51d21fbcab4074f06","text_start":15808},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1829,"page_start":1747,"pdf_page":10,"printed_page":9,"text":"Form 3800 137 E53240 Work Opportunity - Jobs (included in general business credit)","text_end":15944,"text_sha256":"04ffe18138bd89fc8e2ed9b6a255a172545bd7d10f8f1ee51d21fbcab4074f06","text_start":15862},"name":"E53240","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Work Opportunity - Jobs (included in general business credit)","publisher_position":137,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":180,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1907,"page_start":1830,"pdf_page":10,"printed_page":9,"text":"138 E53280 Research and experimentation (included in general business credit)","text_end":15968,"text_sha256":"c7f398db6179e25e290cfa86be9a505ed073e18d379a91f7f2021c02ecda2d6e","text_start":15891},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1907,"page_start":1830,"pdf_page":10,"printed_page":9,"text":"138 E53280 Research and experimentation (included in general business credit)","text_end":16022,"text_sha256":"c7f398db6179e25e290cfa86be9a505ed073e18d379a91f7f2021c02ecda2d6e","text_start":15945},"name":"E53280","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Research and experimentation (included in general business credit)","publisher_position":138,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":181,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1944,"page_start":1908,"pdf_page":10,"printed_page":9,"text":"139 E53300 Low Income Housing Credit","text_end":16005,"text_sha256":"5e6ad4b9c3949946111e65220b5a39cc22a498f6addd0d9ea6c95dea26d0c50f","text_start":15969},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1944,"page_start":1908,"pdf_page":10,"printed_page":9,"text":"139 E53300 Low Income Housing Credit","text_end":16059,"text_sha256":"5e6ad4b9c3949946111e65220b5a39cc22a498f6addd0d9ea6c95dea26d0c50f","text_start":16023},"name":"E53300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Low Income Housing Credit","publisher_position":139,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":182,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":2004,"page_start":1945,"pdf_page":10,"printed_page":9,"text":"140 E53317 Employer Credit for Social Security Tax on Tips ","text_end":16065,"text_sha256":"3ff76b72ecc28a54974e1b4b03b09533be8fa6914d3045885eddaa50878c949f","text_start":16006},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":2004,"page_start":1945,"pdf_page":10,"printed_page":9,"text":"140 E53317 Employer Credit for Social Security Tax on Tips ","text_end":16119,"text_sha256":"3ff76b72ecc28a54974e1b4b03b09533be8fa6914d3045885eddaa50878c949f","text_start":16060},"name":"E53317","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Employer Credit for Social Security Tax on Tips","publisher_position":140,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":183,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":118,"page_start":64,"pdf_page":11,"printed_page":10,"text":"Form 4952 141 E58950 Total investment Interest Expense","text_end":16184,"text_sha256":"5e89a62c32c9b5f4d0f8c8c2e8125e4a9e36fec55e118a28236158a7dd203d2c","text_start":16130},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":119,"page_start":64,"pdf_page":11,"printed_page":10,"text":"Form 4952 141 E 58950 Total investment Interest Expense","text_end":16239,"text_sha256":"252b38b9a5704bde9efc197e7a808036d8834aff474a41ad3705e4283f762123","text_start":16184},"name":"E58950","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total investment Interest Expense","publisher_position":141,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":184,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":190,"page_start":119,"pdf_page":11,"printed_page":10,"text":"142 E58990 Investment income Elected Amount (Form 4952 part 2 line 4g) ","text_end":16256,"text_sha256":"64688151556142b16e63690f72f67a35deb45e35a60ee1ab53ccd532b8806c6d","text_start":16185},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":192,"page_start":120,"pdf_page":11,"printed_page":10,"text":"142 E58990 I nvestment income Elected Amount (Form 4952 part 2 line 4g) ","text_end":16312,"text_sha256":"d3db63182f82930facb1ceaf405f367af661295d6633fa5448402dbbc5e20522","text_start":16240},"name":"E58990","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Investment income Elected Amount (Form 4952 part 2 line 4g)","publisher_position":142,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":185,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":268,"page_start":191,"pdf_page":11,"printed_page":10,"text":"Form 6251 143 P60100 Net operating loss tax preference plus other adjustments","text_end":16334,"text_sha256":"072bb169590f7846991ef28bb423645172a6e15ce3586e22cbe40c2f3825db20","text_start":16257},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":271,"page_start":193,"pdf_page":11,"printed_page":10,"text":"Form 6251 143 P 60100 Net operating loss tax preference plus other adjustments","text_end":16391,"text_sha256":"2bf5756fbd6496a7e8be5558aa19f5086f021f4584a1539bf7c41e69c9d8c577","text_start":16313},"name":"P60100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Net operating loss tax preference plus other adjustments","publisher_position":143,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":186,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":320,"page_start":269,"pdf_page":11,"printed_page":10,"text":"144 P61850 Total adjustments and preferences (+/-) ","text_end":16386,"text_sha256":"255e12d7e57c6002eb605e9c2edcf9bdce94fb2920530a0b3acba4cd09108aa0","text_start":16335},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":324,"page_start":272,"pdf_page":11,"printed_page":10,"text":"144 P61850 T otal adjustments and preferences (+/-) ","text_end":16444,"text_sha256":"f287aeeaf4b188642d977a6cf4b59f27df24c4364a08068098c801bf0a78a4b3","text_start":16392},"name":"P61850","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total adjustments and preferences (+/-)","publisher_position":144,"publisher_signed_marker":true},{"admission":"not_selected","delivered_index":187,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":384,"page_start":321,"pdf_page":11,"printed_page":10,"text":"145 E60000 Form 1040 Taxable Income for AMT (Form 6251, Line 1)","text_end":16450,"text_sha256":"31fa42fd1d1eafc9f6a3d2739c76dd770faaf9f6c608b6a6d1c5622139248e40","text_start":16387},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":389,"page_start":325,"pdf_page":11,"printed_page":10,"text":"145 E60000 F orm 1040 Taxable Income for AMT (Form 6251, Line 1)","text_end":16509,"text_sha256":"d4379bb4e120357f1c29c8a73b4008da40ad5899f5865a5bcf49a6aa84a88c27","text_start":16445},"name":"E60000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Form 1040 Taxable Income for AMT (Form 6251, Line 1)","publisher_position":145,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":188,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":430,"page_start":385,"pdf_page":11,"printed_page":10,"text":"146 E62100 Alternative minimum taxable income","text_end":16496,"text_sha256":"28fa4b9e9eb924190bcfcb0b41a748f7a0c3aac07cafa84db6446302c55c3ac3","text_start":16451},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":436,"page_start":390,"pdf_page":11,"printed_page":10,"text":"146 E62100 A lternative minimum taxable income","text_end":16556,"text_sha256":"11a936b34f7f0873a3352cf0c705915559f906f357a39db2d7389cc9438c264f","text_start":16510},"name":"E62100","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum taxable income","publisher_position":146,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":189,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":477,"page_start":431,"pdf_page":11,"printed_page":10,"text":"147 E62900 Alternative tax foreign tax credit ","text_end":16543,"text_sha256":"62649650ef597f51c764fce34dc395c78434e5e3e0120253791be153408a0f52","text_start":16497},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":484,"page_start":437,"pdf_page":11,"printed_page":10,"text":"147 E62900 A lternative tax foreign tax credit ","text_end":16604,"text_sha256":"6975e8ed410762963bcdb26a7a3d41b8f26e0bbb373879003a3fb5938868f778","text_start":16557},"name":"E62900","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative tax foreign tax credit","publisher_position":147,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":190,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":551,"page_start":478,"pdf_page":11,"printed_page":10,"text":"148 E62720 Alternative minimum Schedule D less investment interest amount","text_end":16617,"text_sha256":"34af121bbf5475ac2597cde5cde08e116b6398e6dd1df8f4745a492aba43c2f1","text_start":16544},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":559,"page_start":485,"pdf_page":11,"printed_page":10,"text":"148 E62720 A lternative minimum Schedule D less investment interest amount","text_end":16679,"text_sha256":"e0266013087a7cef6f77c04b0e8bd989e94a0ea6a4f956e0c775c3e101dce065","text_start":16605},"name":"E62720","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum Schedule D less investment interest amount","publisher_position":148,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":191,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":624,"page_start":552,"pdf_page":11,"printed_page":10,"text":"149 E62730 Alternative minimum Schedule D unrecaptured section 1250 gain","text_end":16690,"text_sha256":"b127f9881cd80e952eba67db0ee44585c5db04b52462d3da20bd6950e8088590","text_start":16618},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":633,"page_start":560,"pdf_page":11,"printed_page":10,"text":"149 E62730 A lternative minimum Schedule D unrecaptured section 1250 gain","text_end":16753,"text_sha256":"9e994e3bb4620cce4c5b4ad7d645ac89a4573815be8bd18c566e18e13835bb0a","text_start":16680},"name":"E62730","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum Schedule D unrecaptured section 1250 gain","publisher_position":149,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":192,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":675,"page_start":625,"pdf_page":11,"printed_page":10,"text":"150 E62740 Alternative minimum capital gain amount","text_end":16741,"text_sha256":"3f0e4f8610f28fef1b1059da930b02ae1b9e53a3a5c8104e02ae8ce0aa62db5f","text_start":16691},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":685,"page_start":634,"pdf_page":11,"printed_page":10,"text":"150 E62740 A lternative minimum capital gain amount","text_end":16805,"text_sha256":"8983a99dd7ced7fb7c64d16cb1caac795b6ecb3f93749f45deea3acb2dcc59c3","text_start":16754},"name":"E62740","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Alternative minimum capital gain amount","publisher_position":150,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":193,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":722,"page_start":676,"pdf_page":11,"printed_page":10,"text":"Form 8582 151 P65300 Total passive net income ","text_end":16788,"text_sha256":"18c73ccd9957b717d5cad787a63b63ae1450bf1929efe9a2b2a6cebcdb82d8ca","text_start":16742},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":733,"page_start":686,"pdf_page":11,"printed_page":10,"text":"Form 8582 151 P 65300 Total passive net income ","text_end":16853,"text_sha256":"5df06c3a1dc98670573115f414b7a42a2f68765ed730fb9e975bd6ef51ff377b","text_start":16806},"name":"P65300","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive net income","publisher_position":151,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":194,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":754,"page_start":723,"pdf_page":11,"printed_page":10,"text":"152 P65400 Total passive losses","text_end":16820,"text_sha256":"55eab1e0a505311110697cecd36aea2ee3712b75b9ae79d346279ecf5c1b1e5c","text_start":16789},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":766,"page_start":734,"pdf_page":11,"printed_page":10,"text":"152 P65400 T otal passive losses","text_end":16886,"text_sha256":"6c793fc771664f415b4b0170f15271ace694dcf94fe2f5b77f999ba84aa50530","text_start":16854},"name":"P65400","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total passive losses","publisher_position":152,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":195,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":814,"page_start":755,"pdf_page":11,"printed_page":10,"text":"153 E68000 Total losses allowed from all passive activities","text_end":16880,"text_sha256":"eb2b6e50257d12b9c7f8dce236a0388c2b5d522d09343099dcc652a58c0f9760","text_start":16821},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":827,"page_start":767,"pdf_page":11,"printed_page":10,"text":"153 E68000 T otal losses allowed from all passive activities","text_end":16947,"text_sha256":"0076d8bc39224c5e88840043bee988c96f577dcb1f5d56d682c2ebb7546bb845","text_start":16887},"name":"E68000","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total losses allowed from all passive activities","publisher_position":153,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":196,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":871,"page_start":815,"pdf_page":11,"printed_page":10,"text":"Form 8801 154 E82200 Carry forward of minimum tax credit","text_end":16937,"text_sha256":"59aaf65f7d9bda290af1799c886cea58b219ea0fd131e6782c487981a8978460","text_start":16881},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":885,"page_start":828,"pdf_page":11,"printed_page":10,"text":"Form 8801 154 E 82200 Carry forward of minimum tax credit","text_end":17005,"text_sha256":"7114e0e1beb6c8ddb0bb3090feafca9f7e34b1196d374bd258ead395a915c158","text_start":16948},"name":"E82200","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Carry forward of minimum tax credit","publisher_position":154,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":197,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":913,"page_start":872,"pdf_page":11,"printed_page":10,"text":"Schedule J 155 T27800 Elected Farm Income","text_end":16979,"text_sha256":"be7feb33b6d1bb698b7471b0eb76d53828ff8fdd71ec3a873c5350a35e04fcb2","text_start":16938},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":928,"page_start":886,"pdf_page":11,"printed_page":10,"text":"Schedule J 155 T27800 E lected Farm Income","text_end":17048,"text_sha256":"c9f5371d024a6b00856e029ebce7efd258e90c37bf3b878e22de4dbd618f2529","text_start":17006},"name":"T27800","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Elected Farm Income","publisher_position":155,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":198,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":961,"page_start":914,"pdf_page":11,"printed_page":10,"text":"156 S27860 Tentative Current and Prior-Year Tax","text_end":17027,"text_sha256":"88b7f1492af50b6ab6fb9e5b1515a91eb1f44c9cfcd9c6e496b9540140cfc578","text_start":16980},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":977,"page_start":929,"pdf_page":11,"printed_page":10,"text":"156 S27860 T entative Current and Prior-Year Tax","text_end":17097,"text_sha256":"f13162d3346d786e547e854b719133cad86374235ef58655b75113d136ed2782","text_start":17049},"name":"S27860","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Tentative Current and Prior-Year Tax","publisher_position":156,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":199,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":994,"page_start":962,"pdf_page":11,"printed_page":10,"text":"157 P27895 Actual Prior-Year Tax","text_end":17060,"text_sha256":"76e1d2ada7d1619e48cc36189dd7d1b0675a5b9cda2ee63c7a528992b04a6393","text_start":17028},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1011,"page_start":978,"pdf_page":11,"printed_page":10,"text":"157 P27895 A ctual Prior-Year Tax","text_end":17131,"text_sha256":"e4171a351ea18aef659363e6a5e321d54382cfbfc5532f04b470eedbc3bac085","text_start":17098},"name":"P27895","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Actual Prior-Year Tax","publisher_position":157,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":200,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1055,"page_start":995,"pdf_page":11,"printed_page":10,"text":"Form 8863 158 P87482 American Opportunity Qualified Expenses","text_end":17121,"text_sha256":"65ce2c35ea2c651a1777a474cfab02698be506018fd268503663d1b328df8e19","text_start":17061},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1073,"page_start":1012,"pdf_page":11,"printed_page":10,"text":"Form 8863 158 P 87482 American Opportunity Qualified Expenses","text_end":17193,"text_sha256":"291f4dc96e2a1af58ea03ffc7d0bee29690136da12c38562a3ae8feccc3ee8b9","text_start":17132},"name":"P87482","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"American Opportunity Qualified Expenses","publisher_position":158,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":201,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1094,"page_start":1056,"pdf_page":11,"printed_page":10,"text":"159 E87521 American Opportunity Credit","text_end":17160,"text_sha256":"66fbe1d0a2afb1dabca85e4e06815cc7cd9e492ab8498550ea5ed8faa77d3858","text_start":17122},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1113,"page_start":1074,"pdf_page":11,"printed_page":10,"text":"159 E87521 A merican Opportunity Credit","text_end":17233,"text_sha256":"c31dea021b1fb8ff758fc124873c7bb68799d02c8c7a1be6860f550ce7e497d0","text_start":17194},"name":"E87521","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"American Opportunity Credit","publisher_position":159,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":202,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1148,"page_start":1095,"pdf_page":11,"printed_page":10,"text":"160 E87530 Lifetime Learning Total Qualified Expenses","text_end":17214,"text_sha256":"d93b29ad481354864195c98bbfa19935d385701e17bb4f92b4dd6eefc74335b4","text_start":17161},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1168,"page_start":1114,"pdf_page":11,"printed_page":10,"text":"160 E87530 Li fetime Learning Total Qualified Expenses","text_end":17288,"text_sha256":"d042b095d6c9b805cf0d86e25a046271deb88169d5962376618b0969bcabfe18","text_start":17234},"name":"E87530","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Lifetime Learning Total Qualified Expenses","publisher_position":160,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":203,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1184,"page_start":1149,"pdf_page":11,"printed_page":10,"text":"161 E87550 Lifetime Learning Credit","text_end":17250,"text_sha256":"74be56787cde243473bd5253de4762f94df31a11763b059c0598773b48d57887","text_start":17215},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1205,"page_start":1169,"pdf_page":11,"printed_page":10,"text":"161 E87550 Li fetime Learning Credit","text_end":17325,"text_sha256":"7c06009e6df97fb8f375ef343e2c2b6fa21ca97a4ef5c5b9fb856d43efccdf7a","text_start":17289},"name":"E87550","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Lifetime Learning Credit","publisher_position":161,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":204,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1226,"page_start":1185,"pdf_page":11,"printed_page":10,"text":"Form 8888 162 P86421 Bond Purchase Amount","text_end":17292,"text_sha256":"1c2b0f09f1951160e78c576d8ee59ab6a183e80b66d47414c3709ebc973f89c7","text_start":17251},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1248,"page_start":1206,"pdf_page":11,"printed_page":10,"text":"Form 8888 162 P 86421 Bond Purchase Amount","text_end":17368,"text_sha256":"31952dc713df089ee582a7bace50bba79e99272644300f7feb558984e98dfb0a","text_start":17326},"name":"P86421","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Bond Purchase Amount","publisher_position":162,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":205,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1320,"page_start":1227,"pdf_page":11,"printed_page":10,"text":"Form 8960 163 E85050 Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","text_end":17386,"text_sha256":"397c4f3ddfe481b56785ca2125f8e8f1f95ec3355786e99e3b444370cf4c17b7","text_start":17293},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1343,"page_start":1249,"pdf_page":11,"printed_page":10,"text":"Form 8960 163 E 85050 Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","text_end":17463,"text_sha256":"bb78df6edcc7919f42e04f398ffa426c1836b5bb2070168808d9efb00f0c53ef","text_start":17369},"name":"E85050","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Rental Real Estate, Royalties, Partnerships, S Corps, Trusts, Etc.","publisher_position":163,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":206,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1383,"page_start":1321,"pdf_page":11,"printed_page":10,"text":"164 E85090 Total Net Gain or Loss from Disposition of Property","text_end":17449,"text_sha256":"705c9e4cb724c99ef271546ea78f83e2ad27363bdf949d876558514e8b011bd8","text_start":17387},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1407,"page_start":1344,"pdf_page":11,"printed_page":10,"text":"164 E85090 T otal Net Gain or Loss from Disposition of Property","text_end":17527,"text_sha256":"9e628010b94bb4b33247d774d7562678f0461a14df56a8054f4dc88d244b146c","text_start":17464},"name":"E85090","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Net Gain or Loss from Disposition of Property","publisher_position":164,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":207,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1418,"page_start":1384,"pdf_page":11,"printed_page":10,"text":"165 E85120 Total Investment Income","text_end":17484,"text_sha256":"b16ed39d6b7e1169732dcd0bbf5d2e0340dc2a2af8018fcb5dcf8b2c42b04457","text_start":17450},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1443,"page_start":1408,"pdf_page":11,"printed_page":10,"text":"165 E85120 T otal Investment Income","text_end":17563,"text_sha256":"08a6dcc6ce09f63f41d681df1a9779a955206f5cefe01f0b0917d922238e59f4","text_start":17528},"name":"E85120","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Investment Income","publisher_position":165,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":208,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1464,"page_start":1419,"pdf_page":11,"printed_page":10,"text":"166 E85180 Total Deductions and Modifications","text_end":17530,"text_sha256":"ef9ab37b928d5a8d1af456371c753cc753ad7dd9a919ab22dc00d9661583b013","text_start":17485},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1490,"page_start":1444,"pdf_page":11,"printed_page":10,"text":"166 E85180 T otal Deductions and Modifications","text_end":17610,"text_sha256":"d0810d5f09a52dcab3abce22b6829dd5984c9234136efefaae7ee89ff788fd2d","text_start":17564},"name":"E85180","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Deductions and Modifications","publisher_position":166,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":209,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1534,"page_start":1465,"pdf_page":11,"printed_page":10,"text":"Form 8962 167 E85570 Dependents Modified Adjusted Gross Income Amount","text_end":17600,"text_sha256":"943aca30c20f5fb020ff080c7c43742bceebed742c39de678fd67ff9a0b33736","text_start":17531},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1561,"page_start":1491,"pdf_page":11,"printed_page":10,"text":"Form 8962 167 E 85570 Dependents Modified Adjusted Gross Income Amount","text_end":17681,"text_sha256":"32701ca71a4560d09f067877f6f3b79b04a0a0e27af16fffd95a619b8ee9a669","text_start":17611},"name":"E85570","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Dependents Modified Adjusted Gross Income Amount","publisher_position":167,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":210,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1584,"page_start":1535,"pdf_page":11,"printed_page":10,"text":"168 E85595 Annual Contribution Health Care Amount","text_end":17650,"text_sha256":"6018e18cd7a2e2470fa74e6a434f1e289dd47ce2729efae65a6414d129dc6168","text_start":17601},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1612,"page_start":1562,"pdf_page":11,"printed_page":10,"text":"168 E85595 A nnual Contribution Health Care Amount","text_end":17732,"text_sha256":"152389beefb4a57a1c1ed6d947d489ba80d3d2e34444ae7f57bd2cb66e13140d","text_start":17682},"name":"E85595","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Annual Contribution Health Care Amount","publisher_position":168,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":211,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1635,"page_start":1585,"pdf_page":11,"printed_page":10,"text":"169 E85600 Monthly Contribution Health Care Amount","text_end":17701,"text_sha256":"4dde94616b6352f9830a0a54c108b59b6ee419c3bd594eb6e94d54a43a86ff6e","text_start":17651},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1664,"page_start":1613,"pdf_page":11,"printed_page":10,"text":"169 E85600 M onthly Contribution Health Care Amount","text_end":17784,"text_sha256":"bc44b59cab85460cf9cb8813099019cf662c2e34404112d0a990d1b8ac4d157d","text_start":17733},"name":"E85600","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Monthly Contribution Health Care Amount","publisher_position":169,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":212,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1678,"page_start":1636,"pdf_page":11,"printed_page":10,"text":"170 E85770 Total Premium Tax Credit Amount","text_end":17744,"text_sha256":"64606a7da5a48e73d24e89218ef70e9ae31fd68c89ccc0d8d3c54884760cfcbf","text_start":17702},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1708,"page_start":1665,"pdf_page":11,"printed_page":10,"text":"170 E85770 T otal Premium Tax Credit Amount","text_end":17828,"text_sha256":"793741882a24cee3341e5eb553e9c81fa8166f8b95ff5355390ebd0446cc47dc","text_start":17785},"name":"E85770","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Total Premium Tax Credit Amount","publisher_position":170,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":213,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1723,"page_start":1679,"pdf_page":11,"printed_page":10,"text":"171 E85775 Advance Premium Tax Credit Amount","text_end":17789,"text_sha256":"be57870e63e21ecc92ee3c7859b22f870e9c8a02fee6d6eadf1829dd0b3cfc7e","text_start":17745},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1754,"page_start":1709,"pdf_page":11,"printed_page":10,"text":"171 E85775 A dvance Premium Tax Credit Amount","text_end":17874,"text_sha256":"4ce5b76a95041589061900712ea4b6097f4b65db7603909770ef8ae95e62f041","text_start":17829},"name":"E85775","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Advance Premium Tax Credit Amount","publisher_position":171,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":214,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1779,"page_start":1724,"pdf_page":11,"printed_page":10,"text":"172 E85785 Excess Advance Payment of Premium Tax Credit","text_end":17845,"text_sha256":"682b60d37c6a051d28c738cbaadcaa27355e5a80cbbd6fac8b118e0c2b81aabd","text_start":17790},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1811,"page_start":1755,"pdf_page":11,"printed_page":10,"text":"172 E85785 E xcess Advance Payment of Premium Tax Credit","text_end":17931,"text_sha256":"8c7824521a2c4f39d248d9788e4864c655682ec4943e4af2413b8598c0779a89","text_start":17875},"name":"E85785","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Excess Advance Payment of Premium Tax Credit","publisher_position":172,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":215,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1818,"page_start":1780,"pdf_page":11,"printed_page":10,"text":"173 E85790 Repayment Limitation Amount","text_end":17884,"text_sha256":"e4e51fa05b88a4444696529e93b4af60cc38004e38683eacdf8775813eef7994","text_start":17846},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1851,"page_start":1812,"pdf_page":11,"printed_page":10,"text":"173 E85790 R epayment Limitation Amount","text_end":17971,"text_sha256":"b5717343afbdf2ffd509aa66483e329aa9f0299f74c8abc8949009e53cbe3f6d","text_start":17932},"name":"E85790","not_selected_reason":"amount_field_outside_this_increment","publisher_group":"amount_or_large_code","publisher_label":"Repayment Limitation Amount","publisher_position":173,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":216,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1853,"page_start":1819,"pdf_page":11,"printed_page":10,"text":"Misc Codes 174 RECID Return ID [1]","text_end":17919,"text_sha256":"c31ab3dc86007f00497fbcbe6a2300dae6ad9fd81d1cd737b9649e9f9935afe9","text_start":17885},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"one_interior_space_from_pdf_text_extraction","footnote_markers":null,"page_end":1887,"page_start":1852,"pdf_page":11,"printed_page":10,"text":"Misc Codes 174 RE CID Return ID [1]","text_end":18007,"text_sha256":"c8567954d56a543c67758acf509172bb52fac2f2c16a485af42ad10ea501a383","text_start":17972},"name":"RECID","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Return ID [1]","publisher_position":174,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":217,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1877,"page_start":1854,"pdf_page":11,"printed_page":10,"text":"175 S006 Decimal weight","text_end":17943,"text_sha256":"7c79365e7c8b795f4dab53c0bd2ba80b10d3f8e65097dd8280e677ebf43a3d77","text_start":17920},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1912,"page_start":1888,"pdf_page":11,"printed_page":10,"text":"175 S006 D ecimal weight","text_end":18032,"text_sha256":"be59a7d865776b309ea490873456224b74aa0ac325f30c52883f95b35bad43b1","text_start":18008},"name":"S006","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Decimal weight","publisher_position":175,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":218,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1899,"page_start":1878,"pdf_page":11,"printed_page":10,"text":"176 S008 Sample count","text_end":17965,"text_sha256":"821d80cd5cb84376b5b94f6e77402c96d9daf1cce9bb1f58ca0221951321c2c0","text_start":17944},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1935,"page_start":1913,"pdf_page":11,"printed_page":10,"text":"176 S008 S ample count","text_end":18055,"text_sha256":"a5e8730451f4fa4869912da3750da459fd10d0580567a20f2dc7128f6267d055","text_start":18033},"name":"S008","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Sample count","publisher_position":176,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":219,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1925,"page_start":1900,"pdf_page":11,"printed_page":10,"text":"177 S009 Population count","text_end":17991,"text_sha256":"e91ae26bc32c45d1a8ac73a3bcf09445274eefaf2a47c35f27838f072adeaaf4","text_start":17966},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1962,"page_start":1936,"pdf_page":11,"printed_page":10,"text":"177 S009 P opulation count","text_end":18082,"text_sha256":"9d6363dc4716f7e51d2a6d595c5d339f4d8faa5603f28e725e4b444fd64b4379","text_start":18056},"name":"S009","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Population count","publisher_position":177,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":220,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1947,"page_start":1926,"pdf_page":11,"printed_page":10,"text":"178 WSAMP Sample code","text_end":18013,"text_sha256":"c658f35e5c9eb6c0039ac9e4057240be468074bc93ca49b175c2ae9c4bc6cfae","text_start":17992},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":1985,"page_start":1963,"pdf_page":11,"printed_page":10,"text":"178 WSAMP S ample code","text_end":18105,"text_sha256":"ede3f968711699b9778a4d39dabd97d3cca4bc4e532a1c8634b768af4fcbbd21","text_start":18083},"name":"WSAMP","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Sample code","publisher_position":178,"publisher_signed_marker":false},{"admission":"not_selected","delivered_index":221,"layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":1974,"page_start":1948,"pdf_page":11,"printed_page":10,"text":"179 TXRT Marginal tax rate","text_end":18040,"text_sha256":"62f98fd3bdca207c721c1ddaeac8f9a6249fc3a102b323776336cf4832348708","text_start":18014},"layout_row_corroboration":{"booklet":"main_2023","code_token_spacing":"exact","footnote_markers":null,"page_end":2013,"page_start":1986,"pdf_page":11,"printed_page":10,"text":"179 TXRT M arginal tax rate","text_end":18133,"text_sha256":"9b89661798b6580b6bbd987fc7a91d8bd36dc8ef58e537144e5c2cf7b688b96e","text_start":18106},"name":"TXRT","not_selected_reason":"misc_code_not_an_amount","publisher_group":"amount_or_large_code","publisher_label":"Marginal tax rate","publisher_position":179,"publisher_signed_marker":false}],"corroborating_edition_tolerant_positions":[{"name":"P04470","position":32},{"name":"E04600","position":33},{"name":"E07180","position":42},{"name":"E06500","position":53},{"name":"E58950","position":141},{"name":"P60100","position":143},{"name":"P65300","position":151},{"name":"E82200","position":154},{"name":"P87482","position":158},{"name":"P86421","position":162},{"name":"E85050","position":163},{"name":"E85570","position":167},{"name":"RECID","position":174}],"delivered_header_width":222,"printed_order_matches_delivered_order":true},"not_selected_reasons":{"amount_field_outside_this_increment":"A publisher amount field that this bounded increment does not project. It stays source-only: no unit, sign or period claim is made about it here, and nothing downstream may treat its absence from the projection as a statement about its value.","code_field_not_an_amount":"The publisher's CORE RECORD LAYOUT lists this name under 'Code Fields'. It is a code, a count or an indicator, never a dollar amount.","misc_code_not_an_amount":"The publisher's own 'Misc Codes' block sits inside the amount-field list and holds a return identifier, a weight with implied decimals, two sample counts, a sample code and a marginal tax rate with an implied decimal point. Membership of the amount list is therefore not evidence that a column is money."},"notable_source_only":{"E00100":"Adjusted gross income (deficit). The reviewed growth slice already retains source-year AGI as provenance; projecting it here would create a second, competing source of the same quantity.","E01100":"Capital gain distributions reported on Form 1040. A capital-gains component this increment does not project, because no publisher statement composes it with E01000, P22250 or P23250.","E01400":"Taxable IRA distribution. Retirement-adjacent but a distinct publisher concept from pensions and annuities; not projected here.","E02000":"Schedule E net income or loss. Business-adjacent but a different publisher concept from Schedule C; not projected here.","E02400":"Gross Social Security benefits. Not projected in this increment.","E05800":"Income tax before credits: the publisher's own before-credits quantity, and the reason E06500 must not be read as one.","E06500":"Total income tax. A tax outcome of 2015 law applied to 2015 amounts. Aging it would not compute any later year's policy, and the archived SOI measure table binds this same code to both 'total income tax' and 'income tax before credits', a conflict a successor must settle. It stays source-year evidence.","E08800":"Income tax after credits (SOI). A tax outcome; source-only.","E10300":"Total tax liability (SOI). A tax outcome; source-only.","E22320":"Long term gain/loss from other forms on Schedule D. See E01100.","E22370":"Schedule D capital gain distributions. See E01100.","E25920":"Partnership total passive loss specifically, not a generic gross loss. The archived code path consumed this code without ever describing it; the label alone establishes no blank or negative decoder, so it stays source-only.","E26270":"Combined partnership and S corporation net income/loss. Not projected; the archived producer both sign-branched and AGI-fallback aged this code, which double-counted it.","E27200":"Farm rent net income or loss. A separate publisher row from the Schedule F farm result; not projected here.","E30400":"The record-layout row says 'primary taxpayer', but the publisher's own definition on the same booklet's detailed pages says the combined primary and secondary net earnings from self-employment *subject to Social Security tax*. The label and the definition disagree, and the definition governs. See the refused uncapped-SE derivation.","E30500":"The secondary taxpayer's capped amount. See E30400 and the refused uncapped-SE derivation."},"prefix_is_not_evidence":"No column's role is inferred from an E, P, S or T prefix or from membership of the amount-field list. RECID, S006, S008, S009, WSAMP and TXRT are inside that list and none is money; TXRT carries an implied decimal point."},"non_claims":{"aggregate_record_disaggregation":false,"amount_growth_or_restatement":false,"fit_or_calibration":false,"microdata_read_by_this_generator":false,"person_or_household_expansion":false,"policyengine_import":false,"release_eligibility":false,"target_year_frame_entry":false,"tax_outcome_projection":false,"weight_promotion":false},"projection":{"amounts_decoded":true,"column_count":12,"columns":[{"concept":"salaries_and_wages","concept_group":"wages","delivered_index":43,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":["multivariate_blurring","itemized_blurring_007","nonitemized_wage_noise"],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00200","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":172,"page_start":137,"pdf_page":7,"printed_page":6,"text":"INCOME 1 E00200 Salaries and wages ","text_end":9732,"text_sha256":"286479cc0c81f7b4bc03dbcea17d4f799c86e6eb4f254ade8161c46aca55f10c","text_start":9697},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Wages carry two documented edits beyond the universal ones: multivariate blurring wherever at least two of wages, state and local income taxes and real estate taxes are nonzero, and up to 10 percent uniform random noise on nonitemized returns sampled at 0.07 percent. A wage value is therefore an edited amount, not one taxpayer's reported wage."],"publisher_label":"Salaries and wages","publisher_position":1,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"taxable_interest_received","concept_group":"interest","delivered_index":44,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00300","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":199,"page_start":173,"pdf_page":7,"printed_page":6,"text":"2 E00300 Interest received","text_end":9759,"text_sha256":"219d03fee69e9a2c8be58b99ca788e01cf0f0aae1b87ad27a511f610db28831c","text_start":9733},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Interest received","publisher_position":2,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"relationship_to_tax_exempt_interest","statement":"Neither booklet states whether E00300 and E00400 partition interest or overlap. They are retained as two independent columns and never summed here."}]},{"concept":"tax_exempt_interest_income","concept_group":"interest","delivered_index":45,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00400","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":235,"page_start":200,"pdf_page":7,"printed_page":6,"text":"3 E00400 Tax-exempt interest income","text_end":9795,"text_sha256":"61c594df2c930155d4be0df33026fca240ada4827c7cc7f7f0f7c746ab251716","text_start":9760},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Tax-exempt interest income","publisher_position":3,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"relationship_to_taxable_interest","statement":"See E00300. No publisher statement partitions the two."}]},{"concept":"dividends_included_in_agi","concept_group":"dividends","delivered_index":46,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00600","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":270,"page_start":236,"pdf_page":7,"printed_page":6,"text":"4 E00600 Dividends included in AGI","text_end":9830,"text_sha256":"485b7d512cdb10d960ed97fe398f806578c56041833f9b1f0cb7547725f195aa","text_start":9796},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Dividends included in AGI","publisher_position":4,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"qualified_dividend_containment","statement":"Neither booklet states that E00650 is a part of E00600. The two are retained independently; no difference is computed."}]},{"concept":"qualified_dividends","concept_group":"dividends","delivered_index":47,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00650","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":299,"page_start":271,"pdf_page":7,"printed_page":6,"text":"5 E00650 Qualified Dividends","text_end":9859,"text_sha256":"4705f8eebef4e0266688c5d2b0b7c8bc241eca6fe39636f95e2ba68885d8051c","text_start":9831},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Qualified Dividends","publisher_position":5,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"qualified_dividend_containment","statement":"See E00600. Containment is not documented, so no ordinary dividend residual is derived."}]},{"concept":"schedule_c_net_profit_or_loss","concept_group":"business_income","delivered_index":50,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E00900","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":427,"page_start":361,"pdf_page":7,"printed_page":6,"text":"8 E00900 Business or profession (Schedule C) net profit/loss (+/-)","text_end":9987,"text_sha256":"d8cc8d4d299e8568244532205e7ed7258c364570cbd8dce81ef35e7b7ce76407","text_start":9921},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["The record layout prints the (+/-) marker, so negative values are documented. This is a net Schedule C result, not gross receipts, and it is not self-employment earnings subject to Social Security tax; see the refused E30400/E30500 derivation."],"publisher_label":"Business or profession (Schedule C) net profit/loss (+/-)","publisher_position":8,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"net_capital_gain_or_loss","concept_group":"capital_gains","delivered_index":51,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01000","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":467,"page_start":428,"pdf_page":7,"printed_page":6,"text":"9 E01000 Net capital gain or loss (+/-)","text_end":10027,"text_sha256":"496540711e148f39ab9cb6f3709c30f8c82b201f7b5486ce7bc7454eebd5050e","text_start":9988},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Marker-signed on the record layout row."],"publisher_label":"Net capital gain or loss (+/-)","publisher_position":9,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"No publisher statement composes E01000 out of P22250 and P23250, and the statutory capital loss limitation is not described in either booklet. The three columns are retained independently and never reconciled here."}]},{"concept":"total_pensions_and_annuities_received","concept_group":"pensions","delivered_index":55,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01500","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":648,"page_start":601,"pdf_page":7,"printed_page":6,"text":"13 E01500 Total pensions and annuities received","text_end":10208,"text_sha256":"376b86060dcb7bdb82b03d9222e420ab131e5c714d911b3ccafede38b6d509d2","text_start":10161},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Total pensions and annuities received","publisher_position":13,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"taxable_share_relationship","statement":"The labels say 'received' and 'included in AGI' but neither booklet defines E01700 as a part of E01500. No taxable share is derived."}]},{"concept":"pensions_and_annuities_included_in_agi","concept_group":"pensions","delivered_index":56,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E01700","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":697,"page_start":649,"pdf_page":7,"printed_page":6,"text":"14 E01700 Pensions and annuities included in AGI","text_end":10257,"text_sha256":"be9375282e0b5a5244416a7ff7838a246525e895291bd2b5541e899ad5171d46","text_start":10209},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":[],"publisher_label":"Pensions and annuities included in AGI","publisher_position":14,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"not_documented_unresolved","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"taxable_share_relationship","statement":"See E01500."}]},{"concept":"schedule_f_net_profit_or_loss","concept_group":"farm_income","delivered_index":58,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"E02100","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":786,"page_start":744,"pdf_page":7,"printed_page":6,"text":"16 E02100 Schedule F net profit/loss (+/-)","text_end":10346,"text_sha256":"c72eddc815454026fdb9b64f9f970f1153e0df6599bc3ef0d8770d665937342f","text_start":10304},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["Schedule F net profit or loss, marker-signed. Farm rent (E27200) is a separate publisher row and is not selected here."],"publisher_label":"Schedule F net profit/loss (+/-)","publisher_position":16,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":[],"negative_values":"documented_by_record_layout_marker","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":true},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[]},{"concept":"short_term_gains_less_losses_net_of_carryover","concept_group":"capital_gains","delivered_index":143,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"P22250","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":124,"page_start":57,"pdf_page":10,"printed_page":9,"text":"Schedule D 101 P22250 Short-term gains less losses net of carryover","text_end":14185,"text_sha256":"88fc87d07990ef01476e2cc37c31aeefbdb732742c0c11a215cd73af52fa664e","text_start":14118},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["The record-layout row prints no (+/-) marker, but the publisher's own definition is 'all short-term capital gains less losses ... except for the short-term capital loss carryover', which is a netted quantity. The marker's absence is recorded verbatim rather than treated as a nonnegative domain.","E22250 does not exist in the delivered layout or in either booklet. It is never aliased onto this column."],"publisher_label":"Short-term gains less losses net of carryover","publisher_position":101,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":["p22250_definition"],"negative_values":"documented_by_publisher_definition_text","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"See E01000."}]},{"concept":"long_term_gains_less_losses_net_of_carryover","concept_group":"capital_gains","delivered_index":146,"derived_combinations":"refused","disclosure_aggregate_rows":"retained_and_reported_separately; never a person, never disaggregated","dtype":"int64","edit_status":{"column_anchors":[],"universal_anchors":["rounding_grid","rebalancing","aggregate_records","not_one_taxpayer"]},"field":"P23250","fit_admission":"not_admitted_pending_basis_review","grammar":"ascii_decimal_signed_integer","layout_row":{"booklet":"main_2022","footnote_markers":null,"page_end":301,"page_start":246,"pdf_page":10,"printed_page":9,"text":"104 P23250 Long-term gains less losses net of carryover","text_end":14362,"text_sha256":"4b83e62edc2e013cbe9a3cf5f483fe2aef646471443d89b201fef4c47086c4da","text_start":14307},"lexical_width":13,"missingness":{"absent_demographic_row":"irrelevant_to_this_column; the projection is main-file left and every delivered main record is retained, matched or not","empty_token":"refused","empty_token_basis":"Neither booklet declares a missing-value sentinel or a blank convention for an amount field, so an empty token is a refusal to be preserved and reviewed, not a zero to be invented.","zero_basis_anchor":"rounding_grid","zero_meaning":"delivered_zero_in_edited_public_file"},"notes":["As P22250: netted by definition, unmarked on the layout row."],"publisher_label":"Long-term gains less losses net of carryover","publisher_position":104,"return_grain":{"anchors":["not_one_taxpayer","national_estimates_only"],"grain":"source_return","statement":"One delivered main record. Not a person, not a household, and not necessarily one taxpayer's return."},"sign":{"accepted_grammar":"optional_leading_minus_then_decimal_digits","definition_anchors":["p23250_definition"],"negative_values":"documented_by_publisher_definition_text","negative_zero":"refused","policy":"The parser accepts a signed token for every projected column and records the observed negative and zero counts. A missing (+/-) marker is preserved as a missing marker; it is never enforced as a nonnegative domain, and no absolute value is ever taken.","publisher_signed_marker":false},"source_period_status":{"anchors":["flpdyr_printed_range","fiscal_year_conversion"],"statement":"The cover identifies Tax Year 2015, the code definitions print FLPDYR 2011-2014, the disclosure section says returns filed for years earlier than Tax Year 2012 were removed, and the reviewed header audit observed 2012-2015. The row period is therefore unresolved and no growth, restatement or target-year use is admitted.","status":"unresolved_raw_FLPDYR"},"target_year_use":"refused_pending_independent_growth_decision","units":{"basis_anchors":["amount_field_width","rounding_grid"],"currency":"USD","field_width_digits":12,"implied_decimals":0,"note":"Every amount field is rounded by the publisher before release; nonzero amounts under $5 become $2 with the sign retained. Whole dollars is the delivered scale. A delivered zero is a value in the edited public file; the rounding rule does not prove the original return reported zero or rule out other disclosure edits.","scale":"whole_dollars"},"unresolved":[{"code":"composition_from_schedule_d_rows","statement":"See E01000."}]}],"concept_groups":["business_income","capital_gains","dividends","farm_income","interest","pensions","wages"],"grain":"source_return","period_semantics":"unresolved_raw_FLPDYR","row_policy":"every delivered main record is projected, including the 88,021 with no demographic row and the four disclosure aggregates","target_year_use":"refused_pending_independent_growth_decision"},"raw_source_definition":{"bytes":9706,"relationship":"This document extends the reviewed raw source definition. It restates none of that document's pins as its own authority and never modifies it; the artifact binds both.","schema":"microcosm.us.puf_2015_raw_source_definition","schema_version":1,"sha256":"c2ac73aa86b0b210e9919fd551f26c469098e6bade91bf8b1e03e1cf91e308e4"},"refused_derivations":{"uncapped_self_employment_earnings":{"anchors":["e30400_definition","e30500_definition"],"reason":"E30400 is the combined primary and secondary net earnings from self-employment *subject to Social Security tax* — the lesser of net Schedule SE earnings and the maximum amount subject to the tax — and E30500 is the secondary taxpayer's own capped amount. Subtracting one from the other yields a capped primary residual, not uncapped self-employment earnings, and grossing the difference up cannot recover what the cap removed.","refused_inputs":["E30400","E30500"]}},"refused_fields":{"E22250":{"policy":"never_aliased_to_P22250","reason":"The archived producer's screened-field handling and several downstream discussions treat E22250 and P22250 as one column. The delivered header carries P22250 only, and neither booklet prints E22250 anywhere. Normalising one onto the other would be a guess, so a declaration or a delivered header naming E22250 is a refusal instead.","state":"absent_from_delivered_layout_and_from_both_retained_booklets"}},"route":"packaged","schema":"microcosm.us.puf_2015_monetary_source_projection","schema_version":1,"source_conflicts":{"record_count_includes_header":{"anchors":["record_count_intro","record_count_technical"],"statement":"The technical description says 207,696 CSV records with the first record containing field names, implying 207,695 data records. The accepted raw-byte audit observed 207,696 data records plus a header. Preserve the measured count without silently reconciling this wording conflict.","status":"documentary_wording_unresolved"},"rounding_five_dollar_tie":{"anchors":["rounding_grid"],"statement":"The nearest-ten-dollar rule does not state how a five-dollar tie is resolved. A delivered zero does not identify reported zero, absence, nonapplicability or another edit.","status":"documentary_rounding_tie_unresolved"}},"sources":{"demographic":{"bytes":2225050,"codec":"us-puf-2015-demographic-raw-v1","data_records":119675,"delivered_header_width":7,"git_blob_sha1":"46c63e94d94a3662672304b60008dbd514b10e1b","header_record_canonical_sha256":"856cadc0522d2da8c2be67ac7cf6b44b257d97e5e725f9e352ee9fa1a403ccdf","sha256":"2d275cc405a1d0b254855628eabefe6c6484570ab789b2777a037ea604a82e81","source_name":"puf_2015_demographic"},"main":{"bytes":126034649,"codec":"us-puf-2015-main-raw-v1","data_records":207696,"delivered_header_width":222,"git_blob_sha1":"a9fa4cd5b4b3f93abaf2a068cf709620c88ea33b","header_record_canonical_sha256":"3652a41e23a9f51e5e1ffe5fa7bcbb05c89f8b2ba53eec789dcad026789c98f2","sha256":"0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df","source_name":"puf_2015_main"}}} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json new file mode 100644 index 000000000..22aeff056 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_2015_raw_source_definition.json @@ -0,0 +1 @@ +{"aggregates":{"demographic_row_policy":"tolerated_typed_and_tagged_publisher_aggregate","disaggregation":"never","documentary_pins":{"main_pdf_page":4,"printed_page":3,"second_main_pdf_page":11,"second_printed_page":10},"expected_records":4,"mars_code":0,"recids":[999996,999997,999998,999999]},"artifact":{"amounts_decoded":false,"grain":"source_return","owns_population_columns":false,"period_semantics":"unresolved_raw_FLPDYR","row_order":"delivered_main_file_order","rows":207696,"type_name":"microcosm.us.puf_2015_raw_return_status","type_version":1,"untyped_main_columns":216,"weight_units":"S006_hundredths"},"authority":"irs_soi_2015_public_use_file_local_delivery","canonical_header_hash_method":"sha256 of json.dumps(header_list, sort_keys=True, separators=(\",\",\":\"), ensure_ascii=True, allow_nan=False).encode(\"ascii\")","csv_profile":{"block_bytes":1048576,"bom_policy":"preserve_in_first_decoded_header_field_and_report; no repair","delimiter":",","doublequote":true,"encoding":"utf-8-strict","field_size_limit":131072,"header_field_cap":1024,"key_character_cap":64,"logical_record_character_cap":1048576,"newline":"","physical_line_character_cap":1048576,"quotechar":"\"","quoting":"QUOTE_MINIMAL","record_cap_per_file":1000000,"strict":true},"derived_fields":[{"dtype":"uint8","logical":"boolean flag stored as an unsigned byte","name":"disclosure_aggregate","rule":"RECID in the four publisher aggregate RECIDs; must agree with MARS 0"},{"absent_records_expected":88021,"dtype":"int8","name":"demographic_status","rule":"1 when a demographic record matched this RECID exactly, 0 when absent"}],"documentary_sources":{"demographic_booklet":{"code_pages_pdf":[5,6,7,8],"code_pages_printed":[12,13],"domain_extract_text_sha256":"d2074c3a2e2dabb6d25da4a9a62b8d9302e001cc5d86939edc0bdd09719d8c9a","pdf_bytes":345247,"pdf_pages":13,"pdf_sha256":"7bb623c7c7631e30eb620f0a522d74bff59487c1778d0b2023b2536cead5f38b","source_zero_meaning":"Not present inside a delivered demographic record for age/dependent/earnings fields; not equivalent to an unmatched demographic row. GENDER has no documented zero.","text_sha256":"d2074c3a2e2dabb6d25da4a9a62b8d9302e001cc5d86939edc0bdd09719d8c9a"}},"fields":[{"documentary_pins":{"demographic_pdf_page":5,"demographic_printed_pages":[8,10],"join_section_pdf_page":10,"join_section_printed_page":9,"main_pdf_page":4,"printed_page":11,"second_printed_page":25},"domain":null,"dtype":"int64","file":"main","grammar":"ascii_decimal_unsigned","lexical_width":64,"name":"RECID","role":"identity","sentinel":null,"unique":true},{"documentary_pins":{"main_pdf_page":14,"printed_page":13},"documented_conflict":"main booklet pdf 14 / printed 13 prints 2011-2014; recorded verbatim, never corrected","domain":[2012,2013,2014,2015],"domain_basis":"observed_in_root_verified_header_audit","dtype":"int16","file":"main","grammar":"ascii_decimal_unsigned","lexical_width":8,"name":"FLPDYR","role":"period","sentinel":null},{"documentary_pins":{"main_pdf_page":14,"printed_page":13},"domain":[12],"domain_basis":"observed_in_root_verified_header_audit","dtype":"int8","file":"main","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"FLPDMO","role":"period","sentinel":null},{"documentary_pins":{"main_pdf_page":16,"printed_page":15},"domain":[0,1,2,3,4],"dtype":"int8","file":"main","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"MARS","role":"status","sentinel":null},{"documentary_pins":{"main_pdf_page":13,"printed_page":12},"domain":[0,1],"dtype":"int8","file":"main","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"DSI","role":"status","sentinel":null},{"documentary_pins":{"main_pdf_page":25,"printed_page":24},"domain":null,"dtype":"int64","file":"main","grammar":"ascii_decimal_unsigned","grammar_note":"proposed nonnegative source contract; the header audit did not establish the complete S006 token domain, so a first genuine parse may refuse. Preserve the refusal; never widen silently.","implied_decimals":2,"lexical_width":32,"name":"S006","never_divided":true,"role":"weight","sentinel":null,"units":"hundredths"},{"documentary_pins":{"excerpt":{"text_end":4844,"text_sha256":"76f3cd08c8c5865969ead50505bd805d4b1bf378a0d5f4b0a239201466186447","text_start":3777}},"domain":[0,1,2,3,4,5,6],"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"AGEDP1","role":"demographic","sentinel":-1},{"documentary_pins":{"excerpt":{"text_end":6025,"text_sha256":"7d7d2b37434c5d05206a5e48108919a36a22995f937882f85ec180e7dfff50aa","text_start":4844}},"domain":[0,1,2,3,4,5,6],"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"AGEDP2","role":"demographic","sentinel":-1},{"documentary_pins":{"excerpt":{"text_end":7151,"text_sha256":"a5d002540f841c7967cfca01cf583f07f1a542b64f13e4359a029bd3b406b98d","text_start":6025}},"domain":[0,1,2,3,4,5,6],"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"AGEDP3","role":"demographic","sentinel":-1},{"conditional_domain":{"cases":{"0":[0,1,2,3,4,5,6],"1":[0,1,2,3]},"on":"DSI"},"documentary_pins":{"excerpt":{"text_end":8562,"text_sha256":"5c27974693ec7554ca3b9b9f064968f7aa37c54bbf1dc007f7b87e3c4c98a494","text_start":7151}},"domain":null,"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"AGERANGE","role":"demographic","sentinel":-1},{"documentary_pins":{"excerpt":{"text_end":9138,"text_sha256":"b1f5790f29d1e5d6e43aa9804f97caf5a5af820d89789c98966627cec8a47df7","text_start":8562}},"domain":[0,1,2,3],"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"EARNSPLIT","role":"demographic","sentinel":-1},{"documentary_pins":{"excerpt":{"text_end":9388,"text_sha256":"74dc411a43098003e982afe5e9c04775e8343bc5b58a2f1a090a2f502ae05b6c","text_start":9138}},"domain":[1,2],"dtype":"int8","file":"demographic","grammar":"ascii_decimal_unsigned","lexical_width":4,"name":"GENDER","role":"demographic","sentinel":-1,"zero_never_admitted":true}],"join":{"demographic_orphan_keys_allowed":false,"expected_main_unmatched_records":88021,"expected_matched_keys":119675,"grain":"source_return","leading_zero_spelling_never_merged":true,"method":"exact_lexical_RECID; main-left; no repair","missing_demographics_are_absent_not_zero":true,"numeric_key_uniqueness_checked":true},"non_claims":{"alias_repair":false,"amounts_decoded":false,"growth_factor_admission":false,"household_or_person_mapping":false,"period_resolution":false,"policyengine_import":false,"population_created":false,"release_eligibility":false},"route":"packaged","schema":"microcosm.us.puf_2015_raw_source_definition","schema_version":1,"sources":{"demographic":{"bytes":2225050,"codec":"us-puf-2015-demographic-raw-v1","data_records":119675,"delivered_header":["AGEDP1","AGEDP2","AGEDP3","AGERANGE","EARNSPLIT","GENDER","RECID"],"delivered_header_width":7,"documented_header_order_matches_delivered":true,"git_blob_sha1":"46c63e94d94a3662672304b60008dbd514b10e1b","header_record_canonical_sha256":"856cadc0522d2da8c2be67ac7cf6b44b257d97e5e725f9e352ee9fa1a403ccdf","leading_utf8_bom":false,"sha256":"2d275cc405a1d0b254855628eabefe6c6484570ab789b2777a037ea604a82e81","source_name":"puf_2015_demographic"},"main":{"bytes":126034649,"codec":"us-puf-2015-main-raw-v1","data_records":207696,"delivered_header":["AGIR1","CLAIM8965","DSI","EFI","EIC","ELECT","FDED","FLPDYR","FLPDMO","F2441","F3800","F6251","F8582","F8606","F8829","F8867","F8949","F8959","F8960","F8962","F8965","IE","MARS","MIDR","N24","N25","N30","PREP","PREMNTHS","SCHB","SCHCF","SCHE","TFORM","TXST","XFPT","XFST","XOCAH","XOCAWH","XOODEP","XOPAR","XTOT","XTOT8962","XTOT8965","E00200","E00300","E00400","E00600","E00650","E00700","E00800","E00900","E01000","E01100","E01200","E01400","E01500","E01700","E02000","E02100","E02300","E02400","E02500","E03150","E03210","E03220","E03230","E03260","E03270","E03240","E03290","E03300","E03400","E03500","E00100","P04470","E04600","E04800","E05100","E05200","E05800","E06000","E06200","E06300","E09600","E07180","E07200","E07220","E07230","E07240","E07260","E07300","E07400","E07600","P08000","E07150","E06500","E08800","E09400","E09700","E09730","E09740","E09750","E09800","E09900","E10300","E10700","E10900","E10960","E59560","E59680","E59700","E59720","E11550","E11560","E11561","E11562","E11563","E11070","E11100","E11200","E11300","E11400","E11601","E11602","E11603","E10605","E11900","E12000","E12200","E17500","E18400","E18500","E19200","E19550","E19800","E20100","E19700","E20550","E20600","E20400","E20800","E20500","E21040","P22250","E22320","E22370","P23250","E24515","E24516","E24518","E24560","E24598","E24615","E24570","P25350","P25380","E25550","P25700","E25820","E25850","E25860","E25940","E25980","E25920","E25960","E26110","E26170","E26190","E26160","E26180","E26270","E26100","E26390","E26400","E27200","E30400","E30500","E32800","E33000","E53240","E53280","E53300","E53317","E58950","E58990","P60100","P61850","E60000","E62100","E62900","E62720","E62730","E62740","P65300","P65400","E68000","E82200","T27800","S27860","P27895","P87482","E87521","E87530","E87550","P86421","E85050","E85090","E85120","E85180","E85570","E85595","E85600","E85770","E85775","E85785","E85790","RECID","S006","S008","S009","WSAMP","TXRT"],"delivered_header_width":222,"documented_header_order_matches_delivered":true,"git_blob_sha1":"a9fa4cd5b4b3f93abaf2a068cf709620c88ea33b","header_record_canonical_sha256":"3652a41e23a9f51e5e1ffe5fa7bcbb05c89f8b2ba53eec789dcad026789c98f2","leading_utf8_bom":false,"sha256":"0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df","source_name":"puf_2015_main"}}} From dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 11:23:09 -0400 Subject: [PATCH 018/255] Correct full65 attachment replay identity and regression coverage --- PROGRESS.md | 6 + .../us_runtime/graph_full_puf_enrichment.py | 28 +- .../test_us_graph_full_puf_enrichment.py | 273 ++++++++++++++---- 3 files changed, 236 insertions(+), 71 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index dea72ea44..e4a322c6d 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -776,3 +776,9 @@ Root reviews exact ordinary guard/source/resource admissions before execution. F ### Integration source-resource closure — 2026-09-09 Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. + +## Layer14 full65 replay correction — 2026-09-09 + +State: exact accepted two-file correction staged; successor57 integration execution pending. +Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. +Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py index 182825a83..1a5b370b1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py @@ -57,6 +57,12 @@ from microcosm.graph.keys import node_key, opaque_artifact_key, platform_fingerprint from microcosm.graph.population import Population +from . import ( + acs_income_universe, + operator_column_contracts, + qbi_inputs, + support_provenance, +) from . import full_puf_enrichment as full from . import survey_population_replay as replay @@ -351,14 +357,9 @@ def retain_full_puf_attachment( fit_nodes == expected_fits and apply_nodes == expected_applies, "FULL_PUF_CHAIN_DECLARATION", ) - # Structural patching labels all carried columns with the version owner; - # that runtime label is not a same-version declaration conflict. The real - # compiler checks inherited rewrite claims against the version's base. - require( - not _OUTPUT_COORDINATES - & {(o.entity, o.column) for o in population_node.outputs}, - "FULL_PUF_UPSTREAM_OUTPUT_CONFLICT", - ) + # Structural nodes have no ordinary output declarations. The compiler is + # the enforcer of same-version ownership conflicts and inherited rewrites; + # the integrating host must compile the entire graph before execution. for entity, name in MASKS.items(): require( name not in population.frame.table(entity), "FULL_PUF_MASK_ALREADY_PRESENT" @@ -393,9 +394,11 @@ def retain_full_puf_attachment( def _params(binding): + # The physical Population seals are in-process mutation checks only. + # Nullable backing storage beneath nulls is canonicalized by Frame-store + # v2. Graph column/frame keys bind the upstream values across sessions. return { "scope": SCOPE, - "upstream_state": binding.expected_stamp, "donor_values": binding.donor_stamp, "predictor_knownness": binding.known_stamp, "matrix_producer_key": binding.matrix.producer_key, @@ -566,7 +569,8 @@ def _checked_artifacts(binding, edges, artifacts, producer_keys=None): values["placement"].payload == _placement(binding), "FULL_PUF_PLACEMENT_BINDING", ) - for names in (("raw_064", "apply_state"), ("last_model", "training_state")): + last_raw = f"raw_{len(binding.apply_nodes) - 1:03d}" + for names in ((last_raw, "apply_state"), ("last_model", "training_state")): require( len({values[name].producer_key for name in names}) == 1, "FULL_PUF_SIBLING_PRODUCER", @@ -626,6 +630,10 @@ def implementation_hash(self): sys.modules[__name__], full, full.support, + support_provenance, + operator_column_contracts, + qbi_inputs, + acs_income_universe, replay, population_ops, store_ops, diff --git a/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py b/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py index 3f0e3f9b8..d5dc391d5 100644 --- a/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py +++ b/packages/microcosm-build/tests/test_us_graph_full_puf_enrichment.py @@ -109,6 +109,9 @@ def run(self, context): def _complete_frame(): frame = _recipient() + frame.table("tax_unit")["health_savings_account_ald"] = np.linspace( + 701.0, 1701.0, frame.n("tax_unit"), dtype=np.float64 + ) people = frame.table("person") people["self_employment_income_would_be_qualified"] = np.resize( np.array([False, True], dtype=bool), len(people) @@ -453,6 +456,21 @@ def test_full65_cold_and_required_replay_preserve_complete_population(attached_f case = attached_full65 mask_node, attach_node = case.nodes assert len(case.fits) == len(case.applies) == 65 + hsa = next( + o + for o in attach_node.outputs + if (o.entity, o.column) == ("tax_unit", "health_savings_account_ald") + ) + assert hsa.rewrite and hsa.dtype == "float64" + for result in case.results: + table = result.population.frame.table("tax_unit") + original = case.binding.expected_population.frame.table("tax_unit") + native = ~support.puf_tax_detail_clone_mask(original, entity="tax_unit") + pd.testing.assert_series_equal( + table.loc[native, "health_savings_account_ald"], + original.loc[native, "health_savings_account_ald"], + check_exact=True, + ) assert len(full.PERSON_OUTPUTS) == 56 and len(full.TAX_UNIT_OUTPUTS) == 9 assert "prior_year_wages" not in full.TARGETS assert "employment_income_last_year" not in full.TARGETS @@ -607,7 +625,12 @@ def test_actual_artifact_value_identity_and_sibling_mutations_are_refused( producer_keys[node_id] = "f" * 64 elif mutation == "producer_key": producer_keys[case.applies[0].id] = "e" * 64 - with pytest.raises(ValueError): + expected_refusal = ( + r"^Legacy QRF artifact content digest mismatch\.$" + if mutation == "model_bytes" + else "FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_" + ) + with pytest.raises(ValueError, match=expected_refusal): _verify(case, result, artifacts=artifacts, producer_keys=producer_keys) @@ -700,7 +723,7 @@ def test_complete_materialized_population_mutations_are_refused( elif mutation == "frame_mass_log": assert changed.frame.mass_log changed = _copy_population(changed, mass_log=changed.frame.mass_log[:-1]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_"): _verify(case, result, population=changed) @@ -737,7 +760,7 @@ def test_upstream_replay_checks_full_state_beyond_matrix_values( ).matrix == result.artifacts["matrix"].payload ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_"): _verify(case, result, upstream_population=changed) @@ -788,7 +811,7 @@ def test_retained_live_inputs_cannot_change_after_binding(attached_full65, mutat binding, input_owners={**binding.input_owners, ("person", "age"): "different.owner"}, ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_"): _verify(case, result, binding=binding) @@ -838,7 +861,7 @@ def test_boundary_requires_complete_chain_and_explicit_population_owner_binding( elif mutation == "incomplete_owner_map": arguments["input_owners"] = dict(arguments["input_owners"]) arguments["input_owners"].pop(("person", "age")) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_"): placement.retain_full_puf_attachment(**arguments) @@ -913,7 +936,7 @@ def test_complete_input_knownness_and_incumbent_physical_dtypes_are_required( ), ) arguments["expected_population"] = _copy_population(arguments["population"]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="FULL_PUF_|SURVEY_POPULATION_REPLAY_|PUF_"): placement.retain_full_puf_attachment(**arguments) @@ -937,67 +960,107 @@ def test_actual_manifest_and_compiled_producer_contract_mutations_are_refused( records = dict(manifest.nodes) node_id = case.fits[-1].id record = records[node_id] - # Constructors may themselves reject inconsistent typed ancestry. In the - # cases which remain structurally valid, the actual loader must reject the - # changed implementation/declaration against the retained complete chain. - with pytest.raises(ValueError): - if mutation == "kernel_ref": - records[node_id] = replace(record, kernel_ref="test.different.fit@1") - elif mutation == "kernel_impl": - records[node_id] = replace(record, kernel_impl_hash="e" * 64) - elif mutation == "producer_key": - records[node_id] = replace(record, key="e" * 64) - elif mutation in ("producer_descriptor_type", "consumer_descriptor_type"): - if mutation == "consumer_descriptor_type": - node_id = case.nodes[-1].id - record = records[node_id] - typed = { - direction: { - name: {**value, "type": dict(value["type"])} - for name, value in entries.items() - } - for direction, entries in record.typed_artifacts.items() + if mutation == "kernel_ref": + records[node_id] = replace(record, kernel_ref="test.different.fit@1") + elif mutation == "kernel_impl": + records[node_id] = replace(record, kernel_impl_hash="e" * 64) + elif mutation == "producer_key": + records[node_id] = replace(record, key="e" * 64) + elif mutation in ("producer_descriptor_type", "consumer_descriptor_type"): + if mutation == "consumer_descriptor_type": + node_id = case.nodes[-1].id + record = records[node_id] + typed = { + direction: { + name: {**value, "type": dict(value["type"])} + for name, value in entries.items() } - direction, name = ( - ("outputs", "model") - if mutation == "producer_descriptor_type" - else ("inputs", "last_model") + for direction, entries in record.typed_artifacts.items() + } + direction, name = ( + ("outputs", "model") + if mutation == "producer_descriptor_type" + else ("inputs", "last_model") + ) + typed[direction][name]["type"]["name"] = "test.different.model" + records[node_id] = replace(record, typed_artifacts=typed) + elif mutation == "compiled_fit_params": + fit = case.fits[-1] + changed = replace(fit, params={**fit.params, "n_estimators": 3}) + compiled = compile_graph( + replace( + compiled.graph, + nodes=tuple( + changed if node.id == fit.id else node + for node in compiled.graph.nodes + ), ) - typed[direction][name]["type"]["name"] = "test.different.model" - records[node_id] = replace(record, typed_artifacts=typed) - elif mutation == "compiled_fit_params": - fit = case.fits[-1] - changed = replace(fit, params={**fit.params, "n_estimators": 3}) - compiled = compile_graph( - replace( - compiled.graph, - nodes=tuple( - changed if node.id == fit.id else node - for node in compiled.graph.nodes - ), - ) + ) + elif mutation == "compiled_apply_edge": + apply = case.applies[-1] + changed = replace( + apply, + artifact_inputs=tuple( + replace(edge, producer=case.applies[1].id) + if edge.name == "prior_000" + else edge + for edge in apply.artifact_inputs + ), + ) + compiled = compile_graph( + replace( + compiled.graph, + nodes=tuple( + changed if node.id == apply.id else node + for node in compiled.graph.nodes + ), ) - elif mutation == "compiled_apply_edge": - apply = case.applies[-1] - changed = replace( - apply, - artifact_inputs=tuple( - replace(edge, producer=case.applies[1].id) - if edge.name == "prior_000" - else edge - for edge in apply.artifact_inputs + ) + if mutation in ( + "producer_key", + "producer_descriptor_type", + "consumer_descriptor_type", + ): + # Make both ends of typed ancestry internally consistent. Construction + # must succeed; the owned loader then rejects the false graph contract. + producer_id = case.fits[-1].id + producer = records[producer_id] + changed_type = ArtifactType("test.different.model", 1) + outputs = { + name: descriptor( + producer=producer_id, + artifact=name, + type_=( + changed_type + if name == "model" and mutation != "producer_key" + else ArtifactType( + value["type"]["name"], value["type"]["schema_version"] + ) ), + producer_key=producer.key, + capabilities=producer.capabilities, ) - compiled = compile_graph( - replace( - compiled.graph, - nodes=tuple( - changed if node.id == apply.id else node - for node in compiled.graph.nodes - ), - ) + for name, value in producer.typed_artifacts["outputs"].items() + } + records[producer_id] = replace( + producer, + typed_artifacts={**producer.typed_artifacts, "outputs": outputs}, + opaque_artifacts={name: value["key"] for name, value in outputs.items()}, + ) + for consumer_id, consumer in tuple(records.items()): + if not consumer.typed_artifacts: + continue + inputs = { + name: outputs[value["artifact"]] + if value["producer"] == producer_id + else value + for name, value in consumer.typed_artifacts["inputs"].items() + } + records[consumer_id] = replace( + consumer, typed_artifacts={**consumer.typed_artifacts, "inputs": inputs} ) - manifest = replace(manifest, nodes=records) + manifest = replace(manifest, nodes=records) + with pytest.raises(ValueError, match="FULL_PUF_"): placement.load_full_puf_attachment_artifacts( case.binding, compiled=compiled, manifest=manifest, store=case.store ) @@ -1020,3 +1083,91 @@ def test_person_and_tax_unit_masks_must_describe_the_same_memberships(attached_f # mask-membership invariant, rather than a changed-state seal, must reject it. with pytest.raises(ValueError, match="FULL_PUF_MASK_MEMBERSHIP"): placement.retain_full_puf_attachment(**arguments) + + +def test_rebuilt_binding_from_fresh_frame_store_requires_full_cached_replay( + attached_full65, tmp_path +): + """Nullable physical mutation seals cannot become cross-session graph keys.""" + case = attached_full65 + original = case.retention_arguments + cold_expected = _copy_population(original["expected_population"]) + # Exercise the real codec's difference between masked storage and value. + hidden = cold_expected.frame.table("person")["unrelated_nullable"].array + nulls = hidden._mask + assert nulls.any() + hidden._data[nulls] = True + assert hidden._data[nulls].all() + cold_binding = placement.retain_full_puf_attachment( + **{**original, "expected_population": cold_expected} + ) + cold_nodes = placement.full_puf_attachment_nodes(cold_binding) + assert cold_nodes == case.nodes + + retained_path = tmp_path / "new-session-upstream" + writer = ContentStore(retained_path) + frame_key = full.codec.sha(b"rebuilt-full65-upstream") + writer.put_frame(frame_key, cold_expected.frame) + # A distinct store object reads actual codec output; no in-memory reuse. + reader = ContentStore(retained_path) + restored = reader.load_frame(frame_key) + assert not restored.person["unrelated_nullable"].array._data[nulls].any() + replay.same_replayed_frame(cold_expected.frame, restored) + warm_expected = replace(_copy_population(cold_expected), frame=restored) + warm_upstream = _copy_population(case.results[-1].upstream) + warm_binding = placement.retain_full_puf_attachment( + **{ + **original, + "population": warm_upstream, + "expected_population": warm_expected, + "input_owners": dict(warm_upstream.owners), + } + ) + assert cold_binding.expected_stamp != warm_binding.expected_stamp + warm_nodes = placement.full_puf_attachment_nodes(warm_binding) + assert warm_nodes == cold_nodes + assert placement._placement(warm_binding) == placement._placement(cold_binding) + rebuilt_graph = compile_graph( + replace( + case.compiled.graph, + nodes=tuple( + {node.id: node for node in warm_nodes}.get(node.id, node) + for node in case.compiled.graph.nodes + ), + ) + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + placement.FullPufMaskKernel(warm_binding), + placement.FullPufAttachKernel(warm_binding), + ): + kernels.register(kernel) + observed = {} + manifest = run_graph( + rebuilt_graph, + sources=case.sources, + store=case.store, + kernels=kernels, + resume="require", + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + assert all(record.hit for record in manifest.nodes.values()) + assert manifest.key == case.results[-1].manifest.key + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + warm_binding, compiled=rebuilt_graph, manifest=manifest, store=case.store + ) + placement.verify_materialized_full_puf_attachment( + warm_binding, + upstream_population=observed[warm_binding.population_node.id], + population=observed[warm_nodes[-1].id], + artifacts=artifacts, + producer_keys=producer_keys, + ) + replay.same_replayed_population( + case.results[-1].population, observed[warm_nodes[-1].id] + ) From a85c9cbb79081a980e7d3afe8916c80153de5bf8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 11:33:16 -0400 Subject: [PATCH 019/255] Prepare US graph build consolidation for public review --- PROGRESS.md | 6 + docs/us-launch-review.md | 104 ++++++++++++++++++ .../us_runtime/acs_housing_universe_source.py | 4 +- .../us_runtime/acs_native_coverage_binding.py | 75 ++++++++++--- .../microcosm/build/us_runtime/acs_pums.py | 18 +-- .../us_runtime/graph_composed_asec_binding.py | 6 +- .../graph_composed_asec_measures.py | 2 + .../build/us_runtime/multispine_pool.py | 9 +- .../build/us_runtime/puf_detail_transfer.py | 1 + .../us_runtime/puf_diagnostic_consumer.py | 1 + .../survey_population_preparation.py | 4 + .../tests/test_us_current_survey_puf_host.py | 11 +- .../tests/test_us_runtime_facade_union.py | 1 - .../tests/test_us_survey_catalogue_digest.py | 3 +- ...st_us_survey_catalogue_digest_authority.py | 3 +- ...st_us_survey_catalogue_digest_fast_path.py | 1 + .../src/microcosm/graph/attachments.py | 1 - 17 files changed, 209 insertions(+), 41 deletions(-) create mode 100644 docs/us-launch-review.md diff --git a/PROGRESS.md b/PROGRESS.md index e4a322c6d..78fd642ad 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -782,3 +782,9 @@ Root preflight found seven JSON source definitions declared by the ordinary guar State: exact accepted two-file correction staged; successor57 integration execution pending. Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. + +## Source review publication — 2026-09-09 + +The user explicitly authorized pushing the current source work and creating PRs for Anthony to review. This supersedes earlier source-only local restrictions for source publication; it does not authorize a data release, merge or deployment. + +Integration controls passed 36 cases; the exact replay correction subsequently passed all 57 cases at dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17. Source/control, model-declaration and resource hashes matched externally after both runs. A final formatting/import cleanup and explicit test-observer loop binding are included for CI; no release result is claimed. See docs/us-launch-review.md for current scope, evidence and related PRs. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md new file mode 100644 index 000000000..fda0967c0 --- /dev/null +++ b/docs/us-launch-review.md @@ -0,0 +1,104 @@ +# US graph build: review guide + +This branch consolidates the US graph build work for review. The intended file +supports national and 119th congressional district analysis from one population. +It is still a development build: complete enrichment, fiscal calibration and +release verification have not passed together. + +## Architecture and review order + +The construction order is ACS + CPS ASEC → combined survey multispine → PUF +clone and donor enrichment → supported joint geography → rules evaluation and +calibration → verified full and pruned exports. The PUF operator acts on the +combined survey frame. A separate ASEC–PUF base is not the intended architecture. + +| Area | Main source entry points | What to review | +| --- | --- | --- | +| Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | +| Survey source preparation | `packages/microcosm-build/src/microcosm/build/us_runtime/{acs_population_catalogue,asec_population_catalogue,survey_population_preparation}.py` | Source identity, entity links, source universes and current income transformations | +| Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | +| Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | +| Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | +| Geography | `us_runtime/{puma_ladder,puma_ladder_sources,graph_geography}.py` | Joint county/district support, vintage and source/PUMA preservation | +| Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | +| Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | + +Paths beginning `us_runtime/` are relative to +`packages/microcosm-build/src/microcosm/build/`. +The branch contains a substantial consolidation; review each area and its tests +before deciding how to land it. Shared runtime changes overlap with +[#873](https://github.com/PolicyEngine/microcosm/pull/873) and Anthony's +[#885](https://github.com/PolicyEngine/microcosm/pull/885). Their final contracts +need reconciliation; this draft does not supersede either review. + +## Verification completed on the integration checkout + +- The first integration selection passed **36 tests**: eight solver, eleven + snapshot/storage, nine geography and eight public-facade controls. +- After applying the independently reviewed replay correction, **57 enrichment + graph tests** passed at `dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17`. These include + cold execution, required replay, reconstruction through a fresh Frame store, + inherited tax-unit values and malformed artifact/ancestry refusals. +- The second run took 44.09 seconds wall time, 35.49 seconds CPU and 620.6 MB + peak RSS. Its 273 source/control files, 5,983 model declaration files and eleven + code resources matched before, after and in the external postcheck. The model + declaration checks did not execute the tax-benefit engine. +- The CI test inventory accounts for all 413 tracked test files. This is not a + claim that all 413 files or the complete repository CI have passed locally. + +Both test selections use invented inputs. The replay correction also received +an independent Fable source review that closed seven findings. That review did +not rerun the tests or certify a population file. + +The ordinary execution receipts are identified by SHA-256 +`bd9f944ed95e08de5775b36bca2158c38dc18bed858e9d304ccdfa1f983a8d43` +and `5f8fb487153db50826695d10a6341eb5bd621d88343af70c4ebfe472599528f2`. +Private runtime records and generated population artifacts are kept outside this +source PR. Receipt identifiers are audit references, not reproducibility inputs. + +## Component evidence and remaining work + +Separate component work has verified the national Census joint-geography source +artifact: 2,462 PUMAs, 87,841 joint cells, all 50 states plus DC and 436 districts +including DC. PUF source ingestion retained 207,692 ordinary records and excluded +four disclosure records. A corrected SCF wage-code interpretation passed source +preparation and model-mechanism checks. Those results belong to their respective +component revisions; they do not establish a calibrated result on this branch. + +The following work remains open: + +1. Finish current survey predictors and demographic graph bindings, then connect + them to the genuine combined survey population. +2. Complete the 59-output PUF donor profile, target-year growth and SCF loan + inputs. The existing 65-output mechanism remains a separately tested + compatibility path; a passing synthetic donor does not supply the missing + source information. +3. Verify every applicable model input on each source/clone channel. The proposed + national/CD profile retains 161 required inputs, dropping block and tract + requirements while retaining state and county as model inputs. Prior wages + remain excluded. A source operator's existence is not complete cell coverage. +4. Resolve repeated per-cell identity encoding in the survey verifier. One genuine + successor run reached persisted-artifact verification but exhausted its + two-hour CPU limit. That run failed; it is not a verified build. Performance + changes must preserve the identity bytes and pass small controls first. +5. Run real model evaluation, national/CD calibration, holdout checks and complete + export/replay verification, followed by a dashboard bound to that exact file. + Check every pruned analysis file separately. + +These are active workstreams. No release, merge or deployment is implied by this +draft, and the source changes do not relax the outstanding acceptance checks. + +## Related reviews + +- [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): + already published separately. It preserves the supplied schema metadata and + exact large integers; graph visibility does not infer domain verdicts. +- [Orrery review index, #6](https://github.com/TheAxiomFoundation/orrery/pull/6) + and [search improvements, #4](https://github.com/TheAxiomFoundation/orrery/pull/4). +- [Dynamics graph integration, #420](https://github.com/PolicyEngine/microcosm-dynamics/pull/420): + owned by the separate retirement-model task and tested on synthetic inputs. +- [Candidate methods and paper updates, site #72](https://github.com/PolicyEngine/microcosm.institute/pull/72): + public component evidence and remaining release checks. + +The accepted local shared viewer remains a separate consumer. There is no package +upgrade or competing generic graph shell in this consolidation. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py index 0e0a39f1a..104c15936 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py @@ -898,7 +898,9 @@ def prepare_acs_housing_population(source_dir, *, snapshot_root, serialnos=None) "full_source_frame_before_selection": serialnos is None, "full_source_lexical_projection": True, "native_selection_before_person_accumulation": serialnos is not None, - "selection_kind": "all" if serialnos is None else "engineering_exact_keys", + "selection_kind": "all" + if serialnos is None + else "engineering_exact_keys", "requested_serialnos": serialnos, "full_source_inclusion_probability": None, "pre_promotion_frame_sha256": before, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py index 9d3daf1cc..5e853585b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -207,13 +207,25 @@ def _preflight(paths, serialnos=None): # Preserve the whole-source route and its pre-construction row ceiling. for role, path in paths.items(): inventories[role], _empty = coverage._inventory(path, role, frozenset()) - _require(sum(m["rows"] for m in inventories[role]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + _require( + sum(m["rows"] for m in inventories[role]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) if role == "person": - _require(sum(m["rows"] for m in inventories[role]) <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") + _require( + sum(m["rows"] for m in inventories[role]) + <= coverage.literal.MAX_SELECTED_ROWS, + "NATIVE_ROW_BUDGET", + ) return inventories, None selected = frozenset(serialnos) - inventories["household"], households = coverage._inventory(paths["household"], "household", selected) - _require(sum(m["rows"] for m in inventories["household"]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + inventories["household"], households = coverage._inventory( + paths["household"], "household", selected + ) + _require( + sum(m["rows"] for m in inventories["household"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) _require(set(households) == selected, "SELECTION_UNKNOWN") _require(all(0 <= n <= 20 for n in households.values()), "SELECTED_NP") expected_rows = sum(households.values()) @@ -221,8 +233,13 @@ def _preflight(paths, serialnos=None): _require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") # _inventory bounds the actual selected row/body accumulation, independently # of reported NP. Every source member is still streamed and hashed. - inventories["person"], roster = coverage._inventory(paths["person"], "person", selected) - _require(sum(m["rows"] for m in inventories["person"]) <= MAX_SOURCE_ROWS, "SOURCE_ROW_BUDGET") + inventories["person"], roster = coverage._inventory( + paths["person"], "person", selected + ) + _require( + sum(m["rows"] for m in inventories["person"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) counts = dict.fromkeys(selected, 0) for serial, _line in roster: counts[serial] += 1 @@ -398,7 +415,9 @@ def verify_acs_native_coverage(issuance, frame=None): raise ACSNativeCoverageBindingError("NATIVE_VERIFICATION_REFUSED") from None -def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, candidate_path=None): +def issue_acs_native_coverage( + source_dir, *, snapshot_root, serialnos=None, candidate_path=None +): """Run real closed owners with optional exact engineering household selection. The complete selected roster is bounded before native construction. Full @@ -410,8 +429,12 @@ def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, cand # Snapshot path-like inputs once too, before producer/capture work. source_dir = Path(source_dir).absolute() snapshot_root = Path(snapshot_root).absolute() - candidate_path = None if candidate_path is None else Path(candidate_path).absolute() - coverage._json(serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES)) + candidate_path = ( + None if candidate_path is None else Path(candidate_path).absolute() + ) + coverage._json( + serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES) + ) pins = housing._pins() archives = _archives(pins) _require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, "ARCHIVE_BUDGET") @@ -428,15 +451,23 @@ def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, cand private, snapshot_root=roots[0], serialnos=serialnos ) _verify_prepared_frame(prepared) - _require(prepared.receipt["format"] == "microcosm.acs_housing_preparation.v2", "PREPARATION_PROTOCOL") - _require(prepared.receipt["requested_serialnos"] == (None if serialnos is None else list(serialnos)), "PREPARATION_SELECTION") + _require( + prepared.receipt["format"] == "microcosm.acs_housing_preparation.v2", + "PREPARATION_PROTOCOL", + ) + _require( + prepared.receipt["requested_serialnos"] + == (None if serialnos is None else list(serialnos)), + "PREPARATION_SELECTION", + ) keys, _roster_sha = coverage._native_roster(prepared.frame) if selected_roster is not None: households, roster = selected_roster _require( {s: n for s, n in households.items() if n > 0} == keys.groupby("SERIALNO").size().to_dict() - and set(roster) == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + and set(roster) + == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), "SELECTED_NATIVE_ROSTER", ) literal = coverage.load_authenticated_acs_person_coverage( @@ -479,10 +510,20 @@ def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, cand "requested_serialnos": serialnos, "complete_selected_roster": True, "person_rows": prepared.frame.n("person"), - "raw_person_keys_sha256": coverage._sha(coverage._json(sorted(zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True)), coverage.MAX_BODY_BYTES)), - "vacant_serialnos": None if selected_roster is None else sorted(s for s, n in selected_roster[0].items() if n == 0), + "raw_person_keys_sha256": coverage._sha( + coverage._json( + sorted( + zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True) + ), + coverage.MAX_BODY_BYTES, + ) + ), + "vacant_serialnos": None + if selected_roster is None + else sorted(s for s, n in selected_roster[0].items() if n == 0), "vacancy_status": "source_record_without_population_rows", - "before_person_accumulation_and_unit_assignment": serialnos is not None, + "before_person_accumulation_and_unit_assignment": serialnos + is not None, "full_source_inclusion_probability": None, "representative_sample": False, }, @@ -509,7 +550,9 @@ def issue_acs_native_coverage(source_dir, *, snapshot_root, serialnos=None, cand "kind": "design", "source": "WGTP; single-person PWGTP for WGTP=0 GQ placeholders", "renormalized": False, - "original_weight_literals_projection_sha256": coverage._sha(prepared.source.projection_json), + "original_weight_literals_projection_sha256": coverage._sha( + prepared.source.projection_json + ), "weight_sha256": coverage._sha( prepared.frame.weights_for("household").values.tobytes() ), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index 0038c91d7..f4431a860 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -217,7 +217,9 @@ def load_acs_pums_tables( serialnos = AcsPumsSource.snapshot_serialnos(serialnos) if serialnos is not None and source.max_households is not None: - raise ValueError("ACS exact serialnos and max_households are ambiguous together.") + raise ValueError( + "ACS exact serialnos and max_households are ambiguous together." + ) if chunksize <= 0: raise ValueError("chunksize must be positive.") household, household_members = _read_archive( @@ -250,9 +252,7 @@ def load_acs_pums_tables( if serialnos is not None: if not set(serialnos) <= all_household_serials: raise ValueError("ACS exact selection contains absent household keys.") - weights = pd.to_numeric(household.WGTP, errors="coerce").to_numpy( - dtype=float - ) + weights = pd.to_numeric(household.WGTP, errors="coerce").to_numpy(dtype=float) if not np.isfinite(weights).all() or (weights < 0).any(): raise ValueError("ACS WGTP must be finite and nonnegative.") _validate_amount_columns( @@ -262,7 +262,9 @@ def load_acs_pums_tables( drop=True ) if household.NP.sum() > MAX_EXACT_PERSON_ROWS: - raise ValueError("ACS selected complete roster exceeds native person budget.") + raise ValueError( + "ACS selected complete roster exceeds native person budget." + ) if source.max_households is not None and len(household) > source.max_households: household = _smoke_household_selection(household, source.max_households) selected_serials = frozenset(household["SERIALNO"].tolist()) @@ -526,9 +528,9 @@ def _validate_source_chunk(person, roster): def _validate_source_roster(household, roster): - for serial, count, kind in household[ - ["SERIALNO", "NP", "TYPEHUGQ"] - ].itertuples(index=False, name=None): + for serial, count, kind in household[["SERIALNO", "NP", "TYPEHUGQ"]].itertuples( + index=False, name=None + ): state = dict.get(roster, serial) if (len(state[0]) if state else 0) != int(count): raise ValueError("ACS NP/person row-count mismatch in complete source.") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py index 0686001aa..7ce6a9b24 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py @@ -98,9 +98,13 @@ from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer from .graph_composed_contracts import ( BIND_NODE as BIND_NODE, - US_COMPOSED_ASEC_BINDING_TYPE as US_COMPOSED_ASEC_BINDING_TYPE, +) +from .graph_composed_contracts import ( US_COMPOSED_ASEC_ARM_ROWS_TYPE as US_COMPOSED_ASEC_ARM_ROWS_TYPE, ) +from .graph_composed_contracts import ( + US_COMPOSED_ASEC_BINDING_TYPE as US_COMPOSED_ASEC_BINDING_TYPE, +) from .graph_composed_population import ( CREATE_NODE, US_COMPOSED_SOURCE_ORIGIN_TYPE, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py index 2a937fd55..a945506b4 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py @@ -85,6 +85,8 @@ ) from .graph_composed_contracts import ( LEAVES_NODE as LEAVES_NODE, +) +from .graph_composed_contracts import ( REPORTED_INCOME_NODE as REPORTED_INCOME_NODE, ) from .graph_composed_population import ( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 62d3e1362..03c602c75 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -885,9 +885,8 @@ def _resolve_take_up_program_bindings( for program in load_take_up_contract().programs ) for index, binding in enumerate(bindings): - if ( - len(binding) != 3 - or not all(isinstance(value, str) and value for value in binding) + if len(binding) != 3 or not all( + isinstance(value, str) and value for value in binding ): raise ValueError( "Take-up manifest program binding must contain three non-empty " @@ -1357,9 +1356,7 @@ def surface_provision(variable: str) -> str: variable, execution_scope="whole_pool", provision=provision, - available_by=( - "transferred" if variable in transfer_owned else "seeded" - ), + available_by=("transferred" if variable in transfer_owned else "seeded"), fallback=fallback, ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py index 530782446..683696bbc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_detail_transfer.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd + from microcosm.fit import _graph_legacy_qrf as codec from microcosm.fit.model_input import encode_recipient_matrix from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py index a2ac9942f..79b34e697 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd + from microcosm.fit import _graph_legacy_qrf as codec from microcosm.fit import model_input diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 681a16a19..6145dfc2f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -26,6 +26,7 @@ import numpy as np import pandas as pd + from microcosm.build import survey_domain_sample from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights from microcosm.graph import population as graph_population @@ -146,6 +147,7 @@ def _catalogue_fast_path(value): return False return True + def _catalogue_chunks(value): # Decide for the whole value before encoding. Unexpected shapes retain # the generic encoder's full-depth scalar pass and error precedence. @@ -171,12 +173,14 @@ def _catalogue_chunks(value): yield "]" yield "]" + def _catalogue_digest(value): digest = hashlib.sha256() for chunk in _catalogue_chunks(value): digest.update(chunk.encode("utf-8")) return digest.hexdigest() + def _value(value): if isinstance(value, Enum): return value.value diff --git a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py index 0ccb0fa75..50454a91d 100644 --- a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py +++ b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py @@ -11,6 +11,10 @@ import numpy as np import pandas as pd import pytest +from test_us_graph_survey_population import authenticated_arguments +from test_us_puf_detail_transfer import host as legacy_host +from test_us_survey_population_preparation import fixture as source_fixture + from microcosm.build.us_runtime import asec_current_money as money from microcosm.build.us_runtime import graph_combined_clone as clone from microcosm.build.us_runtime import graph_puf_diagnostic_consumer as bridge @@ -26,9 +30,6 @@ from microcosm.frame import Frame, Weights from microcosm.graph import NodeRejected, compile_graph, executor, run_graph from microcosm.graph.keys import opaque_artifact_key -from test_us_graph_survey_population import authenticated_arguments -from test_us_puf_detail_transfer import host as legacy_host -from test_us_survey_population_preparation import fixture as source_fixture def _copy_frame(frame): @@ -355,8 +356,8 @@ def test_actual_current_sources_cold_and_replayed_host(tmp_path, monkeypatch): kernels=live.kernels, sources=live.sources, resume=resume, - _population_observer=lambda name, population: observed.__setitem__( - name, population + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) ), ) actual = [] diff --git a/packages/microcosm-build/tests/test_us_runtime_facade_union.py b/packages/microcosm-build/tests/test_us_runtime_facade_union.py index 3287a7a4d..771a474d6 100644 --- a/packages/microcosm-build/tests/test_us_runtime_facade_union.py +++ b/packages/microcosm-build/tests/test_us_runtime_facade_union.py @@ -8,7 +8,6 @@ from microcosm.build import us_runtime as facade - # Derived independently from __all__ in pinned B 2ebca68e, F eba103bb and # J 6f6087cb. Hash preimage is UTF-8 newline-joined sorted unique export names. _UNION_SHA256 = "213ae67eab1640925fade5075a10eae62d7f219b15c1c04090a9feac8c692aa3" diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py index db141097d..c06fac40c 100644 --- a/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest.py @@ -3,6 +3,7 @@ import hashlib import pytest + from microcosm.build.us_runtime import survey_population_preparation as owner chunks = owner._catalogue_chunks @@ -37,7 +38,7 @@ def _raw(value): ), ( ((("é", '"\\\n', "\x00", "😀", "z", -1, ()),), ()), - '[[["é","\\"\\\\\\n","\\u0000","😀","z",-1,[]]],[]]'.encode("utf-8"), + '[[["é","\\"\\\\\\n","\\u0000","😀","z",-1,[]]],[]]'.encode(), ), ], ) diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py index f1591156c..518f51f46 100644 --- a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_authority.py @@ -4,9 +4,10 @@ from types import SimpleNamespace import pytest -from microcosm.build.us_runtime import survey_population_preparation as owner from test_us_survey_population_preparation import fixture +from microcosm.build.us_runtime import survey_population_preparation as owner + def _must_not_execute(*_args, **_kwargs): raise AssertionError("Changed JSON provider executed before owner refusal") diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py index b557cca6e..27c112d9c 100644 --- a/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_digest_fast_path.py @@ -3,6 +3,7 @@ import hashlib import pytest + from microcosm.build.us_runtime import survey_population_preparation as owner diff --git a/packages/microcosm-graph/src/microcosm/graph/attachments.py b/packages/microcosm-graph/src/microcosm/graph/attachments.py index fd2e16446..83fa61651 100644 --- a/packages/microcosm-graph/src/microcosm/graph/attachments.py +++ b/packages/microcosm-graph/src/microcosm/graph/attachments.py @@ -4,7 +4,6 @@ """ import hashlib - from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path From 815730051f9d51a0fb1a9fc0c3b1820c99fc5dd0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 12:28:49 -0400 Subject: [PATCH 020/255] Add canonical PUF donor and current survey predictor graph Integrate the PUF59 profile, source-specific growth, receipt-bound donor codec, and current survey financial and demographic source owners. Fix checkpoint test coverage and pool registry import order, and document block-first US and small-area UK geography as pending build requirements. --- docs/current-survey-puf59-progress.md | 13 + .../puf2015-target2024/GROWTH-RECIPE.json | 447 ++ .../puf2015-target2024/INDEX-VALUES.json | 8 + .../NATIONAL-GROWTH-EXTRACT.json | 4439 +++++++++++++++++ .../PUBLIC-WORKBOOK-SOURCES.json | 34 + docs/geography-assignment.md | 86 + docs/puf2015-canonical59-and-growth.md | 74 + docs/us-launch-review.md | 96 +- .../us_runtime/asec_demographic_source.py | 1519 ++++++ .../build/us_runtime/cps_carried_current.py | 68 +- .../us_runtime/current_asec_demographics.py | 379 ++ .../us_runtime/current_survey_predictors.py | 381 ++ .../build/us_runtime/full_puf_enrichment.py | 339 +- .../graph_current_survey_predictors.py | 514 ++ .../us_runtime/graph_full_puf_enrichment.py | 110 +- .../build/us_runtime/multispine_pool.py | 12 +- .../build/us_runtime/puf59_canonical.py | 307 ++ .../us_runtime/puf59_canonical_artifact.py | 237 + .../build/us_runtime/puf_full_source.py | 519 ++ .../build/us_runtime/puf_full_source_graph.py | 122 + .../build/us_runtime/puf_qbi_model.py | 369 ++ .../build/us_runtime/puf_target2024_growth.py | 164 + .../tests/test_frame_checkpoint.py | 202 +- .../test_us_asec_2024_native_population.py | 30 +- .../test_us_current_asec_demographics.py | 191 + .../test_us_current_survey_predictors.py | 422 ++ .../tests/test_us_full_puf_output_profiles.py | 1118 +++++ .../tests/test_us_graph_survey_population.py | 4 +- .../tests/test_us_puf59_canonical.py | 517 ++ .../tests/test_us_puf_target2024_growth.py | 235 + .../tests/test_us_runtime_import_order.py | 39 + .../test_us_survey_population_preparation.py | 2 + pyproject.toml | 3 + 33 files changed, 12857 insertions(+), 143 deletions(-) create mode 100644 docs/current-survey-puf59-progress.md create mode 100644 docs/evidence/puf2015-target2024/GROWTH-RECIPE.json create mode 100644 docs/evidence/puf2015-target2024/INDEX-VALUES.json create mode 100644 docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json create mode 100644 docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json create mode 100644 docs/geography-assignment.md create mode 100644 docs/puf2015-canonical59-and-growth.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_qbi_model.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_demographics.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_predictors.py create mode 100644 packages/microcosm-build/tests/test_us_full_puf_output_profiles.py create mode 100644 packages/microcosm-build/tests/test_us_puf59_canonical.py create mode 100644 packages/microcosm-build/tests/test_us_puf_target2024_growth.py create mode 100644 packages/microcosm-build/tests/test_us_runtime_import_order.py diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md new file mode 100644 index 000000000..3d8fe71d1 --- /dev/null +++ b/docs/current-survey-puf59-progress.md @@ -0,0 +1,13 @@ +# PUF59 and current survey predictor implementation status + +The PUF59 profile fits and transfers 56 person outcomes and three tax-unit outcomes. Six detailed mortgage fields remain owned by the separate SCF producer. The historical full65 default remains available. PUF59 conditions on the source-specific PUF2015 filing-status class and capped return-size proxy, plus six monetary predictors. Its self-employment predictor includes ordinary and SSTB Schedule C income. Return-level Boolean outputs use an explicit incidence capacity of one; that is not a physical person count. + +The local PUF59/full65 regression run passed 128 ordinary tests. Those tests execute real weighted QRF fits, sequential target conditioning, draws, finalization, whole-Population preservation and required cache replay. They include excluded mortgage incumbent preservation and source-specific predictor checks. The run took 93.9 seconds and peaked at 622.7 MB RSS. These are invented mechanism fixtures, not held-out fit-quality or launch acceptance. + +Current survey financial completion passed 26 ordinary tests, including 15 ASEC demographic projection controls. The financial graph uses original ASEC design weights to fit interest, dividends and capital-gains outcomes for native ACS people. It preserves ACS observed wage/self-employment amounts and source-owned age-15 universe zeros. A draw is shared with its clone by source identity. The graph retains typed preparation, allocation and context evidence while separating the donor branch from survey allocation, avoiding a structural dependency cycle. Cold execution and required replay passed with the actual fitted model artifacts. The combined run took 121.4 seconds and peaked at 524.0 MB RSS. + +The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. + +Social Security handling remains a launch blocker. The PUF donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. The current reconciliation code can treat unknown recipient components as zero and use equal component shares. PUF59 requires a source-qualified recipient basis, including observed ASEC component/type information and an explicit ACS completion path, before those outputs can support release acceptance. + +These source tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. The integrated checkout still needs its own checks after source formatting and composition. diff --git a/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json new file mode 100644 index 000000000..79974b85d --- /dev/null +++ b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json @@ -0,0 +1,447 @@ +{ + "schema": "microcosm.us.puf2015_target2024_growth/1", + "input_money_year": 2015, + "output_money_year": 2024, + "output_profile": "puf59_scf_mortgage_v1", + "ordered_targets": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald" + ], + "money_fields": { + "employment_income_before_lsr": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "sstb_w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "sstb_self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "taxable_interest_income": { + "positive_family": "taxable_interest", + "negative_family": "taxable_interest", + "positive_factor": "2.599750625636417399355437523686408804619", + "negative_factor": "2.599750625636417399355437523686408804619", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "tax_exempt_interest_income": { + "positive_family": "tax_exempt_interest", + "negative_family": "tax_exempt_interest", + "positive_factor": "0.9295395824924337172788943063597248917385", + "negative_factor": "0.9295395824924337172788943063597248917385", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "qualified_dividend_income": { + "positive_family": "qualified_dividends", + "negative_family": "qualified_dividends", + "positive_factor": "1.409891045424454325551259680558160347719", + "negative_factor": "1.409891045424454325551259680558160347719", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_qualified_dividend_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_sch_d_capital_gains": { + "positive_family": "non_schedule_d_distributions", + "negative_family": "non_schedule_d_distributions", + "positive_factor": "1.120350061633613230084419890940102481805", + "negative_factor": "1.120350061633613230084419890940102481805", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_private_pension_income": { + "positive_family": "taxable_pensions", + "negative_family": "taxable_pensions", + "positive_factor": "1.327588034973940867230285994253866499704", + "negative_factor": "1.327588034973940867230285994253866499704", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_ira_distributions": { + "positive_family": "ira_distributions", + "negative_family": "ira_distributions", + "positive_factor": "1.510871073557979130794209158828995676924", + "negative_factor": "1.510871073557979130794209158828995676924", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "alimony_income": { + "positive_family": "alimony_income", + "negative_family": "alimony_income", + "positive_factor": "1.542033706590010885068081742929926580179", + "negative_factor": "1.542033706590010885068081742929926580179", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "salt_refund_income": { + "positive_family": "state_tax_refund", + "negative_family": "state_tax_refund", + "positive_factor": "0.9296526775720357466732616002244135476521", + "negative_factor": "0.9296526775720357466732616002244135476521", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "short_term_capital_gains": { + "positive_family": "short_current_gain", + "negative_family": "short_current_loss", + "positive_factor": "2.042865587775988756504710527637911842775", + "negative_factor": "1.738139193325432816803529458382722438642", + "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded." + }, + "long_term_capital_gains_before_response": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "long_term_capital_gains_on_collectibles": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "unrecaptured_section_1250_gain": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "social_security_retirement": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_disability": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_dependents": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_survivors": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "rental_income": { + "positive_family": "rental_royalty_profit", + "negative_family": "rental_royalty_loss", + "positive_factor": "1.476663829685885394438774259073237903799", + "negative_factor": "1.690793641559507564235995133214906035185", + "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions." + }, + "estate_income": { + "positive_family": "estate_profit", + "negative_family": "estate_loss", + "positive_factor": "1.476331638358104936021886589313386353228", + "negative_factor": "1.582810027948011789708036602150526302352", + "assumption": "" + }, + "farm_operations_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_rent_income": { + "positive_family": "farm_rent_profit", + "negative_family": "farm_rent_loss", + "positive_factor": "1.629452810459537374541656575493532799084", + "negative_factor": "1.150378978298943195689490419607276484593", + "assumption": "" + }, + "miscellaneous_income": { + "positive_family": "other_property_gain", + "negative_family": "other_property_loss", + "positive_factor": "1.738242164922618767195190248113271033066", + "negative_factor": "1.411645744283618865226902643798237455207", + "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income." + }, + "partnership_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "s_corp_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "partnership_self_employment_net_earnings": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "qualified_bdc_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation." + }, + "qualified_reit_and_ptp_income": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool." + }, + "alimony_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_non_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "real_estate_taxes": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "home_mortgage_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_interest_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_income_elected_form_4952": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "student_loan_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "educator_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "qualified_tuition_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "casualty_loss": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unreimbursed_business_employee_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "traditional_ira_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "self_employed_pension_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "domestic_production_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "health_savings_account_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "sstb_unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + } + }, + "return_incidence_fields": [ + "business_is_sstb", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified" + ], + "source_pins": { + "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8", + "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3", + "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b", + "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c" + }, + "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867", + "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.", + "operation_order": "Decode source statistical2015 \u2192 source canonical transformations \u2192 QBI2015 model (all16 outputs) \u2192 this growth transform once \u2192 2024 canonical donor. Monetary recipient predictors remain 2024.", + "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.", + "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority." +} diff --git a/docs/evidence/puf2015-target2024/INDEX-VALUES.json b/docs/evidence/puf2015-target2024/INDEX-VALUES.json new file mode 100644 index 000000000..5dd19e557 --- /dev/null +++ b/docs/evidence/puf2015-target2024/INDEX-VALUES.json @@ -0,0 +1,8 @@ +{ + "schema": "microcosm.public_index_value_transcript/1", + "retrieved_date": "2026-09-09", + "retrieval": "Official URLs successfully read with web retrieval; exact values transcribed and independently checkable. These are not hashes of original publisher HTML/PDF bytes. Direct SSA urllib acquisition returned HTTP403 before writing any document.", + "awi": {"url": "https://www.ssa.gov/oact/cola/AWI.html", "series": "National Average Wage Index", "2015": "48098.63", "2024": "69846.57", "measurement": "Wages subject to federal income tax and deferred compensation; average-worker index, used as an explicit proxy for PUF wage amount growth."}, + "cola": {"url": "https://www.ssa.gov/oact/cola/colaseries.html", "cash_timing_url": "https://www.ssa.gov/cola/", "december_payable_percent": {"2015": "0.0", "2016": "0.3", "2017": "2.0", "2018": "2.8", "2019": "1.6", "2020": "1.3", "2021": "5.9", "2022": "8.7", "2023": "3.2"}, "timing": "December benefits are received in January of the following year. Product covers January2016 through January2024 cash increases relative to calendar2015 cash benefits. This transports a fixed entitlement; it does not forecast the recipient mix or new awards."}, + "cpi": {"url": "https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf", "series": "CPI-U, US city average, all items, annual average, 1982-84=100", "table": "Historical CPI-U index averages, annual average column", "2015": "237.017", "2023": "304.702", "2024": "313.689", "pages_one_based": [4,5], "monthly_2024_check": ["308.417","310.326","312.332","313.548","314.069","314.175","314.540","314.796","315.301","315.664","315.493","315.605"]} +} diff --git a/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json new file mode 100644 index 000000000..e24f8d37d --- /dev/null +++ b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json @@ -0,0 +1,4439 @@ +{ + "schema": "microcosm.us.puf_observed_growth_sources/1", + "scope": "exact headers and national row only; no AGI rows used", + "tables": { + "15in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "sheet": "TBL14", + "year": 2015, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 150493263, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_profit": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_loss": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "sheet": "TBL14", + "year": 2023, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 160602107, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_profit": { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_loss": { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "s_corp_profit": { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "s_corp_loss": { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "15in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "sheet": "Sheet1", + "year": 2015, + "title": "Table 1.4A Returns with Income or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 3900677, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 38164189, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 7820586, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 250592206, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11735160, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 733313255, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 6375392, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 312723716, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 3923409, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 22196394, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 6269981, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 68624843, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 231076, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 5948464, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 336867, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 1980907, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 483349, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 17194108, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 725964, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16056081, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 2000156, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 171105171, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8440065, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 282493545, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 4441176, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 62343837, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 2177243, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 226671889, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 296456, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 2562723, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1721361, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 216227585, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 552552, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 10694019, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9733033, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 62496866, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 4371225, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 291699768, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "sheet": "Sheet1", + "year": 2023, + "title": "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 6215519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 70482448, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 11370982, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 673910232, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11888516, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 971279947, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 11230240, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 450065138, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 7140149, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 58927145, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 8377604, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 135549749, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 174491, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 8619769, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 401474, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 3833687, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 495780, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 28464220, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 560758, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16759596, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 5151589, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 543302365, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8891406, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 405281941, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 9496262, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 149558927, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 1979380, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 316779349, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 338620, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 5211519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1219800, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 282281744, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 766861, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 22526709, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9487999, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 44095015, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 5622923, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 349944357, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + } + }, + "factors": { + "taxable_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "2.525269343625908636956987730976528139607", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.599750625636417399355437523686408804619" + }, + "tax_exempt_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9029088360274333448808012169263853497016", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9295395824924337172788943063597248917385" + }, + "ordinary_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.586285145427430620786783837166923431342", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.633071660126895406016322292262784767590" + }, + "qualified_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.369498520263452278862567470282453558367", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.409891045424454325551259680558160347719" + }, + "state_tax_refund": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9030186910014518713848306957259555062394", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9296526775720357466732616002244135476521" + }, + "alimony_income": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.497855374161636196047724476262267688168", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.542033706590010885068081742929926580179" + }, + "business_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.187493036701270108472919624578501757792", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.222517420922031096142334753675412822692" + }, + "business_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.669652726148833347538733957583542565766", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.718898116890934040328182671660914217539" + }, + "non_schedule_d_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.088252710423015210712466518141315463440", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.120350061633613230084419890940102481805" + }, + "other_property_gain": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.688442578911762234575993608257254511052", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738242164922618767195190248113271033066" + }, + "other_property_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.371202948062275806522918206792748706765", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.411645744283618865226902643798237455207" + }, + "ira_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.467585531705808482641269279807429143993", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.510871073557979130794209158828995676924" + }, + "taxable_pensions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.289553441251142788324750319651443404751", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.327588034973940867230285994253866499704" + }, + "rental_royalty_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434358304667835504133977902598222238470", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476663829685885394438774259073237903799" + }, + "rental_royalty_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.642353427026338423846026443645930519505", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.690793641559507564235995133214906035185" + }, + "farm_rent_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.582769973612852083106490319603270854083", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.629452810459537374541656575493532799084" + }, + "farm_rent_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.117421316799902418041369349372073484912", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.150378978298943195689490419607276484593" + }, + "estate_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434035630420548027555129084975779987827", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476331638358104936021886589313386353228" + }, + "estate_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.537463478591264240530009559622650667952", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.582810027948011789708036602150526302352" + }, + "farm_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.403329848619305835091312061082920777637", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.444720208215047581256304812994467826979" + }, + "farm_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.678866744726831887395787683833365558775", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.728383897337776476443532509645504817056" + }, + "other_income_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.065879634464584874641519579136196913601", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.097317105419594110775195552571543585636" + }, + "other_income_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.903940252832176698123261573865488200400", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.960095811549227363973941096035769795063" + }, + "short_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_gain", + "short_other_gain", + "short_pass_through_gain" + ], + "amounts_thousands_usd": [ + 45338966, + 96011134 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.984338725063739328074935070060878814103", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.042865587775988756504710527637911842775" + }, + "short_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_loss", + "short_other_loss", + "short_pass_through_loss" + ], + "amounts_thousands_usd": [ + 86661831, + 156143032 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.688342557388515472795249540239320768338", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738139193325432816803529458382722438642" + }, + "long_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_gain", + "long_other_gain", + "long_pass_through_gain", + "long_distributions" + ], + "amounts_thousands_usd": [ + 787889885, + 1048438049 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.246932856321931105568396313543555115004", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.283710381837894882785950440755768785471" + }, + "long_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_loss", + "long_other_loss", + "long_pass_through_loss" + ], + "amounts_thousands_usd": [ + 75600579, + 177297155 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "2.197568722014698372473641296575486125444", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.262384673681395979556694950087192296750" + }, + "partnership_s_corp_profit": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.549554109090802831376907083449129785317", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.595257264233857504616939193376066035754" + }, + "partnership_s_corp_loss": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.912533224223542778513579904641714927631", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.968942227400735507640732147170530268700" + }, + "nonqualified_dividends": { + "method": "residual_dollars_per_ordinary_dividend_return", + "observed_2015_2023_factor": "2.412636223005660751243227924409122950293", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.483795459689869824933006427197633652403", + "source_families": [ + "ordinary_dividends", + "qualified_dividends" + ] + } + }, + "modeled_bridge": { + "method": "constant real conditional amounts after 2023", + "cpi_2023": "304.702", + "cpi_2024": "313.689" + }, + "capital_carryover_reconciliation_thousands_usd": { + "short": { + "2015": { + "net_including_carried_loss": -212428017, + "carried_loss": 171105171, + "current_year_net_by_removing_carryover": -41322846, + "current_year_gross_component_gain": 45338966, + "current_year_gross_component_loss": 86661831, + "current_year_net_from_component_difference": -41322865, + "reported_less_component_net_residual": 19, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": -603427784, + "carried_loss": 543302365, + "current_year_net_by_removing_carryover": -60125419, + "current_year_gross_component_gain": 96011134, + "current_year_gross_component_loss": 156143032, + "current_year_net_from_component_difference": -60131898, + "reported_less_component_net_residual": 6479, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + }, + "long": { + "2015": { + "net_including_carried_loss": 420589539, + "carried_loss": 291699768, + "current_year_net_by_removing_carryover": 712289307, + "current_year_gross_component_gain": 787889885, + "current_year_gross_component_loss": 75600579, + "current_year_net_from_component_difference": 712289306, + "reported_less_component_net_residual": 1, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": 521214809, + "carried_loss": 349944357, + "current_year_net_by_removing_carryover": 871159166, + "current_year_gross_component_gain": 1048438049, + "current_year_gross_component_loss": 177297155, + "current_year_net_from_component_difference": 871140894, + "reported_less_component_net_residual": 18272, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + } + }, + "notes": [ + "Reporting-return means mix intensive-margin growth and changing composition. This is a distribution transport assumption, not panel growth.", + "Positive/loss magnitudes are distinct series. Source zeros and sign states are retained.", + "No total calibration or reporting-incidence claim; source design weights are unchanged." + ] +} diff --git a/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json new file mode 100644 index 000000000..23975da76 --- /dev/null +++ b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json @@ -0,0 +1,34 @@ +[ + { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "bytes": 83456, + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "bytes": 115712, + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "bytes": 64000, + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14acg.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "bytes": 74752, + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14acg.xls" + } +] diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md new file mode 100644 index 000000000..b6bee7cf5 --- /dev/null +++ b/docs/geography-assignment.md @@ -0,0 +1,86 @@ +# Geography assignment in the population graph + +Design decision, 9 September 2026: assign one small census area to each household +after constructing and harmonizing the survey spine. Derive larger geographies +from that area using identified, versioned mappings. PUF and other enrichment +clones inherit the location. Geographic support expansion, if needed, is a +separate declared population operation with explicit identities and weights. + +This document records the intended contract and the implementation gap. It does +not certify a new geographic source, assignment run or population file. + +```mermaid +flowchart LR + S[Combined survey spine] --> A[Assign one small area per household] + C[Observed survey geography constraints] --> A + L[Versioned small-area support] --> A + A --> D[Derive larger geographies] + M[Versioned geographic mappings] --> D + D --> E[Enrichment clones inherit location] + E --> R[Rules and calibration] + R --> P[Pruned analysis file retains location] +``` + +## Country anchors + +| Country | Assigned household anchor | Derived geographies | +| --- | --- | --- | +| US | Census block, with census vintage | Block group, tract, county, state, PUMA and congressional district using the declared mapping/boundary vintages | +| England and Wales | 2021 Output Area | LSOA, MSOA, local authority, ward, constituency and region | +| Scotland | 2022 Output Area | Scottish statistical areas, council area, ward, constituency and region | +| Northern Ireland | 2021 Data Zone | Super Data Zone, local government district, constituency and nation | + +Northern Ireland's 2021 Data Zones replaced the 2011 Small Areas. Scottish +Output Areas use the 2022 census. These are different source systems; the graph +must retain the original area type and vintage rather than imply identical units. +See [NISRA census output geography](https://www.nisra.gov.uk/statistics/census-2021-results/census-output-geography) +and [Scotland's census general report](https://www.scotlandscensus.gov.uk/about/scotlands-census-2022-general-report/). + +Some larger boundaries do not nest exactly. England and Wales small-area estimates +use best-fit mappings for wards and parliamentary constituencies. The derived +edge must identify whether its mapping is exact, best-fit or another explicit +approximation, with its source and vintage. A mapping convention cannot be +presented as observed household location. See the +[ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). + +## Required behavior + +- Respect observed source constraints, such as ACS PUMA and FRS region. Record + inferred geographic detail as modeled. Preserve observed source geography + separately from the assigned location. +- Bind each draw to a stable household identity, seed and assignment definition. + Reordering rows or selecting an existing household must not redraw its location. + Changed source support or boundary mappings produce a new identified revision. +- Assign household members and enrichment clones consistently. Materialized + county, district and other fields must agree with the selected anchor's mapping. +- Show assignment and derivation as separate operations in the graph. Expose the + source constraints, support, sampling weights, mapping conventions and judgment + annotations in the inspector. +- Refuse missing, ambiguous or incompatible mappings unless an explicit, + source-supported resolution rule is part of the assignment definition. +- Verify inheritance and mapping consistency again on every pruned export. + Engine input profiles and geographic build invariants are separate contracts. + +## Existing implementation and next changes + +The US development composed graph attaches geography after harmonization, but +`us_runtime/graph_geography.py` selects a joint tract/congressional-district cell +and emits PUMA, county and district. It does not assign a Census block. Its prior +national source and joint-support checks therefore do not establish block-first +acceptance. The current survey/clone graph needs an explicit block assignment +and derivation connection before enrichment. Block support and current district +mapping require source review before acquisition or execution. + +The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. +Its sampler selects a constituency within FRS region using household counts, +then an OA within constituency using population; one selected ladder row supplies +the other geographies. It currently consumes a shared seeded random stream, so +stable household-keyed assignment under reordering and subsets still needs work. +The graph should expose the final OA/Data Zone assignment and the mapping +derivations explicitly, including that sampling convention. + +The Northern Ireland source builder currently infers Data Zone constituencies +using the modal active-postcode constituency. Review the official +[NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) +before choosing or replacing that approximation. Source existence is not +acceptance of a particular lookup or its application to the population. diff --git a/docs/puf2015-canonical59-and-growth.md b/docs/puf2015-canonical59-and-growth.md new file mode 100644 index 000000000..36733c8fe --- /dev/null +++ b/docs/puf2015-canonical59-and-growth.md @@ -0,0 +1,74 @@ +# PUF 2015 canonical donor and 2024 transport + +These source owners turn the statistical 2015 PUF into an explicit return-level donor with 59 canonical outputs. They preserve raw fields, source status and documentary decisions separately from modeled values. The six detailed mortgage balance, interest and origination-year outputs belong to the downstream SCF stage. + +```mermaid +flowchart LR + A[Authenticated PUF typed source] --> B[Observed and derived return fields] + B --> C[Eleven explicit baseline models] + C --> D[QBI model in 2015 money] + D --> E[Source-specific 2024 monetary transport] + E --> F[Canonical donor with 59 outputs typed donor] + F --> G[Eight-feature PUF59 donor interface] +``` + +The source codec retains all ordinary returns, including those missing the demographic supplement. Disclosure aggregates retain their source lexemes but are excluded from individual donors. Neither a demographic gap nor zero reported exemptions means that a source return is dropped. + +## Count measurements + +PUF 2015 MARS 2 combines joint and qualifying widow(er) returns. The matching predictor maps the survey's separate widow category into that combined class. It is named `puf_2015_filing_status_code`; generic model filing status is unchanged. + +The corresponding `puf_2015_capped_return_size` equals one plus an extra unit for MARS 2 plus the sum of the four reported dependent-category counts. The source owner validates their disclosure caps by filing class. On the survey side, the matching measurement caps the reported dependent total before adding the filing-class units. The result is a statistical predictor, not a count of physical people or co-residents. Actual survey membership remains unchanged. + +The separate `puf_person_incidence_capacity` is one for return-level modeled zero/one QBI incidence. It is used only to validate the canonical outcome representation and is excluded from the eight fitted predictors. + +## Observed fields and explicit models + +The decoder emits 33 directly mapped or algebraically derived baseline fields. Eleven additional baseline outputs explicitly model source gaps: a normalized interest split, a component-neutral Social Security carrier, desired contributions proxied by realized deductions, incomplete tuition and employee-expense proxies, and active-partnership earnings. These transformations retain their assumptions in the receipt. + +The interest model uses the existing [IRS 2015 Table 2.1 workbook](https://www.irs.gov/pub/irs-soi/15in21id.xls). It normalizes the four published component amounts within the source AGI band. Mortgage interest, points and mortgage insurance share the available home-interest leaf; the investment component alone enters investment expense. Complementary rounding preserves the source total exactly. This does not create a mortgage balance, origination year or observed loan structure. + +The QBI owner reproduces the archived data model's assumptions with RECID-keyed random draws, retaining a reusable full-cohort employee calibration. Its 15 added leaves and Schedule C replacement are modeled inputs. The statutory engine owns deductions and limits. Ordinary and SSTB Schedule C branches are summed for the donor's total self-employment predictor, matching the survey measurement. + +## Source-specific monetary transport + +The publisher's statistical 2015 convention is the input basis. Raw filing-year/month fields remain intact; the transform does not apply another row-year CPI adjustment. + +Wages and modeled W2 amounts use the [SSA average wage index](https://www.ssa.gov/oact/cola/AWI.html). Social Security carriers use [SSA COLAs](https://www.ssa.gov/oact/cola/colaseries.html) received from January 2016 through January 2024. This transports fixed entitlements and does not predict changes in recipient composition. + +Other income families use the observed 2015 and 2023 national amount/count cells of [IRS Table 1.4 and 1.4A](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income), with positive and loss magnitudes kept separate. The 2023→2024 step uses the [BLS annual CPI-U series](https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf), explicitly assuming stable real conditional amounts for that year. Expenses, UBIA and selected mixed model outputs use an explicitly named CPI proxy. Every monetary field has a declared rule; incidence fields are unchanged. + +Partnership and S-corporation reporting counts overlap, so their dollar components are summed with the common all-return denominator. Dividend residuals use the ordinary-dividend reporting universe and do not subtract overlapping recipient counts. + +The PUF short- and long-term capital fields remove the effect of carried losses. Published net-loss means that include carryovers cannot be substituted directly. The model uses current-year gross gain/loss components per all return as a declared proxy for signed net flows. The evidence retains both the carryover-adjusted reported net and component-difference net, including their unresolved residual. It does not attribute that difference to rounding without evidence. + +The observed series are source facts. Their application to individual donor amounts, the related-family substitutions, and the last-year bridge are modeling decisions. They do not establish forecast accuracy or current reporting incidence. A same-cohort CPI-only sensitivity is available. Source design weights are divided from S006 hundredths once and are never grown. + +## Validation status + +The ordinary verification executes real source decoding, the QBI model, growth, the deterministic typed envelope and the PUF59 donor interface at 64 and 2,048 invented records. It checks all 59 outputs, sign/zero preservation, physical type and knownness refusal, exact interest conservation, SSTB/base identities, source immutability, stable full-fit subset replay, artifact byte/value replay and malformed-input refusal. + +The 59 invented controls passed. The separately reviewed genuine construction then produced all 59 canonical outcomes for 207,692 ordinary returns in 23.07 seconds. The 64-return subset reproduced the full-cohort outputs when it reused the full QBI employee fit, and the typed donor passed byte/value replay through the canonical donor interface. + +Independent review subsequently found that the version 1 envelope could attach +an old descriptive receipt to altered, domain-valid values. The version 2 codec +now checks the row-ordered identity/weight/source-feature digest and all 59 output +values against that receipt on encode and decode. It also pins the actual +interest-band facts, verifies the embedded growth recipe, requires canonical +JSON and physical integer row metadata, and makes CPI-sensitivity acceptance +explicit. The corrected source passed 80 invented controls, including the 21 +new integrity cases. Formatting followed that run; the composed integration +checkout still needs its own execution checks. + +The original genuine donor and evidence remain on version 1. A version 2 genuine +construction or repacking requires separate verification, and the new decoder +refuses old magic. Serialized receipt consistency does not authenticate an +issuer, establish execution or confer release status. + +This result verifies source donor construction. Social Security component reconciliation, downstream SCF mortgage semantics, conditional-fit quality, recipient placement and calibration remain open release requirements. It does not certify a calibrated population or policy-analysis file. + +## Evidence bundled with the source + +The [national extract](evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json) records the exact public workbook cells, units, hashes, family calculations and unresolved capital residuals. The [index transcript](evidence/puf2015-target2024/INDEX-VALUES.json) identifies the SSA and BLS values and retrieval sources; it is a transcription of retrieved values, not a hash of the original web pages. The [growth recipe](evidence/puf2015-target2024/GROWTH-RECIPE.json) assigns each monetary field a factor and interpretation. + +[Public workbook locators](evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json) provide the four official URLs and exact sizes and hashes without local cache paths. The recipe also preserves hashes of the original local acquisition receipt and extraction script. Those two local files are outside this publication bundle; their recorded hashes do not imply that a reader received them. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index fda0967c0..1e5036a14 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -7,10 +7,17 @@ release verification have not passed together. ## Architecture and review order -The construction order is ACS + CPS ASEC → combined survey multispine → PUF -clone and donor enrichment → supported joint geography → rules evaluation and -calibration → verified full and pruned exports. The PUF operator acts on the -combined survey frame. A separate ASEC–PUF base is not the intended architecture. +The intended construction order is ACS + CPS ASEC → combined survey multispine → +household Census block assignment → derived larger geographies → PUF clone and +donor enrichment → rules evaluation and calibration → verified full and pruned +exports. Enrichment clones inherit their household's assigned location. The PUF +operator acts on the combined survey frame. A separate ASEC–PUF base is not the +intended architecture. + +Block-first assignment is a required change, not completed functionality in this +branch. The existing geography operator chooses a joint tract/district cell. +The US and UK assignment contracts, existing implementation and remaining graph +work are distinguished in [Geography assignment](geography-assignment.md). | Area | Main source entry points | What to review | | --- | --- | --- | @@ -19,7 +26,7 @@ combined survey frame. A separate ASEC–PUF base is not the intended architectu | Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | | Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | | Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | -| Geography | `us_runtime/{puma_ladder,puma_ladder_sources,graph_geography}.py` | Joint county/district support, vintage and source/PUMA preservation | +| Geography | `us_runtime/{puma_ladder,puma_ladder_sources,graph_geography}.py` | Existing joint-cell implementation; pending block assignment, versioned projections and source/PUMA preservation | | Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | | Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | @@ -31,29 +38,42 @@ before deciding how to land it. Shared runtime changes overlap with [#885](https://github.com/PolicyEngine/microcosm/pull/885). Their final contracts need reconciliation; this draft does not supersede either review. -## Verification completed on the integration checkout - -- The first integration selection passed **36 tests**: eight solver, eleven - snapshot/storage, nine geography and eight public-facade controls. -- After applying the independently reviewed replay correction, **57 enrichment - graph tests** passed at `dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17`. These include - cold execution, required replay, reconstruction through a fresh Frame store, - inherited tax-unit values and malformed artifact/ancestry refusals. -- The second run took 44.09 seconds wall time, 35.49 seconds CPU and 620.6 MB - peak RSS. Its 273 source/control files, 5,983 model declaration files and eleven - code resources matched before, after and in the external postcheck. The model - declaration checks did not execute the tax-benefit engine. -- The CI test inventory accounts for all 413 tracked test files. This is not a - claim that all 413 files or the complete repository CI have passed locally. - -Both test selections use invented inputs. The replay correction also received -an independent Fable source review that closed seven findings. That review did -not rerun the tests or certify a population file. - -The ordinary execution receipts are identified by SHA-256 -`bd9f944ed95e08de5775b36bca2158c38dc18bed858e9d304ccdfa1f983a8d43` -and `5f8fb487153db50826695d10a6341eb5bd621d88343af70c4ebfe472599528f2`. -Private runtime records and generated population artifacts are kept outside this +## Verification and review scope + +The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` +passed 36 integration controls and 57 enrichment/replay controls on invented +inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and +eight facade cases. The 57 include cold execution, required replay, reconstruction +through a fresh Frame store, inherited tax-unit values and malformed ancestry +refusals. Source/control, model-declaration and resource pins matched before, +after and in external postchecks; the tax-benefit engine was not executed. + +Those receipts are identified by SHA-256 +`ad8ab6d8761db3d21343bc1fc60023c0dfa305e1367622630a5de6c68892d62a` +and `85a192f8c5c851648875d66b90bea6b8a4ee11e86e67d1875c3ca34a076ee014`. +These checks do not certify subsequent source additions or the entire repository. + +CI exposed test-helper collection failures and a pool/registry circular import. +This revision adds the test helper directory to pytest's path and moves the +registry import after the pool functions it validates. Two fresh-process import +regressions are assigned to the US-engine CI lane; their runtime result remains +pending. No import-order result is inferred from an AST check. + +The corrected checkpoint test selection passed all 58 invented controls against +the current integration checkpoint source, including nullable integer widths, +missing-value masks, deterministic round trips and malformed v4 refusals. It +took 5.36 seconds wall time and 348.5 MB peak RSS. All 45 source/control files, +three provider pins and two code resources matched in the external postcheck. +Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. + +Fable's earlier replay review closed seven findings. A separate review of the +published consolidation identified the checkpoint test gap and additional +grouped-bound, metadata-store and facade coverage work, which remains open. +Environment-specific prepatch byte fixtures do not establish portable CI byte +parity. Existing v1 graph stores remain preserved; v2 execution uses a new store +rather than silently promoting old frames. + +Private runtime records and generated population artifacts stay outside this source PR. Receipt identifiers are audit references, not reproducibility inputs. ## Component evidence and remaining work @@ -69,14 +89,20 @@ The following work remains open: 1. Finish current survey predictors and demographic graph bindings, then connect them to the genuine combined survey population. -2. Complete the 59-output PUF donor profile, target-year growth and SCF loan - inputs. The existing 65-output mechanism remains a separately tested - compatibility path; a passing synthetic donor does not supply the missing - source information. +2. Integrate the genuine 59-output donor and its corrected integrity codec with + the combined population, complete source-qualified Social Security recipient + reconciliation, and supply SCF loan inputs. Separate component runs passed + 128 profile/compatibility controls and 26 current-survey predictor controls; + see [the implementation status](current-survey-puf59-progress.md) and + [donor construction and growth](puf2015-canonical59-and-growth.md). + The genuine 207,692-return donor construction does not establish survey fit + quality, complete enrichment or a releasable population. 3. Verify every applicable model input on each source/clone channel. The proposed - national/CD profile retains 161 required inputs, dropping block and tract - requirements while retaining state and county as model inputs. Prior wages - remain excluded. A source operator's existence is not complete cell coverage. + national/CD profile retains 161 required model inputs, with state and county + but without requiring the engine to consume block and tract. Independently, + the build must retain one assigned Census block and derive its larger + geographies. Prior wages remain excluded. A source operator's existence is + not complete cell coverage. 4. Resolve repeated per-cell identity encoding in the survey verifier. One genuine successor run reached persisted-artifact verification but exhausted its two-hour CPU limit. That run failed; it is not a verified build. Performance diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py new file mode 100644 index 000000000..0a227ebb3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py @@ -0,0 +1,1519 @@ +"""Explicit ASEC demographic source contract: sex, its allocation flag, and the +household reference-person code, restored from the pinned original CSV cohorts. + +Scope fence. This module declares and implements the contract and its reader. +It is deliberately **not** wired into +:mod:`.asec_prepared_source`: the root adjudication of 2026-09-06 authorised +preparing this source code and tiny fixtures independently and authorised no +additional full-source read and no new binding. Nothing here is executed +against a genuine Census member by this lane. + +What the contract refuses to assume, stated once so a later reader cannot +mistake an omission for an oversight: + +* A declared value range is not measured proof that every delivered token is + inside it. ``A_SEX`` is validated against its own printed codes, and a token + that is not a printed code leaves that person's sex **unbound**. Nothing here + maps "not 2" onto male. +* ``AXSEX`` prints two codes inside a wider declared range. A token in the + range but outside the printed codes is an unresolved allocation provenance, + so it leaves that person's sex unbound rather than being smoothed into + "reported". +* ``A_EXPRRP`` codes 1 and 2 are the printed, self-labelled household + reference person. They are the only admitted head evidence here. +* ``P_SEQ`` labels no code in its own dictionary entry. It is carried and + diagnosed against the reference code, and it is never a fallback: a + household whose reference code is absent, duplicated or indeterminate stays + unbound even when exactly one of its members has ``P_SEQ == 1``. +* ``A_FAMREL`` is a **family** relationship and is not read here. ``A_LINENO`` + is a Basic-CPS roster line number read only as a join crosscheck. Neither + substitutes for the household reference code. + +Every printed fact below was read this session from the three pinned public +dictionaries named in :data:`ASEC_DICTIONARY_AUTHORITY`, at the PDF pages +recorded there, and the four entries are textually identical across the three +years. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCASDEMO\x01" +FILENAME = "asec_demographic_source.bin" +ARTIFACT_KIND = "microcosm.asec_demographic_source.v1" +_HEADER_MAX = 262_144 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +class DemographicSourceRefusalError(ValueError): + """Value-free refusal; never carries a row, an identifier or an amount.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise DemographicSourceRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +_ASEC_DICTIONARY_URL_TEMPLATE = ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "{survey_year}/march/asec{survey_year}_ddl_pub_full.pdf" +) +#: Survey year -> the pinned public-use dictionary that prints these entries. +#: The three digests are the same documents the reviewed household +#: reported-income contract already pins; no new pin is introduced here. +ASEC_DICTIONARY_AUTHORITY: Mapping[int, Mapping[str, object]] = MappingProxyType( + { + 2023: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2023), + "sha256": ( + "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117" + ), + } + ), + 2024: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2024), + "sha256": ( + "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840" + ), + } + ), + 2025: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2025), + "sha256": ( + "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f" + ), + } + ), + } +) + + +@dataclass(frozen=True, eq=False) +class AsecDemographicField: + """One printed dictionary entry, restated exactly, with its own role. + + ``named_codes`` holds only codes the dictionary actually prints. Codes + inside ``printed_range`` that it does not print are exposed as + :attr:`unnamed_in_range_codes` and are never treated as a named value. + """ + + name: str + concept: str + printed_length: int + printed_range: tuple[int, int] + printed_position: int + named_codes: Mapping[int, str] + universe_as_printed: str + role: str + quoted_locator: str + pdf_page_1based_by_survey_year: Mapping[int, int] + + @property + def declared_domain(self) -> tuple[int, ...]: + low, high = self.printed_range + return tuple(range(low, high + 1)) + + @property + def unnamed_in_range_codes(self) -> tuple[int, ...]: + return tuple( + code for code in self.declared_domain if code not in self.named_codes + ) + + @property + def token_pattern(self) -> str: + return rf"[0-9]{{1,{self.printed_length}}}" + + def document(self) -> dict: + return { + "name": self.name, + "concept": self.concept, + "printed_length": self.printed_length, + "printed_range": list(self.printed_range), + "printed_position": self.printed_position, + "named_codes": { + str(code): self.named_codes[code] for code in sorted(self.named_codes) + }, + "unnamed_in_range_codes": list(self.unnamed_in_range_codes), + "universe_as_printed": self.universe_as_printed, + "role": self.role, + "quoted_locator": self.quoted_locator, + "token_pattern": self.token_pattern, + "authority": [ + { + "survey_year": year, + "url": ASEC_DICTIONARY_AUTHORITY[year]["url"], + "sha256": ASEC_DICTIONARY_AUTHORITY[year]["sha256"], + "pdf_page_1based": page, + } + for year, page in sorted(self.pdf_page_1based_by_survey_year.items()) + ], + } + + +A_SEX = AsecDemographicField( + name="A_SEX", + concept="Sex", + printed_length=1, + printed_range=(1, 2), + printed_position=92, + named_codes=MappingProxyType({1: "Male", 2: "Female"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_SEX / Sex / 1 (1:2) 92 / Values: 1 = Male 2 = Female / " + "Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 22, 2024: 23, 2025: 24}), +) +AXSEX = AsecDemographicField( + name="AXSEX", + concept="Allocation flag for A_SEX", + printed_length=1, + printed_range=(0, 4), + printed_position=161, + named_codes=MappingProxyType({0: "No change", 4: "Allocated"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="AXSEX / Allocation flag for A_SEX / 1 (0:4) 161 / " + "Values: 0 = No change 4 = Allocated / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 27, 2024: 28, 2025: 29}), +) +A_EXPRRP = AsecDemographicField( + name="A_EXPRRP", + concept="Expanded relationship code", + printed_length=2, + printed_range=(1, 14), + printed_position=82, + named_codes=MappingProxyType( + { + 1: "Reference person with relatives", + 2: "Reference person without relatives", + 3: "Husband", + 4: "Wife", + 5: "Own child", + 7: "Grandchild", + 8: "Parent", + 9: "Brother/sister", + 10: "Other relative", + 11: "Foster child", + 12: "Nonrelative with relatives", + 13: "Partner/roommate", + 14: "Nonrelative without relatives", + } + ), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_EXPRRP / Expanded relationship code / 2 (1:14) 82 / " + "Values: 1 = Reference person with relatives 2 = Reference person " + "without relatives 3 = Husband 4 = Wife 5 = Own child 7 = Grandchild " + "8 = Parent 9 = Brother/sister 10 = Other relative 11 = Foster child " + "12 = Nonrelative with relatives 13 = Partner/roommate 14 = Nonrelative " + "without relatives / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 21, 2024: 22, 2025: 23}), +) +P_SEQ = AsecDemographicField( + name="P_SEQ", + concept="Sequence number of person in hhld", + printed_length=2, + printed_range=(0, 16), + printed_position=10, + named_codes=MappingProxyType({}), + universe_as_printed="All Persons", + role="diagnostic_crosscheck", + quoted_locator="P_SEQ / Sequence number of person in hhld / 2 (00:16) 10 / " + "Values: 0-16 / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 20, 2024: 21, 2025: 22}), +) + +#: The four printed entries this contract covers, in output order. +ASEC_DEMOGRAPHIC_FIELDS: tuple[AsecDemographicField, ...] = ( + A_SEX, + AXSEX, + A_EXPRRP, + P_SEQ, +) +OBSERVATIONS: tuple[str, ...] = tuple(field.name for field in ASEC_DEMOGRAPHIC_FIELDS) +ALIASES: tuple[str, ...] = tuple(f"asec_{name}" for name in OBSERVATIONS) +#: The only codes admitted as household reference-person evidence. Both are +#: printed, self-labelled reference-person codes; the split is with/without +#: relatives and is irrelevant to headship itself. +HOUSEHOLD_REFERENCE_CODES: tuple[int, ...] = (1, 2) + +#: Signals deliberately not adopted as semantic proof of headship, each with +#: the reason it is refused. Declared so the refusal survives a later reader. +NON_ADOPTED_HEAD_SIGNALS: tuple[Mapping[str, str], ...] = ( + MappingProxyType( + { + "signal": "P_SEQ == 1", + "kind": "positional_sequence_within_household", + "refusal": ( + "P_SEQ's own dictionary entry labels no code, including 1, as " + "reference person or household head; the ordering statement " + "that would support it is expressly limited to the ASCII file " + "while this restoration reads the CSV member" + ), + "carried_as": "diagnostic_crosscheck", + } + ), + MappingProxyType( + { + "signal": "A_FAMREL == 1", + "kind": "family_relationship", + "refusal": ( + "A_FAMREL is a family relationship scoped to primary-family " + "membership; a subfamily reference person is not its code 1, " + "so it is not a household role and is not read here" + ), + "carried_as": "not_read", + } + ), + MappingProxyType( + { + "signal": "A_LINENO == 1", + "kind": "basic_cps_roster_line_number", + "refusal": ( + "A_LINENO is a separately assigned Basic-CPS roster line " + "number with no source guarantee of row-for-row agreement " + "with any ASEC sequence; it is read only as a join crosscheck" + ), + "carried_as": "join_crosscheck", + } + ), +) + +#: Person-level derived production fields, with their declared code meanings. +SEX_BINDING_STATES: Mapping[int, str] = MappingProxyType( + {0: "unbound", 1: "bound_male", 2: "bound_female"} +) +SEX_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "sex_code_outside_printed_codes", + 2: "allocation_flag_outside_printed_codes", + } +) +SEX_ALLOCATION_STATES: Mapping[int, str] = MappingProxyType( + {0: "undetermined", 1: "no_change", 2: "census_allocated"} +) +HOUSEHOLD_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "bound_household_reference_person", + 2: "bound_not_household_reference_person", + } +) +HOUSEHOLD_REFERENCE_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "relationship_code_outside_printed_codes", + 2: "household_reference_absent", + 3: "household_reference_duplicated", + 4: "household_relationship_code_indeterminate", + } +) +DERIVED: tuple[str, ...] = ( + "asec_sex_binding_state", + "asec_sex_unbound_reason", + "asec_sex_allocation_state", + "asec_household_reference_state", + "asec_household_reference_unbound_reason", +) + +COORDINATES: tuple[str, ...] = ( + "person_id", + "income_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +#: Every column the artifact carries, in body order. +COLUMNS: tuple[str, ...] = COORDINATES + ALIASES + DERIVED +#: The exact readset: the only columns read from any Census member here. +ASEC_DEMOGRAPHIC_SOURCE_COLUMNS: tuple[str, ...] = ( + "PERIDNUM", + "PH_SEQ", + "A_LINENO", + "A_AGE", + *OBSERVATIONS, +) +_COORDINATE_COLUMNS = ("PH_SEQ", "A_LINENO", "A_AGE") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +#: Income year -> survey year. The observation is the interview household one +#: year after the income year; this module harmonizes no period. +COHORTS: Mapping[int, int] = MappingProxyType( + {year: year + 1 for year, *_rest in _MEMBER_PINS} +) +OBSERVED_PERIOD_KIND = "interview_household_one_year_after_income_year" + + +def _code_document(codes: Mapping[int, str]) -> dict: + """Codes are documented with string keys so a JSON replay compares equal.""" + return {str(code): codes[code] for code in sorted(codes)} + + +def contract_document() -> dict: + """The full declared contract, as it is recorded in every receipt.""" + return { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "fields": [field.document() for field in ASEC_DEMOGRAPHIC_FIELDS], + "readset": list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + "missing_token_policy": "refuse_source_member_without_filling_or_coercion", + "incumbent_relationship_policy": ( + "A_EXPRRP presence does not attest source observation; compare and " + "diagnose discrepancies without replacing or trusting the incumbent recode" + ), + "production_fields": { + "restored_observations": [ + { + "column": alias, + "source_column": name, + "role": field.role, + "dtype": "int64", + } + for alias, name, field in zip( + ALIASES, OBSERVATIONS, ASEC_DEMOGRAPHIC_FIELDS, strict=True + ) + ], + "derived": [ + { + "column": "asec_sex_binding_state", + "dtype": "int64", + "codes": _code_document(SEX_BINDING_STATES), + }, + { + "column": "asec_sex_unbound_reason", + "dtype": "int64", + "codes": _code_document(SEX_UNBOUND_REASONS), + }, + { + "column": "asec_sex_allocation_state", + "dtype": "int64", + "codes": _code_document(SEX_ALLOCATION_STATES), + }, + { + "column": "asec_household_reference_state", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_STATES), + }, + { + "column": "asec_household_reference_unbound_reason", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_UNBOUND_REASONS), + }, + ], + }, + "household_reference_codes": list(HOUSEHOLD_REFERENCE_CODES), + "non_adopted_head_signals": [dict(item) for item in NON_ADOPTED_HEAD_SIGNALS], + "household_and_family_roles_separate": True, + "acs_diagnostic_reference_contract": { + "source_column": "RELSHIPP", + "declared_range": [20, 38], + "housing_unit_reference_code": 20, + "group_quarters_codes": { + "37": "institutionalized", + "38": "noninstitutionalized", + }, + "authority": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pdf_page_1based": 43, + }, + "production_acs_values_changed": False, + }, + "family_relationship_field_read": False, + "line_number_used_for_headship": False, + "sex_binding_requires": [ + "A_SEX in printed codes", + "AXSEX in printed codes", + ], + "household_reference_binding_requires": [ + "every A_EXPRRP in the household inside printed codes", + "exactly one household member carrying code 1 or 2", + ], + "household_membership_verification": { + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "merged_published_households": "refused", + "split_published_households": "refused", + "native_household_id_reuse_across_cohorts": "admitted_and_counted", + }, + "other_than_two_is_male": False, + "cohorts": { + str(year): { + "income_year": year, + "survey_year": survey, + "observed_period_kind": OBSERVED_PERIOD_KIND, + "aged_from_a_date": False, + } + for year, survey in sorted(COHORTS.items()) + }, + "release_eligible": False, + "wired_into_prepared_source": False, + } + + +def _implementation() -> dict: + package = resources.files(__package__) + return { + "schema": 1, + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_demographic_source.py", + "asec_person_income_source.py", + "asec_student_controls.py", + ) + }, + "legacy_verification": legacy._verification_identity(), + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "contract": contract_document(), + } + + +def read_demographic_member(path, *, rows: int, size: int) -> pd.DataFrame: + """Parse one member's readset as exact tokens; never fill, never coerce. + + Lexical admission comes from the dictionary's own printed field length: + a token must be digits only and no longer than the printed width, so a + two-character ``A_SEX`` or a signed token is refused here rather than + reaching the domain classification as a plausible code. + """ + path = Path(path) + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) + and set(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + table = pd.read_csv( + path, + usecols=list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + dtype="string", + na_filter=False, + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for name in _COORDINATE_COLUMNS: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + for field in ASEC_DEMOGRAPHIC_FIELDS: + _require( + bool(table[field.name].str.fullmatch(field.token_pattern).all()), + "MEMBER_TOKEN_WIDTH", + field.name, + ) + table[field.name] = table[field.name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "MEMBER_COORDINATES", + ) + return table + + +def _counts(codes: np.ndarray, meanings: Mapping[int, str]) -> dict: + return {meanings[code]: int((codes == code).sum()) for code in sorted(meanings)} + + +#: The published household key. ``A_EXPRRP`` names a person's role inside the +#: household the Census published, and that household is identified by its +#: native ``PH_SEQ`` **within one income year**: an unrelated household in +#: another year legitimately carries the same native id. +HOUSEHOLD_MEMBERSHIP_KEY: tuple[str, ...] = ("income_year", "source_household_id") +#: The parent roster column headship is grouped by. It is a remapped id, not +#: the published one, so its partition is proven equal to the key above rather +#: than assumed to be. +HOUSEHOLD_GROUPING_COLUMN = "person_household_id" +#: Counters below that any nonzero value would have refused before an artifact +#: exists. Named so a later reader cannot read their zero as a measurement. +MEMBERSHIP_REFUSAL_COUNTERS: tuple[str, ...] = ( + "grouping_households_spanning_multiple_cohorts", + "grouping_households_spanning_multiple_published_households", + "published_households_spanning_multiple_grouping_households", +) +ZERO_COUNTER_SEMANTICS = ( + "derived from the delivered values, but any nonzero value refuses the run " + "before an artifact exists, so a zero in a successful receipt attests that " + "refusal and is not an independent measurement of the source" +) + + +def _dense_codes(*keys: np.ndarray) -> tuple[np.ndarray, int]: + """Number the distinct key tuples 0..n-1 without combining them numerically. + + Key columns are compared component-wise after a lexicographic sort, so no + two coordinates are ever multiplied or concatenated into one integer and + no width assumption can silently collide two distinct households. + """ + order = np.lexsort(tuple(reversed(keys))) + changed = np.zeros(len(order), dtype=bool) + changed[0] = True + for values in keys: + sorted_values = values[order] + changed[1:] |= sorted_values[1:] != sorted_values[:-1] + codes = np.empty(len(order), dtype=np.int64) + codes[order] = np.cumsum(changed) - 1 + return codes, int(changed.sum()) + + +def _representatives(codes: np.ndarray, count: int) -> np.ndarray: + """One arbitrary but deterministic row position per distinct code.""" + positions = np.zeros(count, dtype=np.int64) + positions[codes] = np.arange(len(codes), dtype=np.int64) + return positions + + +def verify_household_membership( + *, + income_year: np.ndarray, + source_household_id: np.ndarray, + household_id: np.ndarray, +) -> dict: + """Prove the grouping column partitions persons exactly as the publisher did. + + Household reference status is a statement about the household the Census + published: the cardinality of ``A_EXPRRP`` 1/2 is evidence only inside + ``(income_year, source_household_id)``. Classification groups instead by + the parent roster's remapped :data:`HOUSEHOLD_GROUPING_COLUMN`, and a + per-row equality between that roster's ``source_household_id`` and the + member's ``PH_SEQ`` says nothing about how either column *groups*. So the + two partitions are proven identical in both directions here, before any + household verdict is formed. + + Refused in both directions: a grouping household carrying persons from + more than one published household (a merge, whether inside one cohort or + across cohorts), and a published household split across more than one + grouping household. Admitted and counted: the same native household id + reused by unrelated households in different income years, which is + ordinary and is what makes the income year part of the key. + + Returns the aggregate verification recorded in the artifact receipt. It + carries counts, column names and cohort years only; no person, household + or native identifier appears in it or in any refusal raised here. + """ + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "MEMBERSHIP_ROWS") + arrays = { + "income_year": income_year, + "source_household_id": source_household_id, + HOUSEHOLD_GROUPING_COLUMN: household_id, + } + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "MEMBERSHIP_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()), + "MEMBERSHIP_COHORT", + "income_year", + ) + for name in ("source_household_id", HOUSEHOLD_GROUPING_COLUMN): + _require(bool((arrays[name] > 0).all()), "MEMBERSHIP_COORDINATE", name) + + published, published_count = _dense_codes(income_year, source_household_id) + grouping, grouping_count = _dense_codes(household_id) + edges, edge_count = _dense_codes(grouping, published) + # The relation between the two partitions is exactly its distinct + # (grouping household, published household) pairs; take one row per pair. + edge_rows = _representatives(edges, edge_count) + edge_grouping = grouping[edge_rows] + edge_published = published[edge_rows] + edge_year = income_year[edge_rows] + + _, grouping_degree = np.unique(edge_grouping, return_counts=True) + _, published_degree = np.unique(edge_published, return_counts=True) + cohort_edges, cohort_edge_count = _dense_codes(edge_grouping, edge_year) + _, cohort_degree = np.unique( + edge_grouping[_representatives(cohort_edges, cohort_edge_count)], + return_counts=True, + ) + native_rows = _representatives(published, published_count) + _, native_degree = np.unique(source_household_id[native_rows], return_counts=True) + + collisions = int((cohort_degree > 1).sum()) + merged = int((grouping_degree > 1).sum()) + split = int((published_degree > 1).sum()) + _require(collisions == 0, "MEMBERSHIP_COHORT_COLLISION", HOUSEHOLD_GROUPING_COLUMN) + _require( + merged == 0, "MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + _require( + split == 0, "MEMBERSHIP_SPLIT_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + # A relation whose edge count equals both vertex counts, with every vertex + # incident to at least one edge by construction, is a bijection. Asserted + # independently of the three counters above so a gap in any one of them + # still cannot let an unequal partition through. + _require( + edge_count == grouping_count == published_count, + "MEMBERSHIP_NOT_BIJECTIVE", + HOUSEHOLD_GROUPING_COLUMN, + ) + + return { + "schema_version": 1, + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "rows": rows, + "grouping_households": grouping_count, + "published_households": published_count, + "membership_edges": edge_count, + # Recomputable from the three counts recorded immediately above it. + "bijective": edge_count == grouping_count == published_count, + "grouping_households_spanning_multiple_cohorts": collisions, + "grouping_households_spanning_multiple_published_households": merged, + "published_households_spanning_multiple_grouping_households": split, + "native_household_ids_reused_across_cohorts": int((native_degree > 1).sum()), + "native_household_id_reuse_across_cohorts_is_admitted": True, + "by_cohort": { + str(year): { + "income_year": int(year), + "survey_year": int(COHORTS[int(year)]), + "rows": int((income_year == year).sum()), + "grouping_households": int( + np.unique(household_id[income_year == year]).size + ), + "published_households": int( + np.unique(published[income_year == year]).size + ), + "native_household_ids": int( + np.unique(source_household_id[income_year == year]).size + ), + } + for year in sorted({int(value) for value in np.unique(income_year)}) + }, + "counters_zero_in_every_successful_receipt": list(MEMBERSHIP_REFUSAL_COUNTERS), + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + + +@dataclass(frozen=True) +class DemographicClassification: + """Per-person states plus value-free counts. Holds no source identifier.""" + + states: Mapping[str, np.ndarray] + diagnostics: dict + + +def classify_asec_demographic_observations( + *, + income_year: np.ndarray, + household_id: np.ndarray, + a_sex: np.ndarray, + axsex: np.ndarray, + a_exprrp: np.ndarray, + p_seq: np.ndarray, +) -> DemographicClassification: + """Bind only what the printed codes and the household cardinality support. + + Sex binds when the sex code is printed and its allocation flag is printed; + the allocation state is carried alongside, never folded into the value. + Household headship binds when every relationship code in the household is + printed and exactly one member carries a reference-person code. Every other + person is reported unbound with the reason, and ``P_SEQ`` is diagnosed + against the result without ever supplying it. + + ``household_id`` is the grouping column, and the reference-person + cardinality this reads is only evidence about the household the Census + published. The caller must therefore have proven that grouping equal to + ``(income_year, source_household_id)`` with + :func:`verify_household_membership` first; the cohort-uniformity check + below is a weaker self-check and is not that proof. + """ + arrays = { + "income_year": income_year, + "household_id": household_id, + "A_SEX": a_sex, + "AXSEX": axsex, + "A_EXPRRP": a_exprrp, + "P_SEQ": p_seq, + } + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "CLASSIFY_ROWS") + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "CLASSIFY_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()) + and bool((household_id > 0).all()), + "CLASSIFY_COORDINATES", + ) + + sex_state = np.zeros(rows, dtype=np.int64) + sex_reason = np.zeros(rows, dtype=np.int64) + allocation = np.zeros(rows, dtype=np.int64) + named_sex = np.isin(a_sex, list(A_SEX.named_codes)) + named_flag = np.isin(axsex, list(AXSEX.named_codes)) + allocation[named_flag & (axsex == 0)] = 1 + allocation[named_flag & (axsex == 4)] = 2 + admissible = named_sex & named_flag + sex_state[admissible & (a_sex == 1)] = 1 + sex_state[admissible & (a_sex == 2)] = 2 + sex_reason[~named_sex] = 1 + sex_reason[named_sex & ~named_flag] = 2 + + named_relationship = np.isin(a_exprrp, list(A_EXPRRP.named_codes)) + reference = np.isin(a_exprrp, list(HOUSEHOLD_REFERENCE_CODES)) + head_state = np.zeros(rows, dtype=np.int64) + head_reason = np.zeros(rows, dtype=np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + households = len(starts) + household_verdicts = np.zeros(households, dtype=np.int64) + for index in range(households): + positions = order[bounds[index] : bounds[index + 1]] + _require( + len(np.unique(income_year[positions])) == 1, "CLASSIFY_HOUSEHOLD_COHORT" + ) + unnamed = ~named_relationship[positions] + if unnamed.any(): + head_reason[positions] = 4 + head_reason[positions[unnamed]] = 1 + household_verdicts[index] = 4 + continue + references = positions[reference[positions]] + if len(references) == 0: + head_reason[positions] = 2 + household_verdicts[index] = 2 + continue + if len(references) > 1: + head_reason[positions] = 3 + household_verdicts[index] = 3 + continue + head_state[positions] = 2 + head_state[references[0]] = 1 + household_verdicts[index] = 1 + + # P_SEQ is diagnosed, never consulted. Nothing above reads it. + p_seq_one = p_seq == 1 + in_declared_range = np.isin(p_seq, list(P_SEQ.declared_domain)) + household_years = income_year[order[bounds[:-1]]] + household_p_seq_ones = np.add.reduceat(p_seq_one[order].astype(np.int64), starts) + + diagnostics: dict = { + "rows": rows, + "households": households, + "by_cohort": {}, + "totals": {}, + "p_seq_crosscheck": {}, + } + cohorts = sorted({int(year) for year in np.unique(income_year)}) + for scope, mask, household_mask in [ + ("totals", np.ones(rows, dtype=bool), np.ones(households, dtype=bool)), + *[ + ( + str(year), + income_year == year, + household_years == year, + ) + for year in cohorts + ], + ]: + block = { + "income_year": None if scope == "totals" else int(scope), + "survey_year": None + if scope == "totals" + else int(COHORTS.get(int(scope), int(scope) + 1)), + "observed_period_kind": OBSERVED_PERIOD_KIND, + "rows": int(mask.sum()), + "households": int(household_mask.sum()), + "field_domains": { + field.name: { + "rows_in_declared_range": int( + np.isin(arrays[field.name][mask], field.declared_domain).sum() + ), + "rows_outside_declared_range": int( + ( + ~np.isin(arrays[field.name][mask], field.declared_domain) + ).sum() + ), + "rows_in_named_codes": int( + np.isin(arrays[field.name][mask], list(field.named_codes)).sum() + ), + "rows_in_unnamed_range_codes": int( + np.isin( + arrays[field.name][mask], field.unnamed_in_range_codes + ).sum() + ), + "role": field.role, + } + for field in ASEC_DEMOGRAPHIC_FIELDS + }, + "sex_binding_state": _counts(sex_state[mask], SEX_BINDING_STATES), + "sex_unbound_reason": _counts(sex_reason[mask], SEX_UNBOUND_REASONS), + "sex_allocation_state": _counts(allocation[mask], SEX_ALLOCATION_STATES), + "household_reference_state": _counts( + head_state[mask], HOUSEHOLD_REFERENCE_STATES + ), + "household_reference_unbound_reason": _counts( + head_reason[mask], HOUSEHOLD_REFERENCE_UNBOUND_REASONS + ), + "household_reference_verdict": _counts( + household_verdicts[household_mask], + MappingProxyType( + { + 1: "exactly_one_reference", + 2: "no_reference", + 3: "multiple_references", + 4: "indeterminate_relationship_code", + } + ), + ), + "p_seq_crosscheck": { + "declared_role": P_SEQ.concept, + "named_codes": {}, + "is_semantic_proof_of_headship": False, + "used_as_fallback": False, + "rows_with_p_seq_one": int(p_seq_one[mask].sum()), + "rows_outside_declared_range": int((~in_declared_range[mask]).sum()), + "households_with_one_p_seq_one": int( + (household_p_seq_ones[household_mask] == 1).sum() + ), + "households_with_no_p_seq_one": int( + (household_p_seq_ones[household_mask] == 0).sum() + ), + "households_with_multiple_p_seq_one": int( + (household_p_seq_ones[household_mask] > 1).sum() + ), + "agreement_rows": int((p_seq_one & reference)[mask].sum()), + "p_seq_one_without_reference_code": int( + (p_seq_one & ~reference)[mask].sum() + ), + "reference_code_without_p_seq_one": int( + (reference & ~p_seq_one)[mask].sum() + ), + "unbound_households_with_one_p_seq_one": int( + ((household_verdicts != 1) & (household_p_seq_ones == 1))[ + household_mask + ].sum() + ), + }, + } + if scope == "totals": + diagnostics["totals"] = block + diagnostics["p_seq_crosscheck"] = block["p_seq_crosscheck"] + else: + diagnostics["by_cohort"][scope] = block + diagnostics["family_relationship_field_read"] = False + diagnostics["line_number_used_for_headship"] = False + diagnostics["household_and_family_roles_separate"] = True + return DemographicClassification( + MappingProxyType( + { + "asec_sex_binding_state": sex_state, + "asec_sex_unbound_reason": sex_reason, + "asec_sex_allocation_state": allocation, + "asec_household_reference_state": head_state, + "asec_household_reference_unbound_reason": head_reason, + } + ), + diagnostics, + ) + + +#: The ACS side is *not* rewritten by this contract. These are the pinned 2024 +#: 1-year PUMS codes the existing reader already branches on, restated so the +#: shared classifier below can read the ACS arm without inventing a head. +ACS_OBSERVED_REFERENCE_CODE = 20 +ACS_GROUP_QUARTERS_CODES: tuple[int, int] = (37, 38) +#: What ``acs_pums`` already writes for a group-quarters-only household. It is +#: a printed A_EXPRRP nonrelative code, deliberately not a reference code, so a +#: synthetic group-quarters representative can never be read as an observed +#: housing-unit reference person. +ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE = 14 +ACS_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "observed_housing_unit_reference_person", + 2: "observed_not_reference_person", + 3: "group_quarters_no_observed_reference_person", + } +) + + +def acs_household_reference_states( + *, household_id: np.ndarray, relshipp: np.ndarray +) -> tuple[np.ndarray, dict]: + """Read the ACS arm's own observed reference person; never manufacture one. + + ``RELSHIPP == 20`` is the self-labelled reference person of a housing unit. + Codes 37 and 38 are the institutional and noninstitutional group-quarters + populations; they are neither a reference person nor a relationship to one, + so their households report *no* observed reference person and stay + distinguishable from a housing unit throughout. + """ + rows = len(household_id) + _require(0 < rows <= _MAX_PERSONS, "ACS_ROWS") + for name, values in (("household_id", household_id), ("RELSHIPP", relshipp)): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "ACS_INPUT", + name, + ) + group_quarters = np.isin(relshipp, list(ACS_GROUP_QUARTERS_CODES)) + valid_relationship = np.isin(relshipp, np.arange(20, 39)) + reference = relshipp == ACS_OBSERVED_REFERENCE_CODE + states = np.where(reference, 1, 2).astype(np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + housing_units = 0 + gq_households = 0 + unbound_housing_units = 0 + bad_reference_cardinality = 0 + mixed_households = 0 + invalid_households = 0 + for index in range(len(starts)): + positions = order[bounds[index] : bounds[index + 1]] + if group_quarters[positions].all(): + states[positions] = 3 + gq_households += 1 + continue + housing_units += 1 + bad_reference_cardinality += int(int(reference[positions].sum()) != 1) + mixed_households += int(group_quarters[positions].any()) + invalid_households += int(not valid_relationship[positions].all()) + if ( + group_quarters[positions].any() + or not valid_relationship[positions].all() + or int(reference[positions].sum()) != 1 + ): + states[positions] = 0 + unbound_housing_units += 1 + diagnostics = { + "rows": rows, + "households": len(starts), + "housing_unit_households": housing_units, + "group_quarters_only_households": gq_households, + "unbound_housing_unit_households": unbound_housing_units, + "housing_unit_households_without_exactly_one_reference": bad_reference_cardinality, + "mixed_housing_unit_group_quarters_households": mixed_households, + "households_with_invalid_relationship_codes": invalid_households, + "state_counts": _counts(states, ACS_REFERENCE_STATES), + "group_quarters_rows": int(group_quarters.sum()), + "institutional_group_quarters_rows": int((relshipp == 37).sum()), + "noninstitutional_group_quarters_rows": int((relshipp == 38).sum()), + "manufactured_group_quarters_reference_person": False, + "synthetic_group_quarters_relationship_code": ( + ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE + ), + } + return states, diagnostics + + +@dataclass(frozen=True) +class AuthenticatedAsecDemographicSource: + """Immutable owned numeric buffers issued only by closed reconstruction.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "DEMOGRAPHIC_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self) -> dict: + return json.loads(self._header.decode("ascii")) + + @property + def content_sha256(self) -> str: + return _sha(self._header + self._body) + + def validate(self) -> None: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "DEMOGRAPHIC_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and _json(header["contract"]) == _json(contract_document()) + and _json(header["implementation"]) == _json(_implementation()), + "DEMOGRAPHIC_CONTENT_CHANGED", + ) + + def array(self, name: str) -> np.ndarray: + self.validate() + _require(name in COLUMNS, "DEMOGRAPHIC_COLUMN") + rows = self.receipt["rows"] + offset = COLUMNS.index(name) * rows * 8 + return np.frombuffer(self._body, dtype=" bytes: + value.validate() + payload = ( + MAGIC + struct.pack(" None: + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource, + "DEMOGRAPHIC_AUTHENTICATED_PARENT", + ) + source.validate() + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + restoration._paths(member_paths) + _parent(source) + before = _implementation() + person = source.frame.person + rows = len(person) + _require(0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require(not set(ALIASES) & set(person), "DEMOGRAPHIC_ALREADY_ATTACHED") + _require( + set(COORDINATES[2:]) <= set(person) and "person_household_id" in person, + "DEMOGRAPHIC_PARENT_ROSTER", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output: dict[str, np.ndarray] = { + "person_id": np.asarray(source.scope.person_ids, dtype=np.int64), + "income_year": years, + } + for name in COORDINATES[2:]: + _require( + person[name].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + name, + ) + output[name] = person[name].to_numpy(copy=True) + # The grouping column is a coordinate of the headship verdict, so it is + # typed here rather than coerced by the conversion that consumes it. + _require( + person[HOUSEHOLD_GROUPING_COLUMN].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + HOUSEHOLD_GROUPING_COLUMN, + ) + household_id = person[HOUSEHOLD_GROUPING_COLUMN].to_numpy(copy=True) + for alias in ALIASES: + output[alias] = np.empty(rows, dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-members-" + ) as directory: + for year, member, archive_pin, pin, member_rows, size in _MEMBER_PINS: + staged = Path(directory) / member + _require( + _snapshot(member_paths[year], staged, size=size) == pin, + "DEMOGRAPHIC_SOURCE_BYTES", + ) + raw = read_demographic_member(staged, rows=member_rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == member_rows, "DEMOGRAPHIC_COHORT_ROWS") + recipient = pd.Index(keys[positions]) + _require(not recipient.has_duplicates, "DEMOGRAPHIC_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == member_rows, + "DEMOGRAPHIC_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + coordinate_conflicts = 0 + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + conflicts = int( + (output[name][positions] != joined[raw_name].to_numpy()).sum() + ) + coordinate_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_NATIVE_KEY_OR_AGE", name) + compared = {} + incumbent_conflicts = 0 + relationship_crosscheck = { + "incumbent_present": "A_EXPRRP" in person, + "compared_rows": 0, + "missing_rows": 0, + "mismatch_rows": 0, + "presence_certifies_source_observation": False, + "used_as_semantic_proof": False, + } + for alias, name in zip(ALIASES, OBSERVATIONS, strict=True): + observed = joined[name].to_numpy(dtype=np.int64) + output[alias][positions] = observed + if name not in person: + compared[name] = None + continue + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + equal = ( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known] + == observed[known] + ) + if name == "A_EXPRRP": + # Older prepared cohorts carry an A_LINENO-derived recode. + # The restored alias owns the CSV observation; an unverified + # incumbent may diagnose a discrepancy but cannot veto it. + relationship_crosscheck.update( + compared_rows=int(known.sum()), + missing_rows=int((~known).sum()), + mismatch_rows=int((~equal).sum()), + ) + compared[name] = int(known.sum()) + continue + conflicts = int((~equal).sum()) + incumbent_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_INCUMBENT_CONFLICT", name) + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": COHORTS[year], + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": member_rows, + "joined_rows": len(positions), + "unreferenced_source_rows": int( + member_rows - len(np.unique(indices)) + ), + "incumbent_compared_rows": compared, + "incumbent_relationship_crosscheck": relationship_crosscheck, + "accepted_source_missing_tokens": dict.fromkeys(OBSERVATIONS, 0), + "incumbent_absent_columns": sorted( + name for name, value in compared.items() if value is None + ), + "incumbent_conflicts": incumbent_conflicts, + "native_key_or_age_conflicts": coordinate_conflicts, + "counters_zero_in_every_successful_receipt": [ + "accepted_source_missing_tokens", + "incumbent_conflicts", + "native_key_or_age_conflicts", + "unreferenced_source_rows", + ], + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + ) + membership = verify_household_membership( + income_year=years, + source_household_id=output["source_household_id"], + household_id=household_id, + ) + classification = classify_asec_demographic_observations( + income_year=years, + household_id=household_id, + a_sex=output["asec_A_SEX"], + axsex=output["asec_AXSEX"], + a_exprrp=output["asec_A_EXPRRP"], + p_seq=output["asec_P_SEQ"], + ) + output.update({name: values for name, values in classification.states.items()}) + _parent(source) + _require( + _json(_implementation()) == _json(before), + "DEMOGRAPHIC_IMPLEMENTATION_CHANGED", + ) + buffers = [output[name].astype(" AuthenticatedAsecDemographicSource: + """Reconstruct from pinned members; a candidate supplies bytes, not content.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + payload = _encode(expected) + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-candidate-" + ) as directory: + digest = _snapshot( + candidate_path, Path(directory) / FILENAME, size=len(payload) + ) + _require(digest == _sha(payload), "DEMOGRAPHIC_CANONICAL_BYTES") + _parent(source) + expected.validate() + return expected + except DemographicSourceRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + re.error, + ): + raise DemographicSourceRefusalError("DEMOGRAPHIC_SOURCE_CONTRACT") from None + + +def _attachment_receipt(frame, source, observations) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_demographic_source_attachment.v1", + "source_identity_sha256": _sha(source.source.identity), + "observations_content_sha256": observations.content_sha256, + "columns": list(ALIASES + DERIVED), + "output_frame_sha256": legacy._frame_signature(frame), + "release_eligible": False, + } + + +def _check_attachment_parents(source, observations) -> None: + _parent(source) + _require( + type(observations) is AuthenticatedAsecDemographicSource, + "DEMOGRAPHIC_AUTHENTICATED_OBSERVATIONS", + ) + observations.validate() + _require( + observations.receipt["source_identity"].encode() == source.source.identity, + "DEMOGRAPHIC_ATTACHMENT_PARENT", + ) + + +@dataclass(frozen=True) +class DemographicSourceAttachedAsec: + """Owned additive Frame beside an unchanged authenticated money parent.""" + + frame: Frame + parent: legacy.AuthenticatedCurrentMoneySource + observations: AuthenticatedAsecDemographicSource + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require( + _token is _COMPOSE_TOKEN, + "DEMOGRAPHIC_ATTACHMENT_CONSTRUCTOR_UNAVAILABLE", + ) + + @property + def receipt(self) -> dict: + return json.loads(self._receipt.decode("ascii")) + + def validate(self) -> None: + try: + _check_attachment_parents(self.parent, self.observations) + _require(type(self.frame) is Frame, "DEMOGRAPHIC_ATTACHMENT_FRAME") + for name in ALIASES + DERIVED: + values = self.frame.person[name] + _require( + values.dtype == np.dtype("int64") + and np.array_equal( + values.to_numpy(), self.observations.array(name) + ), + "DEMOGRAPHIC_ATTACHMENT_OUTPUT", + name, + ) + recovered = restoration._owned_frame(self.frame) + recovered.person.drop(columns=list(ALIASES + DERIVED), inplace=True) + _require( + legacy._frame_signature(recovered) + == self.observations.receipt["source_frame_sha256"] + and self.receipt + == _attachment_receipt(self.frame, self.parent, self.observations), + "DEMOGRAPHIC_ATTACHMENT_CHANGED", + ) + except DemographicSourceRefusalError: + raise + except (ValueError, TypeError, KeyError, AttributeError) as error: + raise DemographicSourceRefusalError( + "DEMOGRAPHIC_ATTACHMENT_CHANGED" + ) from error + + +def attach_asec_demographic_source( + source, observations +) -> DemographicSourceAttachedAsec: + """Append the restored observations and their states; change nothing else.""" + _check_attachment_parents(source, observations) + _require( + not set(ALIASES + DERIVED) & set(source.frame.person), + "DEMOGRAPHIC_ALREADY_ATTACHED", + ) + frame = restoration._owned_frame(source.frame) + for name in ALIASES + DERIVED: + frame.person[name] = observations.array(name).copy() + result = DemographicSourceAttachedAsec( + frame, + source, + observations, + _json(_attachment_receipt(frame, source, observations)), + _token=_COMPOSE_TOKEN, + ) + result.validate() + return result + + +def write_asec_demographic_source(source, *, member_paths, output_dir) -> dict: + """Produce a new immutable local bundle; never replace an existing parent.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + value = load_authenticated_asec_demographic_source( + source, member_paths=member_paths + ) + payload = _encode(value) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-demographic-", dir=destination.parent + ) as directory: + staging = Path(directory) + (staging / FILENAME).write_bytes(payload) + receipt = { + **value.receipt, + "content_sha256": value.content_sha256, + "output_file": FILENAME, + "output_sha256": _sha(payload), + } + (staging / "receipt.json").write_bytes(_json(receipt) + b"\n") + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return receipt + + +__all__ = [ + "ACS_GROUP_QUARTERS_CODES", + "ACS_OBSERVED_REFERENCE_CODE", + "ACS_REFERENCE_STATES", + "ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE", + "ALIASES", + "ARTIFACT_KIND", + "ASEC_DEMOGRAPHIC_FIELDS", + "ASEC_DEMOGRAPHIC_SOURCE_COLUMNS", + "ASEC_DICTIONARY_AUTHORITY", + "COHORTS", + "COLUMNS", + "DERIVED", + "HOUSEHOLD_GROUPING_COLUMN", + "HOUSEHOLD_MEMBERSHIP_KEY", + "HOUSEHOLD_REFERENCE_CODES", + "HOUSEHOLD_REFERENCE_STATES", + "HOUSEHOLD_REFERENCE_UNBOUND_REASONS", + "MEMBERSHIP_REFUSAL_COUNTERS", + "NON_ADOPTED_HEAD_SIGNALS", + "OBSERVATIONS", + "OBSERVED_PERIOD_KIND", + "SEX_ALLOCATION_STATES", + "SEX_BINDING_STATES", + "SEX_UNBOUND_REASONS", + "ZERO_COUNTER_SEMANTICS", + "AsecDemographicField", + "AuthenticatedAsecDemographicSource", + "DemographicClassification", + "DemographicSourceAttachedAsec", + "DemographicSourceRefusalError", + "acs_household_reference_states", + "attach_asec_demographic_source", + "classify_asec_demographic_observations", + "contract_document", + "load_authenticated_asec_demographic_source", + "read_demographic_member", + "verify_household_membership", + "write_asec_demographic_source", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py index fac70b2bc..7270593ea 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py @@ -227,6 +227,61 @@ class CpsCarriedCurrentLeaves: spm_unit: Mapping[str, np.ndarray] +CPS_CURRENT_PREDICTOR_MONEY_FIELDS = ( + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", +) +CPS_CURRENT_PREDICTOR_PERSON_LEAVES = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", +) + + +def derive_cps_current_predictor_leaves(amounts): + """Pure five-field split, sharing the maintained current-money judgments. + + Callers qualify the actual money owner or modeled target artifacts. This + numerical function grants no source authority and never fills an unknown. + Inputs are aligned physical float64 arrays, already in the declared dollar + basis. INT/DIV/CAP fitted for ACS remain modeled, including their splits. + """ + _require(isinstance(amounts, Mapping), "PREDICTOR_MONEY_MAPPING") + _require( + set(amounts) == set(CPS_CURRENT_PREDICTOR_MONEY_FIELDS), + "PREDICTOR_MONEY_ROSTER", + ) + lengths = set() + for value in amounts.values(): + _require( + type(value) is np.ndarray + and value.ndim == 1 + and value.dtype == np.dtype("float64"), + "PREDICTOR_MONEY_TYPE", + ) + _require(bool(np.isfinite(value).all()), "PREDICTOR_MONEY_UNKNOWN") + lengths.add(len(value)) + _require(len(lengths) == 1 and next(iter(lengths)) > 0, "PREDICTOR_MONEY_AXIS") + dividends, gains = amounts["DIV_VAL"], amounts["CAP_VAL"] + return { + "employment_income_before_lsr": amounts["WSAL_VAL"].copy(), + "self_employment_income_before_lsr": amounts["SEMP_VAL"].copy(), + "taxable_interest_income": amounts["INT_VAL"] * TAXABLE_INTEREST_FRACTION, + "qualified_dividend_income": dividends * QUALIFIED_DIVIDEND_FRACTION, + "non_qualified_dividend_income": dividends * (1 - QUALIFIED_DIVIDEND_FRACTION), + "long_term_capital_gains_before_response": gains + * LONG_TERM_CAPITAL_GAIN_FRACTION, + "short_term_capital_gains": gains * (1 - LONG_TERM_CAPITAL_GAIN_FRACTION), + } + + def derive_cps_carried_current_leaves( selected: SelectedCurrentMoney, *, @@ -239,21 +294,14 @@ def derive_cps_carried_current_leaves( rows = selected.person_rows codes = _routing(routing, rows) amounts = _amounts(selected, rows) - dividends = amounts["DIV_VAL"] - gains = amounts["CAP_VAL"] # The money recipe already contributes zero for a declared-NIU annuity, so # the pension base is the plain sum of the two restated amounts. pensions = amounts["PNSN_VAL"] + amounts["ANN_VAL"] person: dict[str, np.ndarray] = { "age": codes["A_AGE"].astype("float64"), - "employment_income_before_lsr": amounts["WSAL_VAL"], - "self_employment_income_before_lsr": amounts["SEMP_VAL"], - "taxable_interest_income": amounts["INT_VAL"] * TAXABLE_INTEREST_FRACTION, - "qualified_dividend_income": dividends * QUALIFIED_DIVIDEND_FRACTION, - "non_qualified_dividend_income": dividends * (1 - QUALIFIED_DIVIDEND_FRACTION), - "long_term_capital_gains_before_response": gains - * LONG_TERM_CAPITAL_GAIN_FRACTION, - "short_term_capital_gains": gains * (1 - LONG_TERM_CAPITAL_GAIN_FRACTION), + **derive_cps_current_predictor_leaves( + {name: amounts[name] for name in CPS_CURRENT_PREDICTOR_MONEY_FIELDS} + ), "taxable_private_pension_income": pensions * TAXABLE_PENSION_FRACTION, "tax_exempt_private_pension_income": pensions * (1 - TAXABLE_PENSION_FRACTION), "taxable_ira_distributions": _ira_distributions(amounts, codes, rows), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py new file mode 100644 index 000000000..12b88afe4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py @@ -0,0 +1,379 @@ +"""Current survey source projection for ASEC sex and household state. + +Reuse the maintained demographic source owner for A_SEX/AXSEX. Add only the +missing GESTFIPS household projection through the original weight owner's +bounded capture/CSV primitives and closed member registry. Returned numerical +values are not source authority; a graph host must re-run this live qualifier +and compare materialized descendants, including after replay. +""" + +from __future__ import annotations + +import csv +import io +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_demographic_source as demographic +from . import asec_original_household_weights as household +from . import source_csv_builtin +from . import survey_population_preparation as preparation_owner +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-demographic-projection.v1" +STATE_COLUMNS = ("H_SEQ", "GESTFIPS") +STATE_CONTRACT = { + "field": "GESTFIPS", + "concept": "State FIPS code", + "entity": "household", + "survey_year": 2025, + "income_year": 2024, + "dictionary_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "dictionary_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 8, + "printed_page": "6A-1", + "printed_length": 2, + "printed_position": 44, + "printed_range": [1, 56], + "universe": "All Households", + "canonical_mapping": "numeric_identity", + "unassigned_jurisdiction_validation": "separate_geography_gate", + "unknown_policy": "preserve_token_and_unbound_state; never_use_carried_state_as_fallback", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_DEMOGRAPHICS_" + reason) + + +def _state_token(token): + """Preserve an unknown literal rather than interpreting it as a state.""" + _require( + type(token) is str and len(token) <= household._MAX_FIELD_CHARS, + "STATE_TOKEN_BOUND", + ) + if token == "": + return None, "missing" + if re.fullmatch(r"[0-9]{1,2}", token, re.ASCII) is None: + return None, "malformed" + code = int(token) + return ( + (code, "in_printed_range") + if 1 <= code <= 56 + else (None, "outside_printed_range") + ) + + +def _read_state_capture(path, pin): + """Low-level literal reader; a caller-supplied pin grants no authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "CSV_READER_CHANGED") + result = {} + with open(path, "rb", buffering=0, opener=household._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = household._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = household._RecordLines(text) + records = reader(lines, strict=True) + header = next(records, []) + _require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(STATE_COLUMNS) <= set(header), + "STATE_HEADER", + ) + positions = [header.index(c) for c in STATE_COLUMNS] + while True: + lines.characters = 0 + try: + row = next(records) + except StopIteration: + break + _require( + len(row) == len(header) and len(result) < pin.rows, + "STATE_ROW_SHAPE", + ) + key, token = (row[i] for i in positions) + _require( + re.fullmatch(r"[0-9]{1,5}", key, re.ASCII) is not None + and 1 <= int(key) <= 99999, + "HOUSEHOLD_KEY", + ) + native = int(key) + _require(native not in result, "HOUSEHOLD_DUPLICATE_KEY") + code, status = _state_token(token) + result[native] = { + "H_SEQ": key, + "GESTFIPS": token, + "state_code": code, + "status": status, + "member_row_1based": len(result) + 1, + } + _require(len(result) == pin.rows, "STATE_ROW_COUNT") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "STATE_CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, k) == getattr(after, k) + for k in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "STATE_CAPTURE_CHANGED", + ) + return result + + +def _load_current_state(member_path, expected_member): + """Closed registry lookup; expected member is retained native-owner evidence.""" + pins = household._registry() + selected = [pin for pin in pins if pin.income_year == 2024] + _require(len(selected) == 1, "CURRENT_STATE_REGISTRY") + pin = selected[0] + _require( + pin.survey_year == 2025 + and pin.canonical_member_id == expected_member["canonical_member_id"] + and pin.member_sha256 == expected_member["member_sha256"] + and pin.archive_sha256 == expected_member["archive_sha256"], + "STATE_NATIVE_MEMBER_BINDING", + ) + registry = household._encode(household._implementation()) + paths = household._member_path_snapshot({2024: member_path}, pins) + with tempfile.TemporaryDirectory(prefix="asec-current-state-") as directory: + capture = Path(directory) / "hhpub25.csv" + _require( + household._capture_owner._snapshot( + paths[2024], capture, size=pin.size_bytes + ) + == pin.member_sha256, + "STATE_SOURCE_SHA256", + ) + rows = _read_state_capture(capture, pin) + _require( + household._encode(household._implementation()) == registry, + "STATE_REGISTRY_CHANGED", + ) + return rows, { + "canonical_member_id": pin.canonical_member_id, + "member_sha256": pin.member_sha256, + "archive_sha256": pin.archive_sha256, + "bytes": pin.size_bytes, + "rows": pin.rows, + } + + +@dataclass(frozen=True) +class CurrentAsecDemographicValues: + """Source-derived selected arrays; no independent authority or launch verdict.""" + + person: pd.DataFrame + household: pd.DataFrame + receipt: bytes + + +def qualify_current_asec_demographics(preparation): + """Reconstruct the real parent sex owner and current household state source. + + The selected current survey is joined after full-source reconstruction. + Missing/unlabelled sex allocation or state codes remain explicitly unknown; + no non-2→male or carried-state fallback is permitted. + """ + _require( + type(preparation) is preparation_owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + native_entry = preparation_owner.asec_native._ISSUED.get(id(native)) + _require( + native_entry is not None + and native_entry[0]() is native + and native.payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + native_receipt = json.loads(native_entry[1]) + _require( + native_receipt["source_year"] == native_receipt["income_year"] == 2024 + and native_receipt["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + observed = demographic.load_authenticated_asec_demographic_source( + parent, + member_paths={ + year: state.root / "asec" / f"pppub{year - 1999}.csv" + for year in (2022, 2023, 2024) + }, + ) + sex_receipt = observed.receipt + _require( + sex_receipt["source_identity"] == parent.source.identity.decode(), + "DEMOGRAPHIC_PARENT_IDENTITY", + ) + fields = native_entry[2].fields.document + _require(len(fields["members"]) == 1, "STATE_MEMBER_ROSTER") + states, state_pin = _load_current_state( + state.root / "asec" / "hhpub25.csv", fields["members"][0] + ) + source_households = { + row["household_id"]: row for row in native_receipt["households"] + } + households = state.frame.table("household") + selected_hh = households.loc[ + households[support_channel_column("household")].eq("asec") + ] + hrows = [] + for stacked, source_hh in selected_hh[ + ["household_id", spine_source_id_column("household")] + ].itertuples(index=False, name=None): + _require(source_hh in source_households, "HOUSEHOLD_NATIVE_JOIN") + current = source_households[source_hh] + year, key = current["native_key"] + _require(year == 2024 and key in states, "HOUSEHOLD_SOURCE_JOIN") + row = states[key] + _require( + row["H_SEQ"] == current["household_fields"]["H_SEQ"], + "HOUSEHOLD_LITERAL_KEY", + ) + hrows.append( + ( + int(stacked), + int(source_hh), + key, + row["GESTFIPS"], + row["state_code"], + row["status"], + ) + ) + htable = pd.DataFrame( + hrows, + columns=( + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + ), + ).set_index("household_id", drop=False) + htable["state_fips"] = pd.array(htable.state_fips, dtype="Int64") + htable["state_known"] = htable.state_fips.notna() + full_ids, years = observed.array("person_id"), observed.array("income_year") + _require( + len(set(zip(years, full_ids, strict=True))) == len(full_ids), "SEX_SOURCE_AXIS" + ) + lookup = pd.MultiIndex.from_arrays((years, full_ids)) + person = state.frame.person + selected = person.loc[person[support_channel_column("person")].eq("asec")] + original_ids = selected[spine_source_id_column("person")].to_numpy(dtype=np.int64) + positions = lookup.get_indexer( + pd.MultiIndex.from_arrays( + (np.full(len(selected), 2024, dtype=np.int64), original_ids) + ) + ) + _require( + (positions >= 0).all() and len(set(positions)) == len(positions), + "SEX_NATIVE_JOIN", + ) + ptable = pd.DataFrame( + { + "person_id": selected.person_id.to_numpy(dtype=np.int64), + "native_person_id": original_ids, + }, + index=pd.Index(selected.person_id.to_numpy(), name="person_id"), + ) + for name in ( + "asec_A_SEX", + "asec_AXSEX", + "asec_sex_binding_state", + "asec_sex_allocation_state", + ): + ptable[name] = observed.array(name)[positions] + binding = ptable.asec_sex_binding_state.to_numpy() + _require(np.isin(binding, (0, 1, 2)).all(), "SEX_BINDING_DOMAIN") + ptable["is_female"] = pd.array( + [False if code == 1 else True if code == 2 else None for code in binding], + dtype="boolean", + ) + ptable["sex_known"] = binding != 0 + ptable["sex_universe"] = "All Persons" + ptable["sex_origin"] = np.where( + ptable.asec_sex_allocation_state.eq(2), + "census_allocated", + np.where( + ptable.asec_sex_allocation_state.eq(1), + "source_no_change", + "unresolved_allocation", + ), + ) + receipt = preparation_owner._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": preparation_owner._sha(entry[1]), + "native_population_sha256": preparation_owner._sha(native_entry[1]), + "demographic_source_content_sha256": observed.content_sha256, + "demographic_source_receipt": sex_receipt, + "state_source": state_pin, + "state_contract": STATE_CONTRACT, + "selected_person_ids": ptable.person_id.tolist(), + "selected_household_ids": htable.household_id.tolist(), + "sex_known_persons": int(ptable.sex_known.sum()), + "sex_unknown_persons": int((~ptable.sex_known).sum()), + "state_known_households": int(htable.state_known.sum()), + "state_unknown_households": int((~htable.state_known).sum()), + "sex_observation_year": 2025, + "state_observation_year": 2025, + "income_year": 2024, + "state_is_income_year_residence_claim": False, + "selected_household_state_tokens": htable.loc[ + :, + [ + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_status", + ], + ].values.tolist(), + "person_projection_sha256": preparation_owner._sha( + ptable.reset_index(drop=True).to_json(orient="table").encode() + ), + "household_projection_sha256": preparation_owner._sha( + htable.reset_index(drop=True).to_json(orient="table").encode() + ), + "source_admission_issued": False, + "release_eligible": False, + } + ) + observed.validate() + preparation_owner._pure_final(state) + _require( + preparation_owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and preparation_owner.asec_native._ISSUED.get(id(native)) is native_entry + and native.payload == native_entry[1] + and native_entry[2].parent is parent, + "FINAL_ISSUANCE", + ) + return CurrentAsecDemographicValues(ptable, htable, receipt) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py new file mode 100644 index 000000000..2e427fbe2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -0,0 +1,381 @@ +"""Source-qualified current ASEC to ACS financial predictor preparation. + +This is a survey-multispine operation before PUF enrichment. The live retained +preparation authenticates source observations; serialized evidence never issues +source authority. Financial ACS leaves are modeled from three current ASEC +money totals, with the maintained split assumptions applied after the draws. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import acs_income_universe as universe +from . import cps_carried_current as leaves +from . import graph_puf_diagnostic_consumer as host +from . import support_provenance as provenance + +source = host.survey_source +FEATURES = ( + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", +) +TARGETS = ("survey_current_INT_VAL", "survey_current_DIV_VAL", "survey_current_CAP_VAL") +MONEY_FIELDS = leaves.CPS_CURRENT_PREDICTOR_MONEY_FIELDS +OUTPUTS = leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES +PROTOCOL = "microcosm.us.current-survey-predictor-completion.v1" +PHASE = "survey_multispine_current_financial_completion" +SEED = 578 + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_PREDICTOR_" + reason) + + +def _numeric(value, *, nullable=False): + series = pd.Series(value, copy=False) + require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "PHYSICAL_NUMERIC_TYPE", + ) + if pd.api.types.is_integer_dtype(series.dtype): + require(bool(series.dropna().abs().le(2**53).all()), "FLOAT64_INTEGER_RANGE") + result = series.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + require(bool(np.isfinite(result[~np.isnan(result)]).all()), "NONFINITE") + require(nullable or not np.isnan(result).any(), "UNKNOWN") + return result + + +def _ids(values): + require(values.dtype == np.dtype("int64"), "ID_DTYPE") + result = values.to_numpy(copy=True) + # Assembly preserves nonnegative source IDs, including native ACS ID zero. + require(bool((result >= 0).all()) and len(set(result)) == len(result), "ID_AXIS") + return result + + +def _acs_earnings(frame): + """Verify both native adjustment identities before the named NIU operator.""" + person = frame.person + mask = person[provenance.support_channel_column("person")].eq("acs") + selected = person.loc[mask] + require(len(selected) > 0, "ACS_EMPTY") + require(selected.source_year.astype(str).eq("2024").all(), "ACS_PERIOD") + age = _numeric(selected.age) + raw_age = _numeric(selected.AGEP) + require( + np.array_equal(age, raw_age) + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "ACS_AGE_IDENTITY", + ) + adjustment = _numeric(selected.ADJINC, nullable=True) + require(np.isfinite(adjustment).all() and (adjustment > 0).all(), "ACS_ADJINC") + for output, raw in universe.ACS_PUMS_EARNINGS_SOURCE_COLUMNS.items(): + require(raw in selected and output in selected, "ACS_EARNINGS_COLUMNS") + original = _numeric(selected[raw], nullable=True) + carried = _numeric(selected[output], nullable=True) + observed = ~np.isnan(original) + require(not (observed & (age < 15)).any(), "ACS_OUTSIDE_UNIVERSE_OBSERVED") + require( + np.array_equal(observed, ~np.isnan(carried)), "ACS_EARNINGS_MISSINGNESS" + ) + if raw == "WAGP": + require((original[observed] >= 0).all(), "ACS_WAGE_DOMAIN") + expected = original * (adjustment / 1_000_000.0) + require( + np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_EARNINGS_ADJUSTMENT", + ) + require(not ((age >= 15) & ~observed).any(), "ACS_ELIGIBLE_EARNINGS_UNKNOWN") + applied = universe.apply_acs_pums_earnings_universe_zeros( + frame, + person_scope=mask, + boundary="current survey financial predictor preparation", + ) + return applied.frame, applied.receipt + + +@dataclass(frozen=True) +class QualifiedSurveyPredictors: + """Computed values; callers requalify live owners at materialization/replay.""" + + projection: bytes + matrix: bytes + source_frame: object + donor_frame: object + donor_columns: pd.DataFrame + native_money: pd.DataFrame + origins: pd.DataFrame + evidence: dict + + +def qualify_current_survey_predictors( + preparation, allocated_population, clone_population +): + """Read the actual full-parent money owner once, then seal all retained state. + + ASEC donors retain original household design weights, before allocation or + cloning. No legacy raw nominal dollar column can stand in for current money. + The source periods are distinct: ASEC 2025 interview/2024 annual income, + restated to 2024 price basis; ACS 2024 rolling prior-12-month amounts adjusted + by ADJINC. This model uses that explicitly declared temporal harmonization; + it does not claim the observation windows are equivalent. + """ + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + view = source.CheckedSurveyPopulationView( + entry[1], state.context, state.frame, state.plan, json.loads(entry[1]) + ) + _, allocation = host.survey_budget._initial( + view, allocated_population, clone_population + ) + seals = tuple( + host.survey_budget._population_identity(p) + for p in (allocated_population, clone_population) + ) + native_entry = source.asec_native._ISSUED.get(id(state.native[1])) + require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "CURRENT_MONEY_PERIOD", + ) + corrected, universe_receipt = _acs_earnings(state.frame) + people = corrected.person + pids = _ids(people.person_id) + channels = people[provenance.support_channel_column("person")].astype(str) + require(set(channels) == {"asec", "acs"}, "SOURCE_CHANNELS") + asec = channels.eq("asec").to_numpy() + acs = ~asec + native_ids = people[provenance.spine_source_id_column("person")].to_numpy( + dtype=np.int64 + ) + require( + set(native_ids[asec]) == set(state.source_frames[1].person.person_id), + "SELECTED_ASEC_ROSTER", + ) + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + require(len(positions) == len(scope.person_ids), "PARENT_ROSTER") + require(all(int(pid) in positions for pid in native_ids[asec]), "PARENT_MEMBER") + take = np.array([positions[int(pid)] for pid in native_ids[asec]], dtype=np.int64) + require(all(scope.person_years[int(i)] == 2024 for i in take), "CURRENT_COHORT") + index = pd.Index(pids, name="person_id") + money = pd.DataFrame(np.nan, index=index, columns=MONEY_FIELDS, dtype=np.float64) + field_evidence = {} + for name in MONEY_FIELDS: + field = ready.field(name) + domain = next(d for d in parent.spec.fields if d.name == name) + require(domain.entity == "person", "MONEY_ENTITY") + amounts = field.amounts[take] + valid = field.validity[take] + require( + (valid == 1).all() and np.isfinite(amounts).all(), "CURRENT_MONEY_UNKNOWN" + ) + money.loc[pids[asec], name] = amounts + field_evidence[name] = { + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "amount_sha256": codec.sha(amounts.astype("= 1) & (size == np.floor(size))).all()), + "PUF_PROFILE_RETURN_SIZE_DOMAIN", + ) + return status, size + + def _ids(values, label): series = pd.Series(values, copy=False) _require(series.dtype == np.dtype("int64"), f"PUF_ID_DTYPE:{label}") @@ -101,6 +200,7 @@ def canonical_full_puf_donor( person_known, tax_unit_known, person_targets_at_tax_unit=(), + profile=FULL65, ): """Reduce canonical columns after upstream interpretation, with no imputation. @@ -109,30 +209,46 @@ def canonical_full_puf_donor( in ``person_targets_at_tax_unit`` and supplied on the tax-unit table. A destination's eventual grain never changes the declared donor grain. Knownness covers every consumed nonstructural field, including true zeros. - Raw-to-canonical judgments (including mortgage decomposition) belong upstream. + Raw-to-canonical judgments belong upstream. PUF59 does not consume or + derive the six detailed mortgage fields reserved for the SCF producer. When all person destinations are return totals, ``person=None`` avoids - inventing donor persons. Supply the separately interpreted canonical - ``tax_unit_person_count`` on the return table in that case. + inventing donor persons. FULL65 retains its canonical return-table + ``tax_unit_person_count`` input. PUF59 instead requires the source-declared + ``puf_person_incidence_capacity`` and its knownness (1 for return-level + 0/1 QBI incidence). Person-table PUF59 donors use membership as capacity; + neither case labels the bound as an observed physical return person count. """ + profile = require_puf_output_profile(profile) _require( type(person_targets_at_tax_unit) is tuple and len(set(person_targets_at_tax_unit)) == len(person_targets_at_tax_unit) - and set(person_targets_at_tax_unit) <= set(PERSON_OUTPUTS), + and set(person_targets_at_tax_unit) <= set(profile.person_outputs), "PUF_DONOR_GRAIN_DECLARATION", ) - pcols = tuple(c for c in PERSON_OUTPUTS if c not in person_targets_at_tax_unit) + pcols = tuple( + c for c in profile.person_outputs if c not in person_targets_at_tax_unit + ) return_only = person is None _require( not return_only - or (person_targets_at_tax_unit == PERSON_OUTPUTS and person_known is None), + or ( + person_targets_at_tax_unit == profile.person_outputs + and person_known is None + ), "PUF_RETURN_ONLY_GRAIN", ) + return_capacity_column = ( + profile.donor_auxiliary_columns[0] + if profile is PUF59 + else "tax_unit_person_count" + ) tcols = ( "weight", "filing_status_code", - *(("tax_unit_person_count",) if return_only else ()), + *((return_capacity_column,) if return_only else ()), *person_targets_at_tax_unit, - *TAX_UNIT_OUTPUTS, + *profile.tax_unit_outputs, + *profile.source_predictors, ) tables = [(tax_unit, ("tax_unit_id", *tcols), "tax_unit")] if not return_only: @@ -176,19 +292,25 @@ def canonical_full_puf_donor( grouped = normalized.groupby(links, sort=False).sum(min_count=1).reindex(tids) count = pd.Series(links).value_counts(sort=False).reindex(tids).to_numpy() else: - count = _numeric(tax_unit.tax_unit_person_count, label="tax_unit_person_count") + count = _numeric(tax_unit[return_capacity_column], label=return_capacity_column) _require( bool(((count >= 1) & (count == np.floor(count))).all()), - "PUF_PERSON_COUNT_DOMAIN", + "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN" + if profile is PUF59 + else "PUF_PERSON_COUNT_DOMAIN", ) donor = pd.DataFrame(index=tax_unit.index) - for column in (*person_targets_at_tax_unit, *TAX_UNIT_OUTPUTS): + for column in (*person_targets_at_tax_unit, *profile.tax_unit_outputs): donor[column] = _numeric(tax_unit[column], label=f"tax_unit.{column}") for column in pcols: donor[column] = _numeric(grouped[column], label=f"reduced.{column}") status = _numeric(tax_unit.filing_status_code, label="filing_status_code") _require(bool(np.isin(status, [1, 2, 3, 4, 5]).all()), "PUF_FILING_STATUS_DOMAIN") - for column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): values = donor[column].to_numpy() _require( bool( @@ -196,7 +318,11 @@ def canonical_full_puf_donor( ), f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", ) - for column in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS: + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): values = donor[column].to_numpy() _require( bool( @@ -211,19 +337,35 @@ def canonical_full_puf_donor( ), f"PUF_YEAR_DOMAIN:{column}", ) + # PUF59 matches the survey's total Schedule C measurement. The QBI owner + # partitions that total between ordinary and SSTB destinations, so neither + # component alone represents the predictor. FULL65 keeps its historical + # ordinary-component alias for compatibility with accepted prior evidence. + schedule_c = donor.self_employment_income_before_lsr.to_numpy() + if profile is PUF59: + schedule_c = ( + schedule_c + donor.sstb_self_employment_income_before_lsr.to_numpy() + ) # These are canonical arithmetic aliases, after complete cell validation. feature_values = ( status, count.astype(np.float64), donor.employment_income_before_lsr.to_numpy(), - donor.self_employment_income_before_lsr.to_numpy(), + schedule_c, donor.taxable_interest_income.to_numpy(), donor.qualified_dividend_income.to_numpy() + donor.non_qualified_dividend_income.to_numpy(), donor.short_term_capital_gains.to_numpy(), donor.long_term_capital_gains_before_response.to_numpy(), ) - for name, values in zip(PREDICTORS, feature_values, strict=True): + # PUF59 first two features are already source-measured. Never label generic + # status/membership arithmetic as the PUF2015 disclosure measurement. + if profile.source_predictors: + feature_values = ( + *_profile_source_values(tax_unit, profile=profile), + *feature_values[2:], + ) + for name, values in zip(profile.predictors, feature_values, strict=True): donor[name] = _numeric(values, label=name) donor["weight"] = _numeric(tax_unit.weight, label="weight") weights = donor.weight.to_numpy() @@ -231,10 +373,21 @@ def canonical_full_puf_donor( bool((weights >= 0).all()) and np.isfinite(weights.sum()) and weights.sum() > 0, "PUF_DONOR_WEIGHT_DOMAIN", ) - return donor.loc[:, [*PREDICTORS, *TARGETS, "weight"]].copy() + for column in profile.donor_auxiliary_columns: + donor[column] = count.astype(np.float64) + return donor.loc[ + :, + [ + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ], + ].copy() -def _recipient_matrix(frame, *, predictor_known): +def _recipient_matrix(frame, *, predictor_known, profile=FULL65): + profile = require_puf_output_profile(profile) tax_unit, person = frame.table("tax_unit"), frame.table("person") mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") ids = _ids(tax_unit.loc[mask, "tax_unit_id"], "recipient") @@ -245,15 +398,20 @@ def _recipient_matrix(frame, *, predictor_known): and predictor_known.index.equals(selected_index), "PUF_RECIPIENT_KNOWNNESS_AXIS", ) - _known(predictor_known, predictor_known, PREDICTORS, "recipient_predictor") + _known(predictor_known, predictor_known, profile.predictors, "recipient_predictor") person_mask = person.person_tax_unit_id.isin(ids) # The maintained resolver below owns ACS source applicability and its exact # age-15 universe zeros. Screen physical numeric types first, since pandas # numeric conversion otherwise accepts strings, booleans or datetime values. - for name in PREDICTORS: + for name in profile.predictors: plan = support._strict_predictor_source_plan( name, tax_unit=tax_unit, person=person ) + if name in profile.source_predictors: + _require( + plan.entity == "tax_unit" and plan.columns == (name,), + "PUF_PROFILE_PREDICTOR_SOURCE:" + name, + ) if plan.source_column == "filing_status_code" or plan.entity == "derived": continue table = person if plan.entity == "person" else tax_unit @@ -266,11 +424,12 @@ def _recipient_matrix(frame, *, predictor_known): nullable=True, ) features, universe = support._strict_recipient_predictor_surface( - frame, mask, PREDICTORS, person_outputs=PERSON_OUTPUTS + frame, mask, profile.predictors, person_outputs=profile.person_outputs ) - selected = features.loc[mask, list(PREDICTORS)].copy() + selected = features.loc[mask, list(profile.predictors)].copy() selected.index = selected_index selected = selected.astype("float64") + _profile_source_values(selected, profile=profile) return ( model_input.encode_recipient_matrix( selected, entity="tax_unit", entity_ids=ids @@ -281,34 +440,61 @@ def _recipient_matrix(frame, *, predictor_known): @dataclass(frozen=True) class FullPufInputs: + """Selected model surfaces, excluding nonmodel donor validation columns.""" + donor: pd.DataFrame donor_frame: Frame matrix: bytes recipient_universe: Mapping[str, object] + profile: PufOutputProfile = FULL65 -def prepare_full_puf_inputs(frame, donor, *, predictor_known): - """Prepare the full input surfaces; absence never falls back to legacy zeros.""" +def prepare_full_puf_inputs(frame, donor, *, predictor_known, profile=FULL65): + """Prepare selected surfaces without imputing absent source values. + + PUF59's canonical donor carries ``puf_person_incidence_capacity`` solely + for Boolean incidence validation. Return-only capacity is source declared; + person-table capacity comes from membership. The disclosure-capped return- + size predictor cannot serve as that bound. Only model predictors, selected + targets and weight reach the maintained donor Frame machinery. + """ + profile = require_puf_output_profile(profile) _require( isinstance(donor, pd.DataFrame) - and tuple(donor.columns) == (*PREDICTORS, *TARGETS, "weight"), + and tuple(donor.columns) + == ( + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ), "PUF_FULL_DONOR_ROSTER", ) for column in donor: _numeric(donor[column], label=f"donor.{column}") - counts = donor[PREDICTORS[1]].to_numpy() + _profile_source_values(donor, profile=profile) + model_counts = donor[profile.predictors[1]].to_numpy() + counts = ( + donor[profile.donor_auxiliary_columns[0]].to_numpy() + if profile.donor_auxiliary_columns + else model_counts + ) weights = donor["weight"].to_numpy() _require( len(donor) > 0 and donor.index.is_unique and bool(((counts >= 1) & (counts == np.floor(counts))).all()) - and bool(np.isin(donor[PREDICTORS[0]], [1, 2, 3, 4, 5]).all()) + and bool(np.isin(donor[profile.predictors[0]], [1, 2, 3, 4, 5]).all()) and bool((weights >= 0).all()) and np.isfinite(weights.sum()) and weights.sum() > 0, "PUF_DONOR_MODEL_DOMAIN", ) - for column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): values = donor[column].to_numpy() _require( bool( @@ -318,7 +504,11 @@ def prepare_full_puf_inputs(frame, donor, *, predictor_known): ), f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", ) - for column in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS: + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): values = donor[column].to_numpy() _require( bool( @@ -333,23 +523,27 @@ def prepare_full_puf_inputs(frame, donor, *, predictor_known): ), f"PUF_YEAR_DOMAIN:{column}", ) - matrix, universe = _recipient_matrix(frame, predictor_known=predictor_known) + model_donor = donor.loc[:, [*profile.predictors, *profile.targets, "weight"]].copy() + matrix, universe = _recipient_matrix( + frame, predictor_known=predictor_known, profile=profile + ) # Every input is now physically numeric, finite, and known. The maintained # helper constructs the actual weighted donor Frame and repeats its strict # source checks; its historical fillna operation has no missing values to fill. inputs = support.prepare_us_puf_tax_detail_chain_inputs( frame, - donor, - predictors=PREDICTORS, - person_outputs=PERSON_OUTPUTS, - tax_unit_outputs=TAX_UNIT_OUTPUTS, + model_donor, + predictors=profile.predictors, + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, require_complete_recipient_predictors=True, ) _require( - inputs.donor.equals(donor) and inputs.recipient_predictor_universe == universe, + inputs.donor.equals(model_donor) + and inputs.recipient_predictor_universe == universe, "PUF_PREPARATION_CHANGED_CANONICAL_INPUT", ) - return FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe) + return FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe, profile) def full_puf_train_apply_nodes( @@ -361,18 +555,20 @@ def full_puf_train_apply_nodes( n_estimators, zero_atol, prefix="puf_full", + profile=FULL65, ): - """Declare all 65 real fits/draws; each draw reads its raw predecessors.""" + """Declare every selected real fit/draw with ordered raw predecessors.""" + profile = require_puf_output_profile(profile) fits = legacy_qrf_train_nodes( prefix + ".fit", population=donor_population, entity="tax_unit", - predictors=PREDICTORS, - targets=TARGETS, + predictors=profile.predictors, + targets=profile.targets, seed=seed, n_estimators=n_estimators, zero_atol=zero_atol, - phase=PHASE, + phase=profile.phase, ) applies = legacy_qrf_apply_matrix_nodes( prefix + ".apply", @@ -380,15 +576,28 @@ def full_puf_train_apply_nodes( fit_nodes=fits, matrix_producer=matrix_producer, seed=seed, - phase=PHASE, + phase=profile.phase, ) return fits, applies def decode_full_puf_draws( - *, matrix, matrix_producer_key, raw_draws, apply_state, training_state, seed + *, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + seed, + profile=FULL65, ): - """Check actual typed-edge payloads against the complete model/raw history.""" + """Check typed payloads against the selected complete model/raw history. + + The exact ordered target roster is intrinsic to fitted model and application + states. Even the common first target has a different model identity across + profiles; truncated FULL65 artifacts cannot stand in for PUF59 artifacts. + """ + profile = require_puf_output_profile(profile) _require(codec._hash(matrix_producer_key), "PUF_MATRIX_PRODUCER_KEY") prepared = model_input.decode_recipient_matrix(matrix) packet = decode_matrix_apply_state(apply_state) @@ -400,9 +609,11 @@ def decode_full_puf_draws( applied_values = chain.to_dict() _require( prepared.entity == chain.entity == "tax_unit" - and tuple(prepared.features.columns) == tuple(chain.predictors) == PREDICTORS - and tuple(chain.targets) == tuple(chain.completed_targets) == TARGETS - and tuple(fitted_values["completed_targets"]) == TARGETS + and tuple(prepared.features.columns) + == tuple(chain.predictors) + == profile.predictors + and tuple(chain.targets) == tuple(chain.completed_targets) == profile.targets + and tuple(fitted_values["completed_targets"]) == profile.targets and all( fitted_values[key] == applied_values[key] for key in ( @@ -426,13 +637,14 @@ def decode_full_puf_draws( "PUF_FULL_MATRIX_BINDING", ) _require( - isinstance(raw_draws, Mapping) and tuple(raw_draws) == TARGETS, "PUF_RAW_ROSTER" + isinstance(raw_draws, Mapping) and tuple(raw_draws) == profile.targets, + "PUF_RAW_ROSTER", ) _require( application["raw_targets"] == [ {"target": target, "sha256": codec.sha(raw_draws[target])} - for target in TARGETS + for target in profile.targets ], "PUF_RAW_HISTORY", ) @@ -441,7 +653,7 @@ def decode_full_puf_draws( target: codec.read_raw_target( raw_draws[target], target=target, index=prepared.features.index ) - for target in TARGETS + for target in profile.targets }, index=prepared.features.index, ) @@ -459,6 +671,7 @@ def finalize_full_puf( training_state, last_model, seed, + profile=FULL65, ): """Use actual finalizer judgments only after the complete raw chain validates. @@ -467,7 +680,10 @@ def finalize_full_puf( PUF masks, and run the existing full-population replay verifier afterwards. This function itself never creates a live Population/source qualification. """ - inputs = prepare_full_puf_inputs(frame, donor, predictor_known=predictor_known) + profile = require_puf_output_profile(profile) + inputs = prepare_full_puf_inputs( + frame, donor, predictor_known=predictor_known, profile=profile + ) _require(matrix == inputs.matrix, "PUF_RECIPIENT_MATRIX_CHANGED") raw = decode_full_puf_draws( matrix=matrix, @@ -476,6 +692,7 @@ def finalize_full_puf( apply_state=apply_state, training_state=training_state, seed=seed, + profile=profile, ) # The final fitted target consumed every preceding donor outcome. Binding # its actual trusted producer bytes therefore checks the entire donor value @@ -486,18 +703,20 @@ def finalize_full_puf( last_model, expected_sha256=training["models"][-1]["sha256"] ) _require( - last.target == TARGETS[-1] + last.target == profile.targets[-1] and last.next_training_state == fitted_state and last.training_id == training["models"][-1]["training_id"], "PUF_FINAL_MODEL_BINDING", ) resolved = qrf._resolve_qrf_fit_input( - inputs.donor_frame, list(PREDICTORS), list(TARGETS), "design" + inputs.donor_frame, list(profile.predictors), list(profile.targets), "design" ) qrf.RegimeGatedQRF._validate_chain_donor(last.training_state._chain(), resolved) _require( last.donor_sha256 - == qrf_target._consumed_values_sha256(resolved.table, (*PREDICTORS, *TARGETS)), + == qrf_target._consumed_values_sha256( + resolved.table, (*profile.predictors, *profile.targets) + ), "PUF_DONOR_CONSUMED_BYTES", ) mask = support.puf_tax_detail_clone_mask(frame.table("tax_unit"), entity="tax_unit") @@ -509,19 +728,25 @@ def finalize_full_puf( frame, inputs.donor, raw.copy(deep=True), - person_outputs=PERSON_OUTPUTS, - tax_unit_outputs=TAX_UNIT_OUTPUTS, + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, tail_bound_diagnostics=caps, absent_cells=support.PUF_ABSENT_CELLS_PRESERVE_NULLS, ) return result, { - "scope": PHASE, - "target_order": list(TARGETS), + "scope": profile.phase, + "output_profile": profile.value, + "predictor_order": list(profile.predictors), + "donor_auxiliary_columns": list(profile.donor_auxiliary_columns), + "person_target_count": len(profile.person_outputs), + "tax_unit_target_count": len(profile.tax_unit_outputs), + "target_count": len(profile.targets), + "target_order": list(profile.targets), "recipient_universe": dict(inputs.recipient_universe), "tail_bounds": caps, "matrix_sha256": codec.sha(matrix), "raw_target_sha256": { - target: codec.sha(raw_draws[target]) for target in TARGETS + target: codec.sha(raw_draws[target]) for target in profile.targets }, "source_admission_issued": False, "release_eligible": False, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py new file mode 100644 index 000000000..82e66f8fc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py @@ -0,0 +1,514 @@ +"""Graph training, draw and attachment of current survey financial predictors. + +The ASEC fitting branch leaves the source CREATE before importance allocation; +its household-derived person weights remain DESIGN. No PUF clone is a donor. +All fitted totals, prior-target conditioning and final source-origin joins are +visible typed graph dependencies. Cold and replay results require the same +materialized verifier, independently of cache or node receipt labels. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_predictors as values +from . import survey_population_replay as replay + +host = values.host +require = values.require +PROJECTION_NODE = "survey_predictors.source_projection" +DONOR_NODE = "survey_predictors.asec_design_donor" +DONOR_COLUMNS_NODE = "survey_predictors.asec_current_columns" +FIT_PREFIX = "survey_predictors.fit" +APPLY_PREFIX = "survey_predictors.apply" +ATTACH_NODE = "survey_predictors.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_predictor_projection", 1) + + +def _inputs(frame): + result = [] + for entity in US_SCHEMA.entities: + structural = {US_SCHEMA.entity_id_column(entity)} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + result.append( + Slice(entity, tuple(c for c in frame.table(entity) if c not in structural)) + ) + return tuple(result) + + +def _projection_edges(): + return ( + ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE), + ArtifactInput( + "matrix", PROJECTION_NODE, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + + +def _attach_edges(): + result = list(_projection_edges()) + for i in range(len(values.TARGETS)): + result.extend( + ( + ArtifactInput( + f"raw_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"state_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + return tuple(result) + + +def _params(qualified, host_pins, n_estimators): + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + require( + type(host_pins) is dict + and set(host_pins) == {e.name for e in host.current_survey_host_edges()}, + "HOST_PINS", + ) + for pin in host_pins.values(): + require( + type(pin) is dict + and set(pin) == {"producer_key", "artifact_key", "payload_sha256"} + and all(codec._hash(v) for v in pin.values()), + "HOST_PIN_DIGESTS", + ) + return { + "protocol": values.PROTOCOL, + "source_projection_sha256": codec.sha(qualified.projection), + "host_edges": codec.encode_json(host_pins).decode(), + "seed": values.SEED, + "n_estimators": n_estimators, + } + + +def current_survey_predictor_nodes( + qualified, clone_frame, *, host_pins, n_estimators=100 +): + require( + type(qualified) is values.QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE" + ) + params = _params(qualified, host_pins, n_estimators) + projection = Node( + PROJECTION_NODE, + CurrentSurveyPredictorProjectionKernel.ref, + # This ordinary node must live outside CREATE: its typed allocation + # evidence is downstream of that version's structural allocation node. + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + artifact_inputs=host.current_survey_host_edges(), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + donor = Node( + DONOR_NODE, + CurrentSurveyPredictorDonorFilterKernel.ref, + base=host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + inputs=_inputs(qualified.source_frame), + mass="free", + params=params, + # Branch from the authenticated CREATE before its weight transition. + # The later projection depends on this FILTER, never conversely. + artifact_inputs=(host.current_survey_host_edges()[0],), + description="Select native ASEC whole households before allocation; retain original design weights for survey financial fitting.", + ) + columns = Node( + DONOR_COLUMNS_NODE, + CurrentSurveyPredictorDonorColumnsKernel.ref, + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64") for c in (*values.FEATURES, *values.TARGETS) + ), + artifact_inputs=_projection_edges(), + ) + fit = legacy_qrf_train_nodes( + FIT_PREFIX, + population=DONOR_NODE, + entity="person", + predictors=values.FEATURES, + targets=values.TARGETS, + seed=values.SEED, + phase=values.PHASE, + n_estimators=n_estimators, + zero_atol=0, + ) + apply = legacy_qrf_apply_matrix_nodes( + APPLY_PREFIX, + population=host.survey_clone.COMBINED_CLONE_NODE, + fit_nodes=fit, + matrix_producer=PROJECTION_NODE, + seed=values.SEED, + phase=values.PHASE, + ) + attach = Node( + ATTACH_NODE, + CurrentSurveyPredictorAttachKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=_inputs(clone_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64", rewrite=(c in clone_frame.person)) + for c in values.OUTPUTS + ), + artifact_inputs=_attach_edges(), + ) + return (projection, donor, columns, *fit, *apply, attach) + + +class _Kernel(host._CurrentSurveyKernel): + def __init__( + self, + preparation, + allocated_population, + clone_population, + *, + host_pins, + n_estimators=100, + ): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + self.host_pins = codec.decode_json(codec.encode_json(host_pins)) + self.n_estimators = n_estimators + + def implementation_hash(self): + return codec.sha( + codec.encode_json( + { + "host": super().implementation_hash(), + "predictors": source_hash( + sys.modules[__name__], + values, + values.leaves, + values.universe, + population_ops, + replay, + qrf, + model_input, + sys.modules[decode_matrix_apply_state.__module__], + sys.modules[legacy_qrf_train_nodes.__module__], + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _qualified(self, context): + require(not context.sources, "UNDECLARED_SOURCE") + result = values.qualify_current_survey_predictors( + self.preparation, self.allocated_population, self.clone_population + ) + nodes = current_survey_predictor_nodes( + result, + self.clone_population.frame, + host_pins=self.host_pins, + n_estimators=self.n_estimators, + ) + expected = {n.id: n for n in nodes} + require(context.node == expected.get(context.node.id), "NODE_DECLARATION") + require( + set(context.artifacts) == {e.name for e in context.node.artifact_inputs}, + "ARTIFACT_ROSTER", + ) + if context.node.id == DONOR_NODE: + edge = host.current_survey_host_edges()[0] + require(edge.name == "preparation", "PREPARATION_EDGE") + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and value.payload == self.preparation.payload, + "DONOR_PREPARATION_BINDING", + ) + elif context.node.id != PROJECTION_NODE: + projection = host.shared.artifact(context, "projection", PROJECTION_TYPE) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + host.shared.siblings(context, ("projection", "matrix")) + require( + projection.payload == result.projection + and matrix.payload == result.matrix, + "SOURCE_ARTIFACT_BYTES", + ) + return result + + +class CurrentSurveyPredictorProjectionKernel(_Kernel): + ref = "us.survey_predictors.source_projection@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + loaded = {} + for edge in host.current_survey_host_edges(): + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + }, + "HOST_EDGE_PIN", + ) + loaded[edge.name] = value.payload + host.shared.siblings(context, ("allocation", "frame_context")) + require( + loaded["preparation"] == self.preparation.payload + and codec.sha(loaded["allocation"]) + == qualified.evidence["allocation_sha256"] + and codec.sha(loaded["frame_context"]) + == codec.decode_json(loaded["allocation"])["output_context_sha256"], + "HOST_EDGE_BYTES", + ) + return KernelResult( + artifacts={"projection": qualified.projection, "matrix": qualified.matrix}, + receipt=qualified.evidence, + ) + + +class CurrentSurveyPredictorDonorFilterKernel(_Kernel): + ref = "us.survey_predictors.asec_design_donor@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=host._CurrentSurveyKernel.capabilities.dependencies, + ) + + def run(self, context): + self._qualified(context) + # CREATE carries raw ACS NIU blanks. The qualified feature frame has + # separately receipted materialized zeros and is not this context. + original = self.preparation._checked()[2].frame + host._current_context_frame(context, original) + person = original.person + keep = ( + person[values.provenance.support_channel_column("person")] + .eq("asec") + .to_numpy() + ) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + receipt={ + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + }, + ) + + +class CurrentSurveyPredictorDonorColumnsKernel(_Kernel): + ref = "us.survey_predictors.asec_current_columns@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + return KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in (*values.FEATURES, *values.TARGETS) + }, + receipt=qualified.evidence, + ) + + +def read_current_survey_draws(matrix, matrix_producer_key, raw_draws, apply_states): + require( + codec._hash(matrix_producer_key) + and type(raw_draws) is tuple + and type(apply_states) is tuple + and len(raw_draws) == len(apply_states) == len(values.TARGETS), + "DRAW_CHAIN_ROSTER", + ) + prepared = model_input.decode_recipient_matrix(matrix) + require( + prepared.entity == "person" + and tuple(prepared.features.columns) == values.FEATURES, + "MATRIX_FEATURE_ROSTER", + ) + result = pd.DataFrame(index=prepared.features.index) + models = [] + for i, target in enumerate(values.TARGETS): + packet = decode_matrix_apply_state(apply_states[i]) + require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "DRAW_MATRIX_BINDING", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + chain.entity == "person" + and tuple(chain.predictors) == values.FEATURES + and tuple(chain.targets) == values.TARGETS + and tuple(chain.completed_targets) == values.TARGETS[: i + 1] + and chain.recipient_index == qrf._index_identity(prepared.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] + == [ + {"target": name, "sha256": codec.sha(raw)} + for name, raw in zip( + values.TARGETS[: i + 1], raw_draws[: i + 1], strict=True + ) + ], + "DRAW_CHAIN_BINDING", + ) + current_models = application["models"] + require( + current_models[:i] == models and len(current_models) == i + 1, + "DRAW_MODEL_HISTORY", + ) + models = current_models + values_array = codec.read_raw_target( + raw_draws[i], target=target, index=prepared.features.index + ) + require(np.isfinite(values_array).all(), "DRAW_UNKNOWN") + result[target] = values_array + return result + + +class CurrentSurveyPredictorAttachKernel(_Kernel): + ref = "us.survey_predictors.attach@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, self.clone_population.frame) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + raw, states = [], [] + for i in range(len(values.TARGETS)): + r, s = f"raw_{i:03d}", f"state_{i:03d}" + host.shared.siblings(context, (r, s)) + raw.append(host.shared.artifact(context, r, codec.RAW_TARGET_TYPE).payload) + states.append( + host.shared.artifact(context, s, MATRIX_APPLY_STATE_TYPE).payload + ) + drawn = read_current_survey_draws( + matrix.payload, matrix.producer_key, tuple(raw), tuple(states) + ) + columns = values.complete_predictor_columns( + qualified, self.clone_population.frame, drawn + ) + return KernelResult( + columns=columns, + receipt={ + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + }, + ) + + +def verify_materialized_current_survey_predictors( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, + matrix_producer_key, + raw_draws, + apply_states, + host_pins, + n_estimators=100, +): + """Use actual executor-observed Population and authenticated typed artifacts. + + Call only after graph store/type/producer-key checks, both cold and replay. + Requalifies the current source owners, then compares every column, identity, + owner, metadata value, weight and mass ledger of the expected full result. + """ + qualified = values.qualify_current_survey_predictors( + preparation, allocated_population, clone_population + ) + require( + projection == qualified.projection and matrix == qualified.matrix, + "MATERIALIZED_SOURCE", + ) + drawn = read_current_survey_draws( + matrix, matrix_producer_key, raw_draws, apply_states + ) + attach = current_survey_predictor_nodes( + qualified, + clone_population.frame, + host_pins=host_pins, + n_estimators=n_estimators, + )[-1] + expected = population_ops.patch( + clone_population, + attach, + KernelResult( + columns=values.complete_predictor_columns( + qualified, clone_population.frame, drawn + ) + ), + ) + replay.same_replayed_population(expected, population) + return { + **qualified.evidence, + "all_output_cells_available": True, + "paired_draws": "source_origin_join", + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py index 1a5b370b1..567162543 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py @@ -1,4 +1,4 @@ -"""Bounded full65 attachment to an independently retained upstream Population. +"""Bounded PUF profile attachment to an independently retained Population. This is a placement owner, not source admission or a source-successor issuer. The caller must already have qualified the complete survey predictor stages and @@ -71,11 +71,42 @@ MASKS = MappingProxyType({"person": PERSON_MASK, "tax_unit": TAX_UNIT_MASK}) PLACEMENT_TYPE = ArtifactType("microcosm.us.full_puf_placement", 1) SCOPE = "full65_attachment_complete_upstream_required" -_ROSTERS = (("person", full.PERSON_OUTPUTS), ("tax_unit", full.TAX_UNIT_OUTPUTS)) -_OUTPUT_COORDINATES = frozenset((e, c) for e, cols in _ROSTERS for c in cols) require = full._require +def _rosters(profile): + profile = full.require_puf_output_profile(profile) + return (("person", profile.person_outputs), ("tax_unit", profile.tax_unit_outputs)) + + +def _scope(profile): + profile = full.require_puf_output_profile(profile) + return profile.value + "_attachment_complete_upstream_required" + + +def _profile_chain(profile, fit_nodes, apply_nodes): + profile = full.require_puf_output_profile(profile) + require( + not {"prior_year_wages", "employment_income_last_year"} & set(profile.targets) + and type(fit_nodes) is type(apply_nodes) is tuple + and len(fit_nodes) == len(apply_nodes) == len(profile.targets), + "FULL_PUF_CHAIN_ROSTER", + ) + require( + all( + type(fit) is type(apply) is Node + and fit.params.get("predictors") == profile.predictors + and fit.params.get("targets") == profile.targets + and fit.params.get("target") == apply.params.get("target") == target + and fit.params.get("phase") == apply.params.get("phase") == profile.phase + for fit, apply, target in zip( + fit_nodes, apply_nodes, profile.targets, strict=True + ) + ), + "FULL_PUF_CHAIN_PROFILE", + ) + + def _digest(parts): digest = hashlib.sha256() for part in parts: @@ -168,14 +199,15 @@ def _structural(frame, entity): ) -def _inputs(frame, *, masks=False): +def _inputs(frame, *, masks=False, profile=full.FULL65): + coordinates = frozenset((e, c) for e, cols in _rosters(profile) for c in cols) result = [] for entity in frame.entities: columns = tuple( c for c in frame.table(entity) if c not in _structural(frame, entity) - and (not masks or (entity, c) not in _OUTPUT_COORDINATES) + and (not masks or (entity, c) not in coordinates) ) # Full attachment must project every table, including its structural IDs. # The graph cannot express a Slice with zero nonstructural columns. @@ -185,9 +217,9 @@ def _inputs(frame, *, masks=False): return tuple(result) -def _outputs(frame): +def _outputs(frame, *, profile=full.FULL65): result = [] - for entity, columns in _ROSTERS: + for entity, columns in _rosters(profile): table = frame.table(entity) for column in columns: boolean = column in full.support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS @@ -232,7 +264,7 @@ def _masks(frame): @dataclass(frozen=True) class FullPufAttachmentBinding: - """Retained comparison inputs; possession confers no source authority.""" + """Retained profile and comparison inputs; no source authority is conferred.""" population: Population expected_population: Population @@ -249,8 +281,10 @@ class FullPufAttachmentBinding: expected_stamp: str donor_stamp: str known_stamp: str + profile: full.PufOutputProfile = full.FULL65 def check(self): + _profile_chain(self.profile, self.fit_nodes, self.apply_nodes) require( _population_stamp(self.population) == self.population_stamp and _population_stamp(self.expected_population) == self.expected_stamp, @@ -295,8 +329,13 @@ def retain_full_puf_attachment( fit_nodes, apply_nodes, prefix="puf_full_attachment", + profile=full.FULL65, ): - """Pin already prepared upstream values and an actual matrix producer edge. + """Pin prepared upstream values, selected outputs and a real matrix edge. + + ``profile`` defaults to the complete legacy FULL65 roster. PUF59 owns only + its selected columns; any excluded incumbent remains upstream-owned and + participates only in the complete Population preservation checks. ``population`` is executor-observed; ``expected_population`` is a separately retained upstream comparison value, not derived from the attachment result. @@ -305,6 +344,7 @@ def retain_full_puf_attachment( a version with inherited incumbents. This function admits neither a donor nor a survey source and does not construct an upstream Population from a Frame. """ + profile = full.require_puf_output_profile(profile) require( type(population) is type(expected_population) is Population and population is not expected_population @@ -326,14 +366,7 @@ def retain_full_puf_attachment( and dict(input_owners) == dict(population.owners), "FULL_PUF_UPSTREAM_GRAPH_BINDING", ) - require( - len(full.PERSON_OUTPUTS) == 56 - and len(full.TAX_UNIT_OUTPUTS) == 9 - and not {"prior_year_wages", "employment_income_last_year"} & set(full.TARGETS) - and type(fit_nodes) is type(apply_nodes) is tuple - and len(fit_nodes) == len(apply_nodes) == 65, - "FULL_PUF_CHAIN_ROSTER", - ) + _profile_chain(profile, fit_nodes, apply_nodes) first = fit_nodes[0] matrix_edges = tuple( e for e in apply_nodes[0].artifact_inputs if e.name == "matrix" @@ -352,6 +385,7 @@ def retain_full_puf_attachment( n_estimators=first.params["n_estimators"], zero_atol=first.params["zero_atol"], prefix=first.id.removesuffix(".fit.000"), + profile=profile, ) require( fit_nodes == expected_fits and apply_nodes == expected_applies, @@ -364,11 +398,11 @@ def retain_full_puf_attachment( require( name not in population.frame.table(entity), "FULL_PUF_MASK_ALREADY_PRESENT" ) - _outputs(population.frame) - _inputs(population.frame) + _outputs(population.frame, profile=profile) + _inputs(population.frame, profile=profile) _masks(population.frame) prepared = full.prepare_full_puf_inputs( - population.frame, donor, predictor_known=predictor_known + population.frame, donor, predictor_known=predictor_known, profile=profile ) _value(edge, matrix) require(matrix.payload == prepared.matrix, "FULL_PUF_PREPARED_MATRIX") @@ -388,6 +422,7 @@ def retain_full_puf_attachment( _population_stamp(expected_population), _table_stamp(donor), _table_stamp(predictor_known), + profile, ) binding.check() return binding @@ -398,7 +433,8 @@ def _params(binding): # Nullable backing storage beneath nulls is canonicalized by Frame-store # v2. Graph column/frame keys bind the upstream values across sessions. return { - "scope": SCOPE, + "scope": _scope(binding.profile), + "output_profile": binding.profile.value, "donor_values": binding.donor_stamp, "predictor_knownness": binding.known_stamp, "matrix_producer_key": binding.matrix.producer_key, @@ -446,13 +482,13 @@ def full_puf_attachment_nodes(binding): FullPufMaskKernel.ref, population=binding.population.version, # Reading fields rewritten by attach would create a graph dependency cycle. - inputs=_inputs(frame, masks=True), + inputs=_inputs(frame, masks=True, profile=binding.profile), outputs=tuple(Owned(e, c, "bool") for e, c in MASKS.items()), params=_params(binding), artifact_inputs=(binding.matrix_edge,), artifact_outputs=(ArtifactOutput("placement", PLACEMENT_TYPE),), ) - inputs = list(_inputs(frame)) + inputs = list(_inputs(frame, profile=binding.profile)) for entity, mask in MASKS.items(): for i, item in enumerate(inputs): if item.entity == entity: @@ -465,7 +501,7 @@ def full_puf_attachment_nodes(binding): FullPufAttachKernel.ref, population=binding.population.version, inputs=tuple(inputs), - outputs=_outputs(frame), + outputs=_outputs(frame, profile=binding.profile), params=_params(binding), artifact_inputs=_attach_edges(binding), ) @@ -538,7 +574,7 @@ def _placement(binding): return codec.encode_json( { **_params(binding), - "target_order": list(full.TARGETS), + "target_order": list(binding.profile.targets), "selected_ids": { entity: frame.table(entity) .loc[mask, frame.schema.entity_id_column(entity)] @@ -569,7 +605,7 @@ def _checked_artifacts(binding, edges, artifacts, producer_keys=None): values["placement"].payload == _placement(binding), "FULL_PUF_PLACEMENT_BINDING", ) - last_raw = f"raw_{len(binding.apply_nodes) - 1:03d}" + last_raw = f"raw_{len(binding.profile.targets) - 1:03d}" for names in ((last_raw, "apply_state"), ("last_model", "training_state")): require( len({values[name].producer_key for name in names}) == 1, @@ -588,15 +624,16 @@ def _finalized_columns(binding, values): matrix_producer_key=values["matrix"].producer_key, raw_draws={ target: values[f"raw_{i:03d}"].payload - for i, target in enumerate(full.TARGETS) + for i, target in enumerate(binding.profile.targets) }, apply_state=values["apply_state"].payload, training_state=values["training_state"].payload, last_model=values["last_model"].payload, seed=binding.fit_nodes[0].params["seed"], + profile=binding.profile, ) masks, columns = _masks(frame), {} - for owned in _outputs(frame): + for owned in _outputs(frame, profile=binding.profile): table, mask = finalized.table(owned.entity), masks[owned.entity] selected = table.loc[mask, owned.column] require(selected.notna().all(), "FULL_PUF_FINALIZED_UNKNOWN:" + owned.column) @@ -659,7 +696,11 @@ def run(self, context): return KernelResult( columns=_mask_result(binding).columns, artifacts={"placement": _placement(binding)}, - receipt={"scope": SCOPE, "source_admission_issued": False}, + receipt={ + "scope": _scope(binding.profile), + "output_profile": binding.profile.value, + "source_admission_issued": False, + }, ) @@ -700,11 +741,16 @@ def verify_materialized_full_puf_attachment( expected = population_ops.patch(expected, attach, result) replay.same_replayed_population(expected, population) binding.check() - return {**result.receipt, "scope": SCOPE, "complete_population_compared": True} + return { + **result.receipt, + "scope": _scope(binding.profile), + "output_profile": binding.profile.value, + "complete_population_compared": True, + } def _manifest_contract(binding, compiled, manifest): - """Bind declarations, real kernel implementations and all 130 chain nodes.""" + """Bind declarations and real kernel identities for the selected complete chain.""" mask, attach = full_puf_attachment_nodes(binding) require( compiled.graph.node(binding.population_node.id) == binding.population_node, @@ -777,7 +823,7 @@ def _manifest_contract(binding, compiled, manifest): def load_full_puf_attachment_artifacts(binding, *, compiled, manifest, store): """Load real producer bytes after verifying the graph and both ends of edges. - ``compiled`` is the actual graph used in this run. Recompute full65 node keys + ``compiled`` is the actual graph used in this run. Recompute selected node keys from its exact declarations and maintained kernel identities before loading trusted model bytes. Upstream source qualification remains the caller's duty. """ diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 03c602c75..1f4a86047 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -104,10 +104,6 @@ from microcosm.build.us_runtime.retirement_distributions import ( with_us_retirement_distribution_inputs, ) -from microcosm.build.us_runtime.spine_agreement import ( - default_spine_agreement_registry, - spine_agreement_gate, -) from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.build.us_runtime.support_provenance import ( SPINE_ASSEMBLY_MANIFEST_KEY, @@ -1736,6 +1732,14 @@ def materialize_pool_deferred_transfer_inputs(frame: Frame) -> PoolStageOutput: return PoolStageOutput(result, {"inputs": receipts}) +# The default agreement registry validates the engine ABI, whose fresh +# manifest reads the pool functions above. Define that surface before importing +# the registry so either module can be the first import in a fresh process. +from microcosm.build.us_runtime.spine_agreement import ( # noqa: E402 + default_spine_agreement_registry, + spine_agreement_gate, +) + POOL_SPINE_AGREEMENT_REGISTRY = default_spine_agreement_registry( pool_transfer_target_families() ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py new file mode 100644 index 000000000..af1290c9f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py @@ -0,0 +1,307 @@ +"""Return-grain canonical PUF59 with explicit model/growth lineage. + +No tax engine, invented donor people, loan balances or loan vintages. The +observed statistical2015 source remains immutable beside modeled outputs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from . import puf_full_source as source_owner +from . import puf_qbi_model as qbi +from . import puf_target2024_growth as growth + +VERSION = "microcosm.us.puf59_canonical_return/2" +INTEREST_ASSET_SHA256 = ( + "c3356ae216487f365cb0e0a7ab1ba46843a6c52950b75eae1bfab9b0b80a735a" +) + +INTEREST_BAND_FACTS_SHA256 = ( + "3ea894c474302dd818f347f5bfde3b331bf80468b0ac928ffe76ed1e6de62cbc" +) +PREFIX_NAMES = ( + "RECID", + "weight", + "puf_person_incidence_capacity", + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", +) + + +def prefix_values_digest(arrays): + """Bind identities, source weights and source-specific features in row order.""" + digest = hashlib.sha256() + for name in PREFIX_NAMES: + digest.update(name.encode("ascii") + b"\0") + digest.update( + np.asarray( + arrays[name], dtype="= -(2**53)) & (a <= 2**53)).all()), + "PUF59_MODEL_INTEGER_PRECISION:" + name, + ) + a = a.astype(np.float64) + _require(bool(np.isfinite(a).all()), "PUF59_MODEL_NONFINITE:" + name) + return a + + +def baseline_modeled_columns(auxiliary, *, n, interest_bands, interest_asset_sha256): + """Eleven explicit baseline outputs absent as exact raw canonical leaves.""" + _require( + interest_asset_sha256 == INTEREST_ASSET_SHA256, "PUF59_INTEREST_ASSET_IDENTITY" + ) + _require(isinstance(auxiliary, Mapping), "PUF59_MODEL_AUXILIARY_MAPPING") + names = ( + "raw_adjusted_gross_income", + "raw_total_interest_deduction", + "raw_total_social_security", + "raw_realized_ira_deduction", + "raw_realized_keogh_deduction", + "raw_tuition_fees_deduction", + "raw_lifetime_learning_qualified_expenses", + "raw_miscellaneous_itemized_deductions", + "raw_partnership_nonpassive_net", + ) + _require(all(k in auxiliary for k in names), "PUF59_MODEL_MISSING_AUXILIARY") + a = {k: _money(auxiliary[k], k, n) for k in names} + for k in names: + if k not in ("raw_adjusted_gross_income", "raw_partnership_nonpassive_net"): + _require(bool((a[k] >= 0).all()), "PUF59_MODEL_NEGATIVE_SOURCE:" + k) + bands = tuple(interest_bands) + _require(len(bands) == 22, "PUF59_INTEREST_BAND_COUNT") + _require( + bands[0].lower_bound is None and bands[-1].upper_bound is None, + "PUF59_INTEREST_FULL_AGI", + ) + for left, right in zip(bands[:-1], bands[1:], strict=True): + _require(left.upper_bound == right.lower_bound, "PUF59_INTEREST_CONTIGUOUS") + total = a["raw_total_interest_deduction"] + agi = a["raw_adjusted_gross_income"] + investment = np.zeros(n) + covered = np.zeros(n, dtype=np.uint8) + band_facts = [] + fields = ( + "home_mortgage_interest_amount", + "deductible_points_amount", + "qualified_mortgage_insurance_premiums_amount", + "investment_interest_amount", + ) + for i, band in enumerate(bands): + _require(band.source_row == 11 + i, "PUF59_INTEREST_SOURCE_ROW") + lo, hi = band.lower_bound, band.upper_bound + _require( + (lo is None or type(lo) in (int, float) and np.isfinite(lo)) + and (hi is None or type(hi) in (int, float) and np.isfinite(hi)) + and (lo is None or hi is None or lo < hi), + "PUF59_INTEREST_BAND_BOUNDS", + ) + amounts = np.asarray([getattr(band, k) for k in fields], dtype=np.float64) + _require( + bool(np.isfinite(amounts).all() and (amounts >= 0).all()) + and amounts.sum() > 0, + "PUF59_INTEREST_COMPONENT_DOMAIN", + ) + _require( + type(band.total_interest_paid_amount) is int + and abs( + band.total_interest_paid_amount - sum(getattr(band, k) for k in fields) + ) + <= 1, + "PUF59_INTEREST_SOURCE_ROUNDING", + ) + mask = np.ones(n, dtype=bool) + if lo is not None: + mask &= agi >= lo + if hi is not None: + mask &= agi < hi + investment[mask] = total[mask] * (amounts[3] / amounts.sum()) + covered[mask] += 1 + band_facts.append( + { + "source_row": band.source_row, + "lower_bound": lo, + "upper_bound": hi, + "amounts_thousands_usd": amounts.tolist(), + } + ) + band_facts_sha256 = hashlib.sha256( + json.dumps(band_facts, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + _require( + band_facts_sha256 == INTEREST_BAND_FACTS_SHA256, + "PUF59_INTEREST_BAND_FACTS_IDENTITY", + ) + _require(bool((covered == 1).all()), "PUF59_INTEREST_PARTITION") + home = total - investment + investment = total - home + _require(np.array_equal(home + investment, total), "PUF59_INTEREST_CONSERVATION") + columns = { + "home_mortgage_interest": home, + "investment_interest_expense": investment, + "social_security_retirement": a["raw_total_social_security"].copy(), + "social_security_disability": np.zeros(n), + "social_security_dependents": np.zeros(n), + "social_security_survivors": np.zeros(n), + "traditional_ira_contributions_desired": a["raw_realized_ira_deduction"].copy(), + "self_employed_pension_contributions_desired": a[ + "raw_realized_keogh_deduction" + ].copy(), + "qualified_tuition_expenses": np.maximum( + a["raw_tuition_fees_deduction"], + a["raw_lifetime_learning_qualified_expenses"], + ), + "unreimbursed_business_employee_expenses": a[ + "raw_miscellaneous_itemized_deductions" + ].copy(), + "partnership_self_employment_net_earnings": a[ + "raw_partnership_nonpassive_net" + ].copy(), + } + return columns, { + "schema": "microcosm.us.puf59_baseline_models/1", + "input_money_year": 2015, + "interest_asset_sha256": interest_asset_sha256, + "interest_band_facts_sha256": band_facts_sha256, + "interest_method": "Normalize all four published component amounts. Map mortgage+points+insurance to the available home interest leaf and investment only to investment expense; complementary rounding conserves E19200 exactly.", + "ss_method": "Component-neutral carrier: total SS in retirement, other component carriers zero; requires recipient-profile SS reconciliation. Zeros are not observations of absent benefits.", + "desired_deductions": "Realized IRA/Keogh deductions proxy for desired amounts; uncensored demand is not observed.", + "tuition": "Maximum of realized tuition deduction and Lifetime Learning qualified expenses; incomplete source expense proxy.", + "employee_expense": "Total miscellaneous itemized deductions proxy; broader than employee business expenses.", + "partnership_earnings": "Nonpassive partnership net earnings proxy; not reconstructed from capped ScheduleSE amounts.", + "mortgage_structure_generated": False, + "origin": "modeled_not_observed", + } + + +@dataclass(frozen=True) +class CanonicalPuf59Result: + columns: Mapping[str, np.ndarray] + source_predictors: Mapping[str, np.ndarray] + recids: np.ndarray + design_weight: np.ndarray + person_incidence_capacity: np.ndarray + money_year: int + qbi_calibration: qbi.QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def construct_canonical_puf59( + source, + *, + interest_bands, + interest_asset_sha256, + selected_recids=None, + qbi_employment_calibration=None, + growth_scheme="family_observed", + seed=0, +): + """Construct canonical outputs from an owner-decoded ordinary source cohort. + + The caller supplies source-authenticated interest bands. Selection is owned + by the source decoder. No data source is opened or sampled in this function. + """ + observed, aux, status = source_owner.observed_and_derived_return_columns( + source, selected_recids=selected_recids + ) + n = len(status["RECID"]) + _require(n > 0, "PUF59_EMPTY_COHORT") + modeled, model_receipt = baseline_modeled_columns( + aux, + n=n, + interest_bands=interest_bands, + interest_asset_sha256=interest_asset_sha256, + ) + _require(not set(observed) & set(modeled), "PUF59_CANONICAL_COLLISION") + canonical = {**observed, **modeled} + known = {k: np.ones(n, dtype=bool) for k in canonical} + qresult = qbi.model_full_puf_qbi( + canonical, + status["RECID"], + known=known, + input_money_year=2015, + seed=seed, + employment_calibration=qbi_employment_calibration, + ) + canonical.update(qresult.columns) + _require(set(canonical) == set(growth.OUTPUTS), "PUF59_CANONICAL_COMPLETE_ROSTER") + known = {k: np.ones(n, dtype=bool) for k in canonical} + result = growth.grow_puf_2015_to_2024( + canonical, known=known, input_money_year=2015, scheme=growth_scheme + ) + weight = status["S006"].astype(np.float64) / 100 + _require( + bool((weight >= 0).all() and np.isfinite(weight).all() and weight.sum() > 0), + "PUF59_WEIGHT_DOMAIN", + ) + predictors = { + k: np.frombuffer(np.asarray(aux[k], dtype=np.int64).tobytes(), dtype=np.int64) + for k in ("puf_2015_filing_status_code", "puf_2015_capped_return_size") + } + prefix_arrays = { + "RECID": status["RECID"], + "weight": weight, + "puf_person_incidence_capacity": np.ones(n, dtype=np.int64), + **predictors, + } + receipt = { + "prefix_values_sha256": prefix_values_digest(prefix_arrays), + "schema": VERSION, + "source_definition_sha256": source.definition_sha256, + "source_sha256": dict(source.source_sha256), + "source_statistical_year": 2015, + "input_money_year": 2015, + "output_money_year": 2024, + "rows": n, + "model_assumptions": model_receipt, + "qbi": dict(qresult.receipt), + "growth": dict(result.receipt), + "return_size": "Source PUF2015 coarsened filing-status plus capped dependent count; no physical person claim.", + "person_incidence_capacity": "Constant1 for zero/one modeled return incidence; excluded from conditioning predictors.", + "weight": "Source S006 exact hundredths divided once by100; no weight growth.", + "recipient_conditioning_self_employment": "Must sum base and SSTB ScheduleC outcomes for the total source predictor.", + "source_observation_status": "33 direct/derived baseline columns; 11 explicit baseline models; 15 added QBI leaves and one replacement; all monetary transport is modeled.", + "mortgage_structure_generated": False, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + def readonly(a): + return np.frombuffer(a.tobytes(), dtype=a.dtype) + + return CanonicalPuf59Result( + result.columns, + MappingProxyType(predictors), + readonly(status["RECID"].astype(np.int64)), + readonly(weight), + readonly(np.ones(n, dtype=np.int64)), + 2024, + qresult.calibration, + MappingProxyType(receipt), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py new file mode 100644 index 000000000..d0b5566c8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py @@ -0,0 +1,237 @@ +"""Bounded deterministic private canonical59 array envelope; no source admission.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Mapping + +import numpy as np + +from .puf59_canonical import PREFIX_NAMES, VERSION, prefix_values_digest +from .puf_target2024_growth import INCIDENCE_FIELDS, OUTPUTS, RECIPE_SHA256, _digest + +MAGIC = b"MCPUF59\x02" +MAX_HEADER = 128 * 1024 +MAX_BODY = 128 * 1024 * 1024 +PREFIX = PREFIX_NAMES +NAMES = (*PREFIX, *OUTPUTS) +INTEGERS = set(PREFIX) - {"weight"} | set(INCIDENCE_FIELDS) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _receipt(receipt, expected_growth_scheme): + _require( + expected_growth_scheme in ("family_observed", "cpi_only"), + "PUF59_ARTIFACT_EXPECTED_SCHEME", + ) + _require(isinstance(receipt, Mapping), "PUF59_ARTIFACT_RECEIPT_TYPE") + r = dict(receipt) + _require( + r.get("schema") == VERSION + and type(r.get("input_money_year")) is int + and r["input_money_year"] == 2015 + and type(r.get("output_money_year")) is int + and r["output_money_year"] == 2024 + and type(r.get("rows")) is int + and 0 < r["rows"] <= MAX_BODY // (len(NAMES) * 8) + and r.get("release_eligible") is False, + "PUF59_ARTIFACT_RECEIPT_CONTRACT", + ) + gr = r.get("growth") + _require( + isinstance(gr, Mapping) and gr.get("scheme") == expected_growth_scheme, + "PUF59_ARTIFACT_GROWTH_SCHEME", + ) + _require(gr.get("recipe_sha256") == RECIPE_SHA256, "PUF59_ARTIFACT_GROWTH_RECIPE") + h = r.pop("sha256", None) + _require( + h + == hashlib.sha256( + json.dumps( + r, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest(), + "PUF59_ARTIFACT_RECEIPT_HASH", + ) + r["sha256"] = h + return r + + +def _validate(arrays, n): + _require( + type(n) is int and 0 < n <= MAX_BODY // (len(NAMES) * 8), "PUF59_ARTIFACT_ROWS" + ) + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + for name in NAMES: + a = np.asarray(arrays[name]) + kind = "iu" if name in INTEGERS else "fiu" + _require( + a.shape == (n,) and a.dtype.kind in kind, + "PUF59_ARTIFACT_COLUMN_TYPE:" + name, + ) + _require(bool(np.isfinite(a).all()), "PUF59_ARTIFACT_NONFINITE:" + name) + if a.dtype.kind in "iu": + _require( + bool(((a >= -(2**53)) & (a <= 2**53)).all()), + "PUF59_ARTIFACT_INTEGER_RANGE:" + name, + ) + ids = arrays["RECID"] + w = arrays["weight"] + _require( + bool((ids > 0).all()) + and len(np.unique(ids)) == n + and not np.isin(ids, [999996, 999997, 999998, 999999]).any(), + "PUF59_ARTIFACT_ID_DOMAIN", + ) + _require( + bool((w >= 0).all()) and np.isfinite(w.sum()) and w.sum() > 0, + "PUF59_ARTIFACT_WEIGHT", + ) + _require( + bool((arrays["puf_person_incidence_capacity"] == 1).all()), + "PUF59_ARTIFACT_CAPACITY", + ) + _require( + bool(np.isin(arrays["puf_2015_filing_status_code"], [1, 2, 3, 4]).all()), + "PUF59_ARTIFACT_STATUS", + ) + size = arrays["puf_2015_capped_return_size"] + _require(bool(((size >= 1) & (size <= 5)).all()), "PUF59_ARTIFACT_RETURN_SIZE") + for name in INCIDENCE_FIELDS: + _require(bool(np.isin(arrays[name], [0, 1]).all()), "PUF59_ARTIFACT_INCIDENCE") + + +def _bindings(arrays, receipt): + _require( + prefix_values_digest(arrays) == receipt.get("prefix_values_sha256"), + "PUF59_ARTIFACT_PREFIX_BINDING", + ) + _require( + _digest(arrays) == receipt["growth"].get("output_values_sha256"), + "PUF59_ARTIFACT_GROWTH_BINDING", + ) + + +def _encode(arrays, receipt, expected_growth_scheme): + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + n = len(arrays["RECID"]) + _validate(arrays, n) + r = _receipt(receipt, expected_growth_scheme) + _bindings(arrays, r) + _require(r["rows"] == n, "PUF59_ARTIFACT_RECEIPT_ROWS") + blocks = [ + np.asarray(arrays[name], dtype=" len(MAGIC) + 4 + length, + "PUF59_ARTIFACT_HEADER_SIZE", + ) + start = len(MAGIC) + 4 + + def pairs(items): + d = {} + for k, v in items: + _require(k not in d, "PUF59_ARTIFACT_DUPLICATE_JSON_KEY") + d[k] = v + return d + + h = json.loads( + payload[start : start + length], + object_pairs_hook=pairs, + parse_constant=lambda _: (_ for _ in ()).throw( + ValueError("PUF59_ARTIFACT_JSON_NONFINITE") + ), + ) + _require( + type(h) is dict + and set(h) == {"schema", "rows", "names", "dtypes", "body_sha256", "receipt"}, + "PUF59_ARTIFACT_HEADER_CONTRACT", + ) + _require( + json.dumps(h, sort_keys=True, separators=(",", ":"), allow_nan=False).encode( + "utf-8" + ) + == payload[start : start + length], + "PUF59_ARTIFACT_HEADER_CANONICAL", + ) + _require( + h["schema"] == "microcosm.us.puf59_canonical_artifact/2" + and h["names"] == list(NAMES) + and h["dtypes"] == ["= 0).all(), "FULL_COUNT_MARS") + for name in ("XFPT", "XFST"): + _require( + np.isin(values[name][ordinary], (0, 1)).all(), "FULL_EXEMPTION_FLAG:" + name + ) + cumulative = np.zeros(len(mars), dtype=np.int64) + for name in DEPENDENT_COLUMNS: + count = values[name][ordinary] + _require(((count >= 0) & (count <= 3)).all(), "FULL_DEPENDENT_DOMAIN:" + name) + cumulative += count + _require((cumulative <= caps).all(), "FULL_DEPENDENT_SEQUENTIAL_CAP:" + name) + xtot = values["XTOT"][ordinary] + _require(((xtot >= 0) & (xtot <= 5)).all(), "FULL_EXEMPTIONS_DOMAIN") + _require( + np.array_equal( + xtot, values["XFPT"][ordinary] + values["XFST"][ordinary] + cumulative + ), + "FULL_EXEMPTIONS_COMPONENT_IDENTITY", + ) + + +def puf_2015_receiver_size_measurement(filing_status_code, dependent_count): + """Match the PUF's coarse status and censored return-size predictor. + + Generic model status codes are single1/joint2/separate3/HOH4/widow5. + PUF code2 combines joint and surviving-spouse returns. Its baseline of two + is a predictor convention for that combined class, not an observed number + of living filers. Physical survey membership must remain unchanged. + """ + status = np.asarray(filing_status_code) + dependents = np.asarray(dependent_count) + _require( + status.ndim == 1 and dependents.shape == status.shape, + "FULL_RECEIVER_COUNT_SHAPE", + ) + _require( + status.dtype.kind in "iu" and dependents.dtype.kind in "iu", + "FULL_RECEIVER_COUNT_TYPE", + ) + _require(np.isin(status, (1, 2, 3, 4, 5)).all(), "FULL_RECEIVER_STATUS_DOMAIN") + _require( + ((dependents >= 0) & (dependents <= 1000)).all(), + "FULL_RECEIVER_DEPENDENT_BOUND", + ) + mars = np.where(status == 5, 2, status).astype(np.int64) + cap = np.select([mars == 1, mars == 2, mars == 3, mars == 4], [2, 3, 1, 3]) + return { + "puf_2015_filing_status_code": mars, + "puf_2015_capped_return_size": 1 + + (mars == 2).astype(np.int64) + + np.minimum(dependents, cap), + } + + +def _frozen_array(values): + values = np.asarray(values) + return np.frombuffer(values.tobytes(), dtype=values.dtype).reshape(values.shape) + + +@dataclass(frozen=True) +class FullPufSource: + """Read-only source values; available is not an admission or truth flag.""" + + definition_sha256: str + source_sha256: Mapping[str, str] + status: Mapping[str, np.ndarray] + values: Mapping[str, np.ndarray] + aggregate_tokens: Mapping[int, Mapping[str, str]] + status_payload: bytes + + @property + def ordinary(self): + return self.status["disclosure_aggregate"] == 0 + + +def decode_full_puf_source(main_bytes, demographic_bytes, definition): + """Authenticate, decode, and align a complete declared delivery in memory.""" + _require(isinstance(definition, raw.PufRawSourceDefinition), "FULL_DEFINITION") + for data, pin in ( + (main_bytes, definition.main), + (demographic_bytes, definition.demographic), + ): + _require(type(data) is bytes and len(data) == pin.bytes, "FULL_SOURCE_SIZE") + _require(hashlib.sha256(data).hexdigest() == pin.sha256, "FULL_SOURCE_SHA256") + status = raw.decode_puf_raw_source(main_bytes, demographic_bytes, definition) + rows = len(status.typed["RECID"]) + _require( + rows * len(PROJECTED_COLUMNS) * 8 <= FULL_SOURCE_MAX_BYTES, + "FULL_PROJECTION_BODY_LIMIT", + ) + profile = definition.document["csv_profile"] + reader = raw._reader(main_bytes, profile) + header = raw._check_header(reader, definition.main, profile=profile) + _require(set(PROJECTED_COLUMNS) <= set(header), "FULL_PROJECTED_HEADER") + positions = {name: header.index(name) for name in PROJECTED_COLUMNS} + recid_position = header.index("RECID") + values = { + name: np.full(rows, OUTSIDE_AMOUNT_UNIVERSE, dtype="= offset + 8, "FULL_HEADER_LENGTH") + size = struct.unpack(" 0, "FULL_HEADER_LIMIT") + offset += 8 + _require(offset + size <= len(payload), "FULL_TRUNCATED_HEADER") + header_bytes = payload[offset : offset + size] + try: + h = json.loads(header_bytes) + except (UnicodeError, ValueError) as error: + raise ValueError("FULL_HEADER_JSON") from error + _require( + type(h) is dict + and set(h) + == { + "definition_sha256", + "source_sha256", + "status_bytes", + "status_sha256", + "rows", + "columns", + "aggregate_tokens", + }, + "FULL_HEADER_KEYS", + ) + _require( + json.dumps( + h, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).encode("ascii") + == header_bytes, + "FULL_HEADER_CANONICAL", + ) + _require( + h["definition_sha256"] == definition.sha256 + and h["source_sha256"] + == { + "main": definition.main.sha256, + "demographic": definition.demographic.sha256, + }, + "FULL_ARTIFACT_SOURCE_BINDING", + ) + rows = h["rows"] + _require( + type(rows) is int and rows == definition.main.data_records, "FULL_ARTIFACT_ROWS" + ) + _require( + type(h["status_bytes"]) is int + and 0 < h["status_bytes"] <= FULL_SOURCE_MAX_BYTES, + "FULL_STATUS_LENGTH", + ) + offset += size + status_payload = payload[offset : offset + h["status_bytes"]] + _require( + len(status_payload) == h["status_bytes"] + and hashlib.sha256(status_payload).hexdigest() == h["status_sha256"], + "FULL_STATUS_SHA256", + ) + status = raw.decode_return_status(status_payload, definition=definition) + offset += h["status_bytes"] + _require( + type(h["columns"]) is list and len(h["columns"]) == len(PROJECTED_COLUMNS), + "FULL_ARTIFACT_COLUMNS", + ) + values = {} + for name, entry in zip(PROJECTED_COLUMNS, h["columns"], strict=True): + _require( + type(entry) is dict + and set(entry) == {"name", "bytes", "sha256"} + and entry["name"] == name + and type(entry["bytes"]) is int + and entry["bytes"] == rows * 8, + "FULL_COLUMN_DECLARATION", + ) + data = payload[offset : offset + entry["bytes"]] + _require( + len(data) == entry["bytes"] + and hashlib.sha256(data).hexdigest() == entry["sha256"], + "FULL_COLUMN_SHA256", + ) + values[name] = np.frombuffer(data, dtype="= -cap) & (value[~aggregate] <= cap)).all() + and (name not in COUNT_COLUMNS or (value[~aggregate] >= 0).all()), + "FULL_ORDINARY_TYPED_DOMAIN", + ) + _validate_count_fields(values, status.typed) + return FullPufSource( + definition.sha256, + MappingProxyType(h["source_sha256"]), + status.typed, + MappingProxyType(values), + MappingProxyType({int(k): MappingProxyType(v) for k, v in tokens.items()}), + status_payload, + ) + + +DIRECT_MAPPINGS = { + "employment_income_before_lsr": "E00200", + "self_employment_income_before_lsr": "E00900", + "taxable_interest_income": "E00300", + "qualified_dividend_income": "E00650", + "tax_exempt_interest_income": "E00400", + "short_term_capital_gains": "P22250", + "long_term_capital_gains_before_response": "P23250", + "long_term_capital_gains_on_collectibles": "E24518", + "non_sch_d_capital_gains": "E01100", + "taxable_private_pension_income": "E01700", + "taxable_ira_distributions": "E01400", + "alimony_income": "E00800", + "alimony_expense": "E03500", + "salt_refund_income": "E00700", + "charitable_cash_donations": "E19800", + "charitable_non_cash_donations": "E20100", + "real_estate_taxes": "E18500", + "investment_income_elected_form_4952": "E58990", + "student_loan_interest": "E03210", + "educator_expense": "E03220", + "casualty_loss": "E20500", + "farm_income": "T27800", + "farm_operations_income": "E02100", + "farm_rent_income": "E27200", + "miscellaneous_income": "E01200", + "domestic_production_ald": "E03240", + "unrecaptured_section_1250_gain": "E24515", + "health_savings_account_ald": "E03290", +} + + +def observed_and_derived_return_columns(source, *, selected_recids=None): + """Return canonical observed/derived quantities plus explicit model inputs. + + This stage intentionally retains total SS, total interest, realized IRA/ + Keogh deductions and total miscellaneous itemized deductions under honest + auxiliary names. The next declared model owner supplies component/proxy + outputs; their absence is never silently converted to a known zero here. + """ + _require(isinstance(source, FullPufSource), "FULL_SOURCE_TYPE") + mask = source.ordinary.copy() + if selected_recids is not None: + ids = tuple(selected_recids) + _require( + len(ids) > 0 + and all(type(i) is int for i in ids) + and len(set(ids)) == len(ids), + "FULL_SELECTION_IDS", + ) + _require( + set(ids) <= set(source.status["RECID"][mask].tolist()), + "FULL_SELECTION_UNIVERSE", + ) + mask &= np.isin(source.status["RECID"], ids) + a = {name: value[mask] for name, value in source.values.items()} + out = {name: a[field].copy() for name, field in DIRECT_MAPPINGS.items()} + out.update( + { + "non_qualified_dividend_income": a["E00600"] - a["E00650"], + "rental_income": a["E25850"] - a["E25860"], + "estate_income": a["E26390"] - a["E26400"], + "partnership_income": ( + a["E25940"] + a["E25980"] - a["E25920"] - a["E25960"] - a["E26110"] + ), + "s_corp_income": ( + a["E26170"] + a["E26190"] - a["E26160"] - a["E26180"] - a["E26100"] + ), + } + ) + auxiliary = { + "raw_adjusted_gross_income": a["E00100"], + "raw_total_social_security": a["E02400"], + "raw_total_interest_deduction": a["E19200"], + "raw_realized_ira_deduction": a["E03150"], + "raw_realized_keogh_deduction": a["E03300"], + "raw_tuition_fees_deduction": a["E03230"], + "raw_lifetime_learning_qualified_expenses": a["E87530"], + "raw_miscellaneous_itemized_deductions": a["E20400"], + "raw_partnership_nonpassive_net": a["E25980"] - a["E25960"], + "raw_partnership_s_corp_combined": a["E26270"], + "raw_exemptions_count": a["XTOT"], + **{"raw_" + name.lower(): a[name] for name in COUNT_COLUMNS[:-1]}, + "reported_dependent_count": sum(a[name] for name in DEPENDENT_COLUMNS), + "puf_2015_filing_status_code": source.status["MARS"][mask].copy(), + "puf_2015_capped_return_size": ( + 1 + + (source.status["MARS"][mask] == 2).astype(np.int64) + + sum(a[name] for name in DEPENDENT_COLUMNS) + ), + "raw_ordinary_dividend_income": a["E00600"], + } + auxiliary["partnership_component_reconciliation_residual"] = ( + out["partnership_income"] + out["s_corp_income"] - a["E26270"] + ) + # Canonical intermediates stay integer dollars until the explicit model + # boundary. S006 stays integer hundredths here too. + status = {name: value[mask].copy() for name, value in source.status.items()} + return out, auxiliary, status diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py new file mode 100644 index 000000000..6f8f13044 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py @@ -0,0 +1,122 @@ +"""Graph owner of the additive complete PUF return-source artifact.""" + +from __future__ import annotations + +import hashlib +from importlib import import_module + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes + +from . import puf_full_source as full +from . import puf_raw_source as raw + +FULL_RETURN_SOURCE_TYPE = ArtifactType("microcosm.us.puf_2015_full_return_source", 3) + + +class FullPufReturnSourceKernel(KernelBase): + ref = "us.puf.full_return_source@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, dependencies=("numpy",) + ) + + def __init__(self, *, definition=None, source_codecs=None): + self.definition = ( + raw.packaged_definition() if definition is None else definition + ) + self.source_codecs = ( + raw.puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self): + base = source_hash( + import_module(__name__), + full, + raw, + dependencies=self.capabilities.dependencies, + ) + return hashlib.sha256( + base.encode("ascii") + b"\0" + canonical_json(raw.csv_acceptance_profile()) + ).hexdigest() + + def run(self, context): + d = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (d.main.source_name, d.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE),) + or dict(context.params) != {"definition": d.params_text} + ): + raise ValueError("FULL_SOURCE_NODE_DECLARATION") + main = load_source_bytes( + d.main.codec, + context.sources[d.main.source_name], + registry=self.source_codecs, + ) + demo = load_source_bytes( + d.demographic.codec, + context.sources[d.demographic.source_name], + registry=self.source_codecs, + ) + decoded = full.decode_full_puf_source(main, demo, d) + payload = full.encode_full_puf_source(decoded) + return KernelResult( + artifacts={"full_return_source": payload}, + receipt={ + "full_puf_return_source": { + "definition_sha256": d.sha256, + "definition_route": d.route, + "source_sha256": dict(decoded.source_sha256), + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "rows": len(decoded.status["RECID"]), + "ordinary_rows": int(decoded.ordinary.sum()), + "projected_monetary_fields": len(full.MONEY_COLUMNS), + "projected_count_fields": len(full.COUNT_COLUMNS), + "amount_units": "source_whole_us_dollars", + "weight_units": "S006_integer_hundredths", + "period": "raw_FLPDYR_preserved", + "canonical_modeling": False, + "target_year_growth": False, + "donor_admission": False, + } + }, + ) + + +def full_puf_return_source_node( + *, population, definition=None, node_id="us_puf_full_source.return_source" +): + if not isinstance(population, str) or not population: + raise ValueError("FULL_SOURCE_POPULATION") + d = raw.packaged_definition() if definition is None else definition + return Node( + node_id, + FullPufReturnSourceKernel.ref, + population=population, + sources=(d.main.source_name, d.demographic.source_name), + params={"definition": d.params_text}, + artifact_outputs=( + ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE), + ), + description="Authenticate and type 52 monetary fields, seven exemption/dependent count fields, and complete PUF source status; no model or growth.", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qbi_model.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qbi_model.py new file mode 100644 index 000000000..a16202aaf --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qbi_model.py @@ -0,0 +1,369 @@ +"""Explicit canonical PUF QBI data model, adapted from the archived PUF owner. + +These modeled qualification and business-input leaves are not raw IRS facts +or tax deductions. PolicyEngine owns the statutory calculation. Source cells +must be complete before this boundary; no missing cell becomes an assumed zero. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from .operator_column_contracts import US_QBI_OUTPUT_COLUMNS + +VERSION = "microcosm.us.puf_canonical_qbi_model/1" +ARCHIVED_COMMIT = "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" +ARCHIVED_CODE_SHA256 = ( + "359fd71242315e5abb7b857bb83d079ac62e46b3edc46adf51481efaa6ac858e" +) +ARCHIVED_ASSUMPTIONS_SHA256 = ( + "b32f29c07061ddf41dae12f8614e033347122aa05943225dcc6bdb2bdfbb48b7" +) +SOURCES = ( + "self_employment_income", + "farm_operations_income", + "farm_rent_income", + "rental_income", + "estate_income", + "partnership_s_corp_income", +) +REQUIRED_INPUTS = ( + "self_employment_income_before_lsr", + "farm_operations_income", + "farm_rent_income", + "rental_income", + "estate_income", + "partnership_income", + "s_corp_income", + "non_qualified_dividend_income", +) +OUTPUTS = (*US_QBI_OUTPUT_COLUMNS, "self_employment_income_before_lsr") +# Exact source-ordered values from archived_qbi_assumptions.yaml. The source +# archive hash is provenance; executable parameter identity hashes these values. +_PARAMETERS_JSON = """{ + "qualification": [0.95,0.98,0.80,0.70,0.60,0.90], + "sstb_sources": ["self_employment_income","partnership_s_corp_income","estate_income"], + "sstb_probabilities": [0.30,0.25,0.15], + "profit_margin": [[2,2,0.50,0.05],[2,4,0.45,0.03],[2,3,0.45,0.05],[2.5,3,0.50,0.05],[2,3,0.45,0.05],[2,3.5,0.45,0.05]], + "employee_logit": {"slope_per_dollar":1.2e-6,"target_share":0.18}, + "labor_ratio": [[2,2,0.25,0],[2,2.5,0.22,0],[1.5,8,0.08,0],[1.5,8,0.08,0],[2,2.5,0.20,0],[2,2,0.22,0]], + "ubia": {"sigma":1.0,"multiple":[1.5,4,8,10,3,3],"capital_probability":[0.25,0.70,0.90,0.95,0.50,0.45]}, + "reit_ptp": [["non_qualified_dividend_income",0.35,2,18,1],["partnership_s_corp_income",0.05,2,8,0.60]], + "bdc": [["non_qualified_dividend_income",0.05,2,18,0.30]] +}""" + + +def model_parameters(): + """Return a fresh copy, avoiding mutable global assumptions.""" + return json.loads(_PARAMETERS_JSON) + + +def _json(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _sha(data): + return hashlib.sha256(data).hexdigest() + + +PARAMETERS_SHA256 = _sha(_json(model_parameters()).encode()) + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _vector(value, name, n): + values = np.asarray(value) + _require( + values.ndim == 1 and len(values) == n and values.dtype.kind in "ifu", + "QBI_PHYSICAL_TYPE:" + name, + ) + if values.dtype.kind in "iu": + _require( + bool(((values >= -(2**53)) & (values <= 2**53)).all()), + "QBI_INTEGER_PRECISION:" + name, + ) + values = values.astype(np.float64) + _require(bool(np.isfinite(values).all()), "QBI_NONFINITE:" + name) + return values + + +def _ids(recids): + values = np.asarray(recids) + _require( + values.ndim == 1 + and len(values) > 0 + and values.dtype.kind in "iu" + and bool(((values > 0) & (values <= np.iinfo(np.int64).max)).all()), + "QBI_RECID_PHYSICAL_TYPE", + ) + _require(len(np.unique(values)) == len(values), "QBI_DUPLICATE_RECID") + return values.astype(" 0 + if not positive.any(): + return 0.0 + term = slope * revenues[positive] + target_logit = np.log(target / (1 - target)) + low, high = target_logit - np.max(term) - 80, target_logit - np.min(term) + 80 + for _ in range(100): + mid = (low + high) / 2 + if _logistic(mid + term).mean() < target: + low = mid + else: + high = mid + return float((low + high) / 2) + + +@dataclass(frozen=True) +class QbiEmploymentCalibration: + """A reusable statistical fit; this numerical record is not source authority.""" + + version: str + parameters_sha256: str + seed: int + fitted_cohort_sha256: str + fitted_rows: int + positive_receipt_rows: int + intercept: float + + +@dataclass(frozen=True) +class QbiModelResult: + columns: Mapping[str, np.ndarray] + calibration: QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def model_full_puf_qbi( + canonical, recids, *, known, input_money_year, seed=0, employment_calibration=None +): + """Model 15 QBI leaves and explicitly replace the ordinary Schedule C leaf. + + Knownness here means complete canonical input, possibly derived upstream; + it does not mean a modeled leaf was observed. Inputs and their RECIDs must + already belong to the ordinary-return cohort selected by the source owner. + Each RNG is keyed by RECID, seed, and the archived stream number. This is + intentionally not byte parity with the archive's row-position RNG. The + employee logit intercept is fit to the unweighted positive-receipt cohort + (the archive's 18% target); reuse the returned fit for stable subset/chunk + application. Fitting a different cohort is a different statistical model. + """ + ids = _ids(recids) + n = len(ids) + _require( + type(input_money_year) is int and input_money_year == 2015, + "QBI_INPUT_MONEY_YEAR", + ) + _require(type(seed) is int and 0 <= seed < 2**64, "QBI_SEED") + _require( + isinstance(canonical, Mapping) and isinstance(known, Mapping), + "QBI_INPUT_MAPPING", + ) + inputs = {} + for name in REQUIRED_INPUTS: + _require(name in canonical and name in known, "QBI_MISSING_INPUT:" + name) + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", "QBI_KNOWNNESS_TYPE:" + name + ) + _require(bool(mask.all()), "QBI_UNKNOWN:" + name) + inputs[name] = _vector(canonical[name], name, n) + source = np.column_stack( + [inputs[name] for name in REQUIRED_INPUTS[:5]] + + [inputs["partnership_income"] + inputs["s_corp_income"]] + ) + _require(bool(np.isfinite(source).all()), "QBI_SOURCE_SUM_NONFINITE") + params = model_parameters() + qualified = np.zeros((n, 6), dtype=bool) + margins, labor = np.zeros((n, 6)), np.zeros((n, 6)) + employees_draw, capital_draw, normal_draw, sstb_draw = np.zeros((4, n)) + investment_draws = np.zeros((n, 3)) + # Generate each record independently; calculations below remain vectorized. + for row, recid in enumerate(ids): + + def rng(stream, recid=int(recid)): + return np.random.Generator( + np.random.PCG64(np.random.SeedSequence([seed, stream, recid])) + ) + + qualified[row] = rng(41).random(6) < params["qualification"] + business = rng(42) + for col, (a, b, scale, shift) in enumerate(params["profit_margin"]): + margins[row, col] = business.beta(a, b) * scale + shift + employees_draw[row] = business.random() + for col, (a, b, scale, shift) in enumerate(params["labor_ratio"]): + labor[row, col] = business.beta(a, b) * scale + shift + capital_draw[row] = business.random() + normal_draw[row] = business.standard_normal() + sstb_draw[row] = rng(64).random() + investment = rng(43) + for col, (_name, probability, a, b, scale) in enumerate( + params["reit_ptp"] + params["bdc"] + ): + receives = investment.random() < probability + share = np.clip(investment.beta(a, b) * scale, 0, 1) + investment_draws[row, col] = share if receives else 0.0 + components = source * qualified + positive = np.maximum(components, 0) + positive_total = positive.sum(axis=1) + _require(bool(np.isfinite(positive_total).all()), "QBI_POSITIVE_TOTAL_NONFINITE") + fractions = np.divide( + positive, + positive_total[:, None], + out=np.zeros_like(positive), + where=positive_total[:, None] > 0, + ) + qbi = components.sum(axis=1) + _require(bool(np.isfinite(qbi).all()), "QBI_NET_TOTAL_NONFINITE") + margin = (fractions * margins).sum(axis=1) + revenues = np.divide(np.maximum(qbi, 0), margin, out=np.zeros(n), where=margin > 0) + _require(bool(np.isfinite(revenues).all()), "QBI_REVENUE_NONFINITE") + cohort_sha = _digest_columns(ids, inputs) + logit = params["employee_logit"] + if employment_calibration is None: + # Sort by identity before reduction so input permutations replay exactly. + employment_calibration = QbiEmploymentCalibration( + VERSION, + PARAMETERS_SHA256, + seed, + cohort_sha, + n, + int((revenues > 0).sum()), + _employee_intercept( + revenues[np.argsort(ids)], + logit["slope_per_dollar"], + logit["target_share"], + ), + ) + _require( + isinstance(employment_calibration, QbiEmploymentCalibration) + and employment_calibration.version == VERSION + and employment_calibration.parameters_sha256 == PARAMETERS_SHA256 + and employment_calibration.seed == seed + and type(employment_calibration.intercept) is float + and np.isfinite(employment_calibration.intercept), + "QBI_EMPLOYMENT_CALIBRATION_IDENTITY", + ) + probability = np.where( + revenues > 0, + _logistic( + employment_calibration.intercept + logit["slope_per_dollar"] * revenues + ), + 0.0, + ) + wages = revenues * (fractions * labor).sum(axis=1) * (employees_draw < probability) + capital_probability = fractions @ params["ubia"]["capital_probability"] + mean_ubia = (fractions @ params["ubia"]["multiple"]) * np.maximum(qbi, 0) + ubia = np.where( + (capital_draw < capital_probability) & (mean_ubia > 0), + mean_ubia + * np.exp( + params["ubia"]["sigma"] * normal_draw - params["ubia"]["sigma"] ** 2 / 2 + ), + 0.0, + ) + # Canonical source names ensure the SSTB draw consumes qualification flags. + sstb_positions = [SOURCES.index(name) for name in params["sstb_sources"]] + sstb_sources = positive[:, sstb_positions] + largest = sstb_sources.argmax(axis=1) + sstb_probability = np.where( + sstb_sources.sum(axis=1) > 0, + np.asarray(params["sstb_probabilities"])[largest], + 0.0, + ) + is_sstb = sstb_draw < sstb_probability + output = { + name + "_would_be_qualified": qualified[:, i].copy() + for i, name in enumerate(SOURCES) + } + schedule_c = inputs["self_employment_income_before_lsr"] + output.update( + { + "business_is_sstb": is_sstb, + "self_employment_income_before_lsr": np.where(is_sstb, 0, schedule_c), + "sstb_self_employment_income_before_lsr": np.where(is_sstb, schedule_c, 0), + "self_employment_income_would_be_qualified": qualified[:, 0] & ~is_sstb, + "sstb_self_employment_income_would_be_qualified": qualified[:, 0] & is_sstb, + "w2_wages_from_qualified_business": wages, + "unadjusted_basis_qualified_property": ubia, + "sstb_w2_wages_from_qualified_business": np.where(is_sstb, wages, 0), + "sstb_unadjusted_basis_qualified_property": np.where(is_sstb, ubia, 0), + "qualified_reit_and_ptp_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 0] + + np.maximum(source[:, 5], 0) * investment_draws[:, 1], + "qualified_bdc_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 2], + } + ) + _require(set(output) == set(OUTPUTS), "QBI_OUTPUT_ROSTER") + for name, values in output.items(): + _require(bool(np.isfinite(values).all()), "QBI_OUTPUT_NONFINITE:" + name) + frozen = { + name: np.frombuffer( + np.asarray(output[name]).tobytes(), dtype=output[name].dtype + ) + for name in OUTPUTS + } + receipt = { + "version": VERSION, + "origin": "modeled_not_observed", + "release_eligible": False, + "archived_commit": ARCHIVED_COMMIT, + "archived_code_sha256": ARCHIVED_CODE_SHA256, + "archived_assumptions_sha256": ARCHIVED_ASSUMPTIONS_SHA256, + "parameters_sha256": PARAMETERS_SHA256, + "input_money_year": input_money_year, + "input_money_unit": "nominal USD", + "source_statistical_baseline": 2015, + "growth": "not applied; employee slope per dollar is bound to this basis", + "sources": list(REQUIRED_INPUTS), + "input_cohort_sha256": cohort_sha, + "rows": n, + "seed": seed, + "randomness": "PCG64 SeedSequence([seed, archived_stream, RECID]); streams 41/42/43/64", + "numpy_version": np.__version__, + "input_knownness": "complete_canonical_not_raw_observation", + "employment_fit": dict(employment_calibration.__dict__), + "applied_positive_receipt_rows": int((revenues > 0).sum()), + "applied_employee_expected_share": float(probability[revenues > 0].mean()) + if (revenues > 0).any() + else 0.0, + "employee_target_weighting": "unweighted positive simulated receipts, archived target 0.18", + "subset_replay": "reuse employment_calibration; refitting a subset changes the model", + "schedule_c": "all-or-nothing SSTB split preserving the signed input total", + "wages_and_ubia": "base leaves are total pools; SSTB leaves are conditional copies", + "output_columns": list(OUTPUTS), + "output_values_sha256": _digest_columns(ids, frozen), + } + receipt["sha256"] = _sha(_json(receipt).encode()) + return QbiModelResult( + MappingProxyType(frozen), employment_calibration, MappingProxyType(receipt) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py new file mode 100644 index 000000000..f68c0e1e0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py @@ -0,0 +1,164 @@ +"""Source-specific, explicit PUF2015 to target2024 monetary transport. + +This is a data model. It computes no tax, deduction limit, benefit eligibility +or recipient membership. Receipt knownness is separate from raw observation. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +# Generated below from the reviewable, source-pinned GROWTH-RECIPE.json. +_RECIPE_JSON = '{\n "schema": "microcosm.us.puf2015_target2024_growth/1",\n "input_money_year": 2015,\n "output_money_year": 2024,\n "output_profile": "puf59_scf_mortgage_v1",\n "ordered_targets": [\n "employment_income_before_lsr",\n "self_employment_income_before_lsr",\n "taxable_interest_income",\n "qualified_dividend_income",\n "non_qualified_dividend_income",\n "tax_exempt_interest_income",\n "short_term_capital_gains",\n "long_term_capital_gains_before_response",\n "long_term_capital_gains_on_collectibles",\n "non_sch_d_capital_gains",\n "taxable_private_pension_income",\n "taxable_ira_distributions",\n "social_security_retirement",\n "social_security_disability",\n "social_security_dependents",\n "social_security_survivors",\n "alimony_income",\n "alimony_expense",\n "salt_refund_income",\n "charitable_cash_donations",\n "charitable_non_cash_donations",\n "real_estate_taxes",\n "home_mortgage_interest",\n "investment_interest_expense",\n "investment_income_elected_form_4952",\n "student_loan_interest",\n "educator_expense",\n "qualified_tuition_expenses",\n "casualty_loss",\n "unreimbursed_business_employee_expenses",\n "traditional_ira_contributions_desired",\n "self_employed_pension_contributions_desired",\n "rental_income",\n "estate_income",\n "farm_income",\n "farm_operations_income",\n "farm_rent_income",\n "miscellaneous_income",\n "partnership_income",\n "s_corp_income",\n "partnership_self_employment_net_earnings",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified",\n "business_is_sstb",\n "qualified_bdc_income",\n "qualified_reit_and_ptp_income",\n "sstb_self_employment_income_before_lsr",\n "sstb_unadjusted_basis_qualified_property",\n "sstb_w2_wages_from_qualified_business",\n "unadjusted_basis_qualified_property",\n "w2_wages_from_qualified_business",\n "domestic_production_ald",\n "unrecaptured_section_1250_gain",\n "health_savings_account_ald"\n ],\n "money_fields": {\n "employment_income_before_lsr": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "sstb_w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "sstb_self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "taxable_interest_income": {\n "positive_family": "taxable_interest",\n "negative_family": "taxable_interest",\n "positive_factor": "2.599750625636417399355437523686408804619",\n "negative_factor": "2.599750625636417399355437523686408804619",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "tax_exempt_interest_income": {\n "positive_family": "tax_exempt_interest",\n "negative_family": "tax_exempt_interest",\n "positive_factor": "0.9295395824924337172788943063597248917385",\n "negative_factor": "0.9295395824924337172788943063597248917385",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "qualified_dividend_income": {\n "positive_family": "qualified_dividends",\n "negative_family": "qualified_dividends",\n "positive_factor": "1.409891045424454325551259680558160347719",\n "negative_factor": "1.409891045424454325551259680558160347719",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_qualified_dividend_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_sch_d_capital_gains": {\n "positive_family": "non_schedule_d_distributions",\n "negative_family": "non_schedule_d_distributions",\n "positive_factor": "1.120350061633613230084419890940102481805",\n "negative_factor": "1.120350061633613230084419890940102481805",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_private_pension_income": {\n "positive_family": "taxable_pensions",\n "negative_family": "taxable_pensions",\n "positive_factor": "1.327588034973940867230285994253866499704",\n "negative_factor": "1.327588034973940867230285994253866499704",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_ira_distributions": {\n "positive_family": "ira_distributions",\n "negative_family": "ira_distributions",\n "positive_factor": "1.510871073557979130794209158828995676924",\n "negative_factor": "1.510871073557979130794209158828995676924",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "alimony_income": {\n "positive_family": "alimony_income",\n "negative_family": "alimony_income",\n "positive_factor": "1.542033706590010885068081742929926580179",\n "negative_factor": "1.542033706590010885068081742929926580179",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "salt_refund_income": {\n "positive_family": "state_tax_refund",\n "negative_family": "state_tax_refund",\n "positive_factor": "0.9296526775720357466732616002244135476521",\n "negative_factor": "0.9296526775720357466732616002244135476521",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "short_term_capital_gains": {\n "positive_family": "short_current_gain",\n "negative_family": "short_current_loss",\n "positive_factor": "2.042865587775988756504710527637911842775",\n "negative_factor": "1.738139193325432816803529458382722438642",\n "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded."\n },\n "long_term_capital_gains_before_response": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "long_term_capital_gains_on_collectibles": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "unrecaptured_section_1250_gain": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "social_security_retirement": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_disability": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_dependents": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_survivors": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "rental_income": {\n "positive_family": "rental_royalty_profit",\n "negative_family": "rental_royalty_loss",\n "positive_factor": "1.476663829685885394438774259073237903799",\n "negative_factor": "1.690793641559507564235995133214906035185",\n "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions."\n },\n "estate_income": {\n "positive_family": "estate_profit",\n "negative_family": "estate_loss",\n "positive_factor": "1.476331638358104936021886589313386353228",\n "negative_factor": "1.582810027948011789708036602150526302352",\n "assumption": ""\n },\n "farm_operations_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_rent_income": {\n "positive_family": "farm_rent_profit",\n "negative_family": "farm_rent_loss",\n "positive_factor": "1.629452810459537374541656575493532799084",\n "negative_factor": "1.150378978298943195689490419607276484593",\n "assumption": ""\n },\n "miscellaneous_income": {\n "positive_family": "other_property_gain",\n "negative_family": "other_property_loss",\n "positive_factor": "1.738242164922618767195190248113271033066",\n "negative_factor": "1.411645744283618865226902643798237455207",\n "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income."\n },\n "partnership_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "s_corp_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "partnership_self_employment_net_earnings": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "qualified_bdc_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation."\n },\n "qualified_reit_and_ptp_income": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool."\n },\n "alimony_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_non_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "real_estate_taxes": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "home_mortgage_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_interest_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_income_elected_form_4952": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "student_loan_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "educator_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "qualified_tuition_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "casualty_loss": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unreimbursed_business_employee_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "traditional_ira_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "self_employed_pension_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "domestic_production_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "health_savings_account_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "sstb_unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n }\n },\n "return_incidence_fields": [\n "business_is_sstb",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified"\n ],\n "source_pins": {\n "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8",\n "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3",\n "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b",\n "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c"\n },\n "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867",\n "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.",\n "operation_order": "Decode source statistical2015 \\u2192 source canonical transformations \\u2192 QBI2015 model (all16 outputs) \\u2192 this growth transform once \\u2192 2024 canonical donor. Monetary recipient predictors remain 2024.",\n "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.",\n "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority."\n}\n' +RECIPE_SHA256 = "f08bcc3c37434730bccbf81643c96089383d8e943c60f76ac8337f57a163e1f4" +if hashlib.sha256(_RECIPE_JSON.encode("utf-8")).hexdigest() != RECIPE_SHA256: + raise ValueError("PUF_GROWTH_RECIPE_IDENTITY") +VERSION = "microcosm.us.puf2015_target2024_growth/1" + + +def growth_recipe(): + return json.loads(_RECIPE_JSON) + + +_RECIPE = growth_recipe() +OUTPUTS = tuple(_RECIPE["ordered_targets"]) +INCIDENCE_FIELDS = tuple(_RECIPE["return_incidence_fields"]) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _digest(columns): + h = hashlib.sha256() + for name in OUTPUTS: + value = columns[name] + h.update(name.encode("ascii") + b"\0") + h.update(np.asarray(value, dtype=" 0, "PUF_GROWTH_NONEMPTY_VECTOR") + n = len(first) + clean = {} + for name in OUTPUTS: + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", + "PUF_GROWTH_KNOWNNESS_TYPE:" + name, + ) + _require(bool(mask.all()), "PUF_GROWTH_UNKNOWN:" + name) + value = np.asarray(canonical[name]) + allowed = "biu" if name in INCIDENCE_FIELDS else "ifu" + _require( + value.shape == (n,) and value.dtype.kind in allowed, + "PUF_GROWTH_PHYSICAL_TYPE:" + name, + ) + if value.dtype.kind in "iu": + _require( + bool(((value >= -(2**53)) & (value <= 2**53)).all()), + "PUF_GROWTH_INTEGER_PRECISION:" + name, + ) + _require(bool(np.isfinite(value).all()), "PUF_GROWTH_NONFINITE:" + name) + if name in INCIDENCE_FIELDS: + _require( + bool(((value == 0) | (value == 1)).all()), + "PUF_GROWTH_INCIDENCE_DOMAIN:" + name, + ) + clean[name] = value.astype(np.int64) + else: + clean[name] = value.astype(np.float64) + outputs = {} + for name, value in clean.items(): + if name in INCIDENCE_FIELDS: + grown = value.copy() + else: + rule = _RECIPE["money_fields"][name] + positive = float(rule["positive_factor"]) + negative = float(rule["negative_factor"]) + if scheme == "cpi_only": + positive = negative = float(_RECIPE["sensitivity_cpi_factor"]) + _require( + np.isfinite(positive) + and np.isfinite(negative) + and min(positive, negative) > 0, + "PUF_GROWTH_FACTOR", + ) + with np.errstate(over="ignore", invalid="ignore"): + grown = value * np.where(value < 0, negative, positive) + _require( + bool(np.isfinite(grown).all()), "PUF_GROWTH_OUTPUT_NONFINITE:" + name + ) + _require( + bool(np.array_equal(value == 0, grown == 0)) + and bool(np.array_equal(np.sign(value), np.sign(grown))), + "PUF_GROWTH_SIGN_OR_ZERO:" + name, + ) + outputs[name] = np.frombuffer(grown.tobytes(), dtype=grown.dtype) + receipt = { + "version": VERSION, + "recipe_sha256": RECIPE_SHA256, + "scheme": scheme, + "input_money_year": 2015, + "output_money_year": 2024, + "money_unit": "nominal USD", + "source_statistical_year": 2015, + "rows": n, + "origin": "modeled_distribution_transport", + "input_knownness": "complete_canonical_after_observed_derived_or_modeled_owner", + "source_pins": _RECIPE["source_pins"], + "monetary_columns": 51, + "unchanged_return_incidence_columns": 8, + "output_profile": _RECIPE["output_profile"], + "input_values_sha256": _digest(clean), + "output_values_sha256": _digest(outputs), + "no_weight_growth": True, + "no_extra_raw_row_year_cpi": True, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return PufGrowthResult(MappingProxyType(outputs), 2024, MappingProxyType(receipt)) diff --git a/packages/microcosm-build/tests/test_frame_checkpoint.py b/packages/microcosm-build/tests/test_frame_checkpoint.py index 7c79ba3f8..d5e052a40 100644 --- a/packages/microcosm-build/tests/test_frame_checkpoint.py +++ b/packages/microcosm-build/tests/test_frame_checkpoint.py @@ -576,6 +576,205 @@ def test_schema_3_checkpoint_requires_nullable_boolean_encoding( load_frame_checkpoint(path) +@pytest.mark.parametrize( + "dtype", ["Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64"] +) +@pytest.mark.parametrize("has_missing", [False, True]) +def test_nullable_integer_round_trip_preserves_width_values_and_null_mask( + tmp_path: Path, dtype: str, has_missing: bool +) -> None: + frame = _nullable_boolean_checkpoint_frame() + numpy_dtype = pd.api.types.pandas_dtype(dtype).numpy_dtype + bounds = np.iinfo(numpy_dtype) + frame.person["nullable_integer"] = pd.Series( + [int(bounds.min), pd.NA if has_missing else 0, int(bounds.max)], + index=frame.person.index, + dtype=dtype, + ) + first_path = tmp_path / "integer-first.h5" + second_path = tmp_path / "integer-second.h5" + + write_frame_checkpoint(first_path, frame) + loaded = load_frame_checkpoint(first_path) + write_frame_checkpoint(second_path, loaded.frame) + + assert first_path.read_bytes() == second_path.read_bytes() + pd.testing.assert_frame_equal(loaded.frame.person, frame.person, check_exact=True) + pd.testing.assert_frame_equal(loaded.frame.link("jobs"), frame.link("jobs")) + h5py = pytest.importorskip("h5py") + with h5py.File(first_path, mode="r") as h5: + metadata, spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + assert metadata["schema_version"] == 4 + assert spec == { + "name": "nullable_integer", + "dtype": dtype, + "encoding": "nullable_integer_v1", + "has_null_mask": has_missing, + } + values = np.asarray(group["values"]) + assert values.dtype == numpy_dtype + assert values.tolist() == [int(bounds.min), 0, int(bounds.max)] + if has_missing: + mask = np.asarray(group["null_mask"]) + assert mask.dtype == np.dtype(np.uint8) + assert mask.tolist() == [0, 1, 0] + else: + assert "null_mask" not in group + + +def _nullable_integer_checkpoint_frame() -> Frame: + frame = _checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + [1, pd.NA, 3], index=frame.person.index, dtype="Int64" + ) + return frame + + +def test_nullable_integer_masked_storage_is_canonical(tmp_path: Path) -> None: + paths = [tmp_path / "hidden-zero.h5", tmp_path / "hidden-nonzero.h5"] + for hidden, path in zip((0, 97), paths, strict=True): + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + pd.arrays.IntegerArray( + np.asarray([1, hidden, 3], dtype=np.int64), + np.asarray([False, True, False], dtype=np.bool_), + ), + index=frame.person.index, + ) + write_frame_checkpoint(path, frame) + assert paths[0].read_bytes() == paths[1].read_bytes() + + +@pytest.mark.parametrize( + "damage", + ["missing", "all_zero", "nonbinary", "wrong_length", "wrong_dtype", "wrong_rank"], +) +def test_nullable_integer_checkpoint_rejects_malformed_null_mask( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-mask.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["null_mask"] + if damage != "missing": + masks = { + "all_zero": np.asarray([0, 0, 0], dtype=np.uint8), + "nonbinary": np.asarray([0, 2, 0], dtype=np.uint8), + "wrong_length": np.asarray([0, 1], dtype=np.uint8), + "wrong_dtype": np.asarray([0, 1, 0], dtype=np.int16), + "wrong_rank": np.asarray([[0, 1, 0]], dtype=np.uint8), + } + group.create_dataset("null_mask", data=masks[damage], track_times=False) + with pytest.raises(ValueError, match="null mask"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("damage", ["wrong_dtype", "wrong_rank", "hidden_nonzero"]) +def test_nullable_integer_checkpoint_rejects_malformed_values( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-values.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["values"] + values = { + "wrong_dtype": np.asarray([1, 0, 3], dtype=np.uint64), + "wrong_rank": np.asarray([[1, 0, 3]], dtype=np.int64), + "hidden_nonzero": np.asarray([1, 2, 3], dtype=np.int64), + } + group.create_dataset("values", data=values[damage], track_times=False) + message = ( + "noncanonical nonzero" + if damage == "hidden_nonzero" + else "nullable integer values" + ) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("version", [2, 3]) +def test_legacy_checkpoint_cannot_smuggle_nullable_integer_encoding( + tmp_path: Path, version: int +) -> None: + path = tmp_path / "forged-legacy-integer.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, _spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + metadata["schema_version"] = version + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=f"schema version {version}.*nullable integer"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("has_null_mask", "yes", "has_null_mask"), + ("dtype", "int64", "must pair a supported pandas integer dtype"), + ("encoding", "numpy", "must pair a supported pandas integer dtype"), + ], +) +def test_nullable_integer_checkpoint_rejects_malformed_spec( + tmp_path: Path, field: str, value: object, message: str +) -> None: + path = tmp_path / "malformed-integer-spec.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + spec[field] = value + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +def test_maskless_nullable_integer_rejects_unexpected_null_mask(tmp_path: Path) -> None: + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = frame.person["nullable_integer"].fillna(0) + path = tmp_path / "unexpected-integer-mask.h5" + write_frame_checkpoint(path, frame) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + group.create_dataset( + "null_mask", data=np.zeros(3, dtype=np.uint8), track_times=False + ) + with pytest.raises(ValueError, match="unexpected null mask"): + load_frame_checkpoint(path) + + +def test_schema_4_checkpoint_requires_nullable_integer_encoding(tmp_path: Path) -> None: + path = tmp_path / "forged-schema-4.h5" + write_frame_checkpoint(path, _nullable_boolean_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata = json.loads(np.asarray(root["metadata_json"]).tobytes()) + metadata["schema_version"] = 4 + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match="schema version 4.*nullable integer"): + load_frame_checkpoint(path) + + def test_frame_checkpoint_fsyncs_parent_directory_after_rename( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -656,10 +855,9 @@ def test_frame_checkpoint_preserves_range_indexes_and_column_axis_name( @pytest.mark.parametrize( "unsupported", [ - pd.Series([1, pd.NA, 3], dtype="Int64"), pd.Series(["a", "b", "a"], dtype="category"), ], - ids=["nullable_integer", "categorical"], + ids=["categorical"], ) def test_frame_checkpoint_rejects_unsupported_dtype_without_replacing_destination( tmp_path: Path, diff --git a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py index 32ff4bbb6..79b14b6ea 100644 --- a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py +++ b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py @@ -44,6 +44,7 @@ def _rebuild_observations( extra_household, legacy_weights, missing_money=None, + current_predictor_money=None, ): """Build larger invented real checkpoints and the actual observation attachment.""" parent_path = tmp_path / "invented-original-v4.h5" @@ -88,6 +89,26 @@ def add_people(rows): # Alter only invented source construction, before identities, attachments # and normal issuance. Never mutate an already authenticated parent. person.iloc[position, person.columns.get_loc(field)] = np.nan + if current_predictor_money is not None: + # Alter only invented original source construction before checkpoint + # identities, observation attachments, private pins and fresh issuance. + person = tables["person"] + positions = np.flatnonzero(person.source_year.to_numpy() == 2024) + assert set(current_predictor_money) == { + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + } + for field, observations in current_predictor_money.items(): + observations = np.asarray(observations) + assert observations.dtype == np.dtype("float64") + assert ( + observations.shape == (len(positions),) + and np.isfinite(observations).all() + ) + person.iloc[positions, person.columns.get_loc(field)] = observations values = ( legacy_weights if legacy_weights is not None @@ -152,11 +173,17 @@ def _fixture( extra_household=False, legacy_weights=None, missing_money=None, + current_predictor_money=None, **changes, ): helpers = _helpers() _, persons = helpers._fixtures(tmp_path, monkeypatch, **changes) - if extra_household or legacy_weights is not None or missing_money is not None: + if ( + extra_household + or legacy_weights is not None + or missing_money is not None + or current_predictor_money is not None + ): _rebuild_observations( tmp_path, monkeypatch, @@ -164,6 +191,7 @@ def _fixture( extra_household=extra_household, legacy_weights=legacy_weights, missing_money=missing_money, + current_predictor_money=current_predictor_money, ) # Both real owners must pin the same final invented original member bytes. monkeypatch.setattr(restoration, "_MEMBER_PINS", coverage._MEMBER_PINS) diff --git a/packages/microcosm-build/tests/test_us_current_asec_demographics.py b/packages/microcosm-build/tests/test_us_current_asec_demographics.py new file mode 100644 index 000000000..128ee0c9d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_demographics.py @@ -0,0 +1,191 @@ +"""Actual source readers over constructed, privately pinned invented source bytes.""" + +import csv +import hashlib +import json +import shutil +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_demographics as owner +from microcosm.build.us_runtime import native_household_origin as origin + + +def _csv(path, headers, rows): + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(headers) + writer.writerows(rows) + data = path.read_bytes() + return SimpleNamespace( + size_bytes=len(data), + rows=len(rows), + member_sha256=hashlib.sha256(data).hexdigest(), + ) + + +@pytest.mark.parametrize( + "literal,code,status", + ( + ("06", 6, "in_printed_range"), + ("36", 36, "in_printed_range"), + ("", None, "missing"), + ("NA", None, "malformed"), + (" 6", None, "malformed"), + ("6.0", None, "malformed"), + ("57", None, "outside_printed_range"), + ("00", None, "outside_printed_range"), + ), +) +def test_household_state_reader_preserves_literal_and_knownness( + tmp_path, literal, code, status +): + path = tmp_path / "invented.csv" + pin = _csv(path, ("H_SEQ", "GESTFIPS", "unread"), [("00007", literal, "anything")]) + value = owner._read_state_capture(path, pin)[7] + assert value == { + "H_SEQ": "00007", + "GESTFIPS": literal, + "state_code": code, + "status": status, + "member_row_1based": 1, + } + + +@pytest.mark.parametrize("defect", ("duplicate", "digest", "size", "rows", "header")) +def test_state_capture_structure_and_exact_bytes_refuse(tmp_path, defect): + path = tmp_path / "invented.csv" + headers, rows = ("H_SEQ", "GESTFIPS"), [("00007", "06")] + if defect == "duplicate": + rows.append(("7", "36")) + if defect == "header": + headers = ("H_SEQ", "other") + pin = _csv(path, headers, rows) + if defect == "digest": + pin.member_sha256 = "f" * 64 + elif defect == "size": + pin.size_bytes += 1 + elif defect == "rows": + pin.rows += 1 + with pytest.raises(ValueError, match="CURRENT_ASEC_DEMOGRAPHICS_"): + owner._read_state_capture(path, pin) + + +def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False): + """Extend invented source members before actual preparation/issuance. + + Rebuild the real person-income attachment after member bytes change. Only + fixture registry pins are changed; no source issuer or ready() is replaced. + """ + arguments = fixture(tmp_path, monkeypatch) + source = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = source / f"pppub{year - 1999}.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + lines = table.A_LINENO.map(int) + table["A_SEX"] = lines.map({1: "1", 2: "2"}) + table["AXSEX"] = lines.map({1: "0", 2: "4"}) + table["A_EXPRRP"] = lines.map({1: "1", 2: "5"}) + table["P_SEQ"] = table.A_LINENO + if unknown and year == 2024: + table.loc[table.index[0], "AXSEX"] = "1" + assert table.notna().all().all() + # Reverse the actual source order, so receiving-row alignment cannot pass. + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in (coverage, restoration, owner.demographic): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "demographic-restored-money" + restoration.restore_asec_person_income_source( + source / "parent.h5", + source / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, source / "person-income-attachment.h5" + ) + path = source / "hhpub25.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + table["GESTFIPS"] = table.H_SEQ.map({"00007": "06", "00008": "36"}) + if unknown: + table.loc[table.H_SEQ.eq("00008"), "GESTFIPS"] = "" + assert table.notna().all().all() + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_actual_current_native_parent_binds_demographics_by_origin( + tmp_path, monkeypatch, unknown +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + prepared = owner.preparation_owner.prepare_authenticated_survey_population( + **arguments + ) + original = prepared.checked_view().frame + retained = {e: original.table(e).copy(deep=True) for e in original.entities} + result = owner.qualify_current_asec_demographics(prepared) + receipt = json.loads(result.receipt) + assert receipt["sex_observation_year"] == receipt["state_observation_year"] == 2025 + assert receipt["income_year"] == 2024 + assert not receipt["state_is_income_year_residence_claim"] + assert not receipt["source_admission_issued"] and not receipt["release_eligible"] + h = result.household.set_index("H_SEQ_integer") + assert h.loc[7, "GESTFIPS"] == "06" and h.loc[7, "state_fips"] == 6 + if unknown: + assert h.loc[8, "GESTFIPS"] == "" and pd.isna(h.loc[8, "state_fips"]) + assert receipt["state_unknown_households"] == 1 + assert receipt["sex_unknown_persons"] == 1 + unbound = result.person.asec_AXSEX.eq(1) + assert result.person.loc[unbound, "is_female"].isna().all() + assert not result.person.loc[unbound, "sex_known"].any() + else: + assert h.loc[8, "state_fips"] == 36 + assert ( + receipt["state_unknown_households"] == receipt["sex_unknown_persons"] == 0 + ) + allocated = result.person.asec_AXSEX.eq(4) + assert allocated.any() and result.person.loc[allocated, "is_female"].all() + assert result.person.loc[allocated, "sex_origin"].eq("census_allocated").all() + for entity, expected in retained.items(): + pd.testing.assert_frame_equal( + expected, original.table(entity), check_exact=True + ) + assert owner.qualify_current_asec_demographics(prepared).receipt == result.receipt + # Value transport is deliberately not an issued authority. Mutating it must + # not affect a fresh live projection or its source-bound receipt. + result.household.loc[result.household.index[0], "state_fips"] = 99 + fresh = owner.qualify_current_asec_demographics(prepared) + assert not fresh.household.state_fips.eq(99).any() diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictors.py b/packages/microcosm-build/tests/test_us_current_survey_predictors.py new file mode 100644 index 000000000..b8b51549b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_predictors.py @@ -0,0 +1,422 @@ +"""Actual source owners and graph replay on pinned invented survey inputs. + +No production source issuer/readiness call is replaced. The maintained fixture +constructs its own real bytes and test pins before normal authentication. The two fixtures exercise both zero and nonconstant positive-regime forests. Neither +fixture establishes held-out fit quality. +""" + +import json +from dataclasses import replace +from types import MappingProxyType + +import numpy as np +import pandas as pd +import pytest +from test_us_current_survey_puf_transfer import _copy_population +from test_us_graph_survey_population import authenticated_arguments + +from microcosm.build.us_runtime import cps_carried_current as leaves +from microcosm.build.us_runtime import current_survey_predictors as values +from microcosm.build.us_runtime import graph_current_survey_predictors as graph +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_support as support +from microcosm.fit import qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import Frame, WeightKind +from microcosm.graph import compile_graph, run_graph +from microcosm.graph.keys import opaque_artifact_key + + +def test_survey_source_ids_preserve_zero_without_relaxing_physical_or_unique_axis(): + original = pd.Series([0, 2, 5], dtype="int64") + result = values._ids(original) + np.testing.assert_array_equal(result, np.array([0, 2, 5], dtype="int64")) + result[0] = 9 + assert original.iloc[0] == 0 + for bad in ( + pd.Series([-1, 0, 2], dtype="int64"), + pd.Series([0, 0, 2], dtype="int64"), + ): + with pytest.raises(ValueError, match="^CURRENT_SURVEY_PREDICTOR_ID_AXIS$"): + values._ids(bad) + for bad in ( + pd.Series([0.0, 1.0]), + pd.Series([False, True]), + pd.Series([0, 1], dtype="Int64"), + ): + with pytest.raises(ValueError, match="^CURRENT_SURVEY_PREDICTOR_ID_DTYPE$"): + values._ids(bad) + + +def _money(): + return { + "WSAL_VAL": np.array([0.0, 50000.0, 100000.0]), + "SEMP_VAL": np.array([-400.0, 0.0, 500.0]), + "INT_VAL": np.array([0.0, 120.0, 300.0]), + "DIV_VAL": np.array([40.0, 0.0, 75.0]), + "CAP_VAL": np.array([-700.0, 0.0, 1100.0]), + } + + +def test_shared_current_money_split_preserves_signs_and_partition_totals(): + source = _money() + result = leaves.derive_cps_current_predictor_leaves(source) + assert set(result) == set(values.OUTPUTS) + # The legacy full leaf owner unpacks this mapping. Preserve its original + # insertion order so consumers constructing a DataFrame see the same schema. + assert tuple(result) == ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "long_term_capital_gains_before_response", + "short_term_capital_gains", + ) + np.testing.assert_array_equal( + result["self_employment_income_before_lsr"], source["SEMP_VAL"] + ) + np.testing.assert_allclose( + result["qualified_dividend_income"] + result["non_qualified_dividend_income"], + source["DIV_VAL"], + rtol=1e-15, + atol=0, + ) + np.testing.assert_allclose( + result["short_term_capital_gains"] + + result["long_term_capital_gains_before_response"], + source["CAP_VAL"], + rtol=1e-15, + atol=0, + ) + np.testing.assert_array_equal( + result["taxable_interest_income"], + source["INT_VAL"] * leaves.TAXABLE_INTEREST_FRACTION, + ) + result["employment_income_before_lsr"][0] = 10.0 + assert source["WSAL_VAL"][0] == 0.0 + + +@pytest.mark.parametrize( + "bad", + [ + np.array([np.nan, 1.0, 2.0]), + np.array([np.inf, 1.0, 2.0]), + np.array(["0", "1", "2"]), + np.array([True, False, True]), + np.array([1, 2, 3]), + np.zeros((3, 1)), + np.zeros(2), + ], +) +def test_shared_money_split_rejects_missing_or_coerced_source(bad): + source = _money() + source["INT_VAL"] = bad + with pytest.raises(ValueError, match="PREDICTOR_MONEY"): + leaves.derive_cps_current_predictor_leaves(source) + + +@pytest.mark.parametrize("nonconstant", (False, True)) +def test_current_predictor_source_fit_attachment_and_required_replay( + tmp_path, monkeypatch, nonconstant +): + observations = { + "WSAL_VAL": np.array([40000.0, 0.0, 140000.0, 0.0]), + "SEMP_VAL": np.array([1200.0, 0.0, 12000.0, 0.0]), + "INT_VAL": np.array([120.0, 0.0, 9000.0, 0.0]), + "DIV_VAL": np.array([400.0, 0.0, 4000.0, 0.0]), + "CAP_VAL": np.array([500.0, 0.0, 14000.0, 0.0]), + } + fixture_kwargs = ( + {"zero": False, "current_predictor_money": observations} if nonconstant else {} + ) + live = survey.run_authenticated_survey_population( + **authenticated_arguments(tmp_path, monkeypatch, **fixture_kwargs), + clones=True, + return_values=True, + ) + preparation, allocated, clone = ( + live.preparation, + live.allocated_population, + live.clone_population, + ) + qualified = values.qualify_current_survey_predictors(preparation, allocated, clone) + assert qualified.evidence["asec_interview_year"] == 2025 + assert qualified.evidence["asec_income_window"] == "calendar_year_2024" + assert qualified.evidence["price_basis_year"] == 2024 + # The producer's actual MappingProxyType receipt stays read-only. The + # portable projection contains an explicit plain snapshot, not a generic + # serializer fallback that could silently coerce unknown evidence objects. + assert type(qualified.evidence["earnings_universe"]) is dict + assert ( + json.loads(qualified.projection)["earnings_universe"] + == qualified.evidence["earnings_universe"] + ) + assert ( + qualified.evidence["model_judgments"]["quality_acceptance"] + == "held_out_fit_quality_not_yet_assessed" + ) + assert qualified.donor_frame.resolve_weights("person").kind is WeightKind.DESIGN + # A separate numerical probe changes only a defensive invented copy. It + # exercises the actual age15 owner; it is never issued as authenticated. + source_frame = qualified.source_frame + probe = Frame( + {e: source_frame.table(e).copy(deep=True) for e in source_frame.entities}, + source_frame.schema, + {e: source_frame.weights_for(e) for e in source_frame.weighted_entities}, + source_frame.strata.copy(deep=True), + metadata=source_frame.metadata, + mass_log=source_frame.mass_log, + ) + acs_index = probe.person.index[ + probe.person[values.provenance.support_channel_column("person")].eq("acs") + ][0] + for name in ("age", "AGEP"): + probe.person.loc[acs_index, name] = 14 + for name in ("WAGP", "SEMP", *values.OUTPUTS[:2]): + probe.person.loc[acs_index, name] = np.nan + completed_probe, receipt = values._acs_earnings(probe) + assert completed_probe.person.loc[acs_index, list(values.OUTPUTS[:2])].eq(0).all() + assert completed_probe.person.loc[acs_index, ["WAGP", "SEMP"]].isna().all() + assert receipt["structurally_absent_person_rows"] > 0 + assert type(receipt) is MappingProxyType + with pytest.raises(TypeError): + receipt["minimum_age"] = 0 + assert receipt["minimum_age"] == 15 + probe.person.loc[acs_index, "age"] = 15 + probe.person.loc[acs_index, "AGEP"] = 15 + with pytest.raises(ValueError, match="ACS_ELIGIBLE_EARNINGS_UNKNOWN"): + values._acs_earnings(probe) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + invented_draw = pd.DataFrame( + { + target: np.arange(1.0, 1.0 + len(matrix.entity_ids)) * (i + 1.0) + for i, target in enumerate(values.TARGETS) + }, + index=matrix.features.index, + ) + original_join = values.complete_predictor_columns( + qualified, clone.frame, invented_draw + ) + reversed_join = values.complete_predictor_columns( + replace( + qualified, + native_money=qualified.native_money.iloc[::-1], + origins=qualified.origins.iloc[::-1], + ), + clone.frame, + invented_draw, + ) + for name in original_join: + pd.testing.assert_series_equal(original_join[name], reversed_join[name]) + assert allocated.frame.resolve_weights("person").kind is WeightKind.IMPORTANCE + if nonconstant: + origins = qualified.origins + ids = origins.index[origins.source.eq("asec")] + # Join the current-money projection by native identity, never row position. + ordered = origins.loc[ids].sort_values("native_person_id").index + assert origins.loc[ordered, "native_person_id"].tolist() == [105, 106, 107, 108] + for field, expected in observations.items(): + np.testing.assert_array_equal( + qualified.native_money.loc[ordered, field], expected + ) + for target in values.TARGETS: + assert qualified.donor_columns[target].nunique() == 3 + weights = qualified.donor_frame.resolve_weights("person") + assert weights.kind is WeightKind.DESIGN and (weights.values > 0).all() + np.testing.assert_array_equal(weights.values, [2552.12, 2552.12, 100.0, 100.0]) + else: + assert qualified.donor_columns.loc[:, list(values.TARGETS)].eq(0).all().all() + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, clone.frame, host_pins=pins, n_estimators=2 + ) + assert nodes[1].base == survey.CREATE_NODE + assert nodes[0].population == graph.DONOR_NODE + assert tuple(edge.name for edge in nodes[1].artifact_inputs) == ("preparation",) + assert all("state" not in c and "female" not in c for c in values.FEATURES) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + # Actual compiler dependencies retain allocation evidence without an + # ordinary node in CREATE depending on its own downstream allocation. + assert graph.PROJECTION_NODE not in compiled.predecessors[survey.ALLOCATION_NODE] + assert graph.DONOR_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert survey.ALLOCATION_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert graph.PROJECTION_NODE not in compiled.predecessors[graph.DONOR_NODE] + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls(preparation, allocated, clone, host_pins=pins, n_estimators=2) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(3): + artifacts.extend( + [ + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ] + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + donor_weights = observed[graph.DONOR_NODE].frame.resolve_weights("person") + assert donor_weights.kind is WeightKind.DESIGN + np.testing.assert_array_equal( + donor_weights.values, + qualified.donor_frame.resolve_weights("person").values, + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + key = opaque_artifact_key(receipt.key, name) + assert receipt.opaque_artifacts[name] == key + payload = live.store.load_bytes(key) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + fit_id = f"{graph.FIT_PREFIX}.{i:03d}" + payload = loaded[fit_id, "model"] + # Actual typed producer/store verification above precedes pickle loading. + artifact = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert artifact.target == target + assert artifact._target_model.columns == ( + *values.FEATURES, + *values.TARGETS[:i], + ) + expected_regime = ( + qrf.Regime.ZERO_INFLATED_POSITIVE + if nonconstant + else qrf.Regime.DEGENERATE_ZERO + ) + assert artifact.regime == expected_regime + if nonconstant: + assert artifact._target_model.gate is not None + assert artifact._target_model.positive is not None + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + raw = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + states = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = manifest.node(graph.PROJECTION_NODE).key + actual = observed[graph.ATTACH_NODE] + verification = dict( + population=actual, + projection=loaded[graph.PROJECTION_NODE, "projection"], + matrix=loaded[graph.PROJECTION_NODE, "matrix"], + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=states, + host_pins=pins, + n_estimators=2, + ) + result = graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **verification + ) + assert ( + result["all_output_cells_available"] is True + and result["release_eligible"] is False + ) + assert not actual.frame.person.loc[:, list(values.OUTPUTS)].isna().any().any() + actual_person = actual.frame.person + for _, rows in actual_person.groupby( + values.provenance.support_source_id_column("person") + ): + assert len(rows) == 2 + np.testing.assert_array_equal( + rows.loc[:, list(values.OUTPUTS)].iloc[0], + rows.loc[:, list(values.OUTPUTS)].iloc[1], + ) + for raw_name in ("WAGP", "SEMP", "ADJINC"): + pd.testing.assert_series_equal( + actual_person[raw_name], clone.frame.person[raw_name] + ) + puf_mask = support.puf_tax_detail_clone_mask( + actual.frame.table("tax_unit"), entity="tax_unit" + ) + features, _ = support._strict_recipient_predictor_surface( + actual.frame, + puf_mask, + support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS, + person_outputs=(), + ) + assert features.shape[1] == 8 and np.isfinite(features.to_numpy()).all() + changed = _copy_population(actual) + changed.frame.person.loc[changed.frame.person.index[0], values.OUTPUTS[2]] += ( + 1.0 + ) + with pytest.raises(ValueError): + graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **{**verification, "population": changed} + ) + with pytest.raises(ValueError, match="DRAW_MATRIX_BINDING"): + graph.read_current_survey_draws(qualified.matrix, "f" * 64, raw, states) + results.append(actual) + graph.replay.same_replayed_population(*results) diff --git a/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py new file mode 100644 index 000000000..53fa59024 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_full_puf_output_profiles.py @@ -0,0 +1,1118 @@ +"""Ordinary profile controls with invented donors and real graph/QRF execution. + +The helper fixtures retain their explicit lack of source admission. These tests +run ordinary maintained fit/draw/finalizer and Population/store/replay paths; +they do not replace source qualification or an independent production gate. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import ( + InventedMatrix, + InventedSource, + _donor_columns, + _known, +) +from test_us_graph_full_puf_enrichment import ( + InventedCompleteBoundary, + _complete_frame, + _copy_population, + _load_artifact, +) +from test_us_graph_full_puf_enrichment import ( + attached_full65 as original_attached_full65, +) + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.population import dtype_for_token + +SCF_MORTGAGE_OUTPUTS = ( + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", +) +PUF59_TAX_UNIT_OUTPUTS = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald", +) + +# Register the existing ordinary fixture unchanged, without invoking its body. +attached_full65 = original_attached_full65 + + +def _puf59_columns(): + person, tax_unit, person_known, tax_unit_known = _donor_columns() + # Invented upstream leaves, not a source measurement implementation or receipt. + tax_unit[full.PUF59.predictors[0]] = [1.0, 2.0, 3.0, 4.0] + tax_unit[full.PUF59.predictors[1]] = [3.0, 5.0, 2.0, 4.0] + for name in full.PUF59.predictors[:2]: + tax_unit_known[name] = True + return ( + person, + tax_unit.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + person_known, + tax_unit_known.drop(columns=list(SCF_MORTGAGE_OUTPUTS)), + ) + + +def _puf59_donor(): + person, tax_unit, person_known, tax_unit_known = _puf59_columns() + return full.canonical_full_puf_donor( + person, + tax_unit, + person_known=person_known, + tax_unit_known=tax_unit_known, + profile=full.PUF59, + ) + + +def _puf59_return_only_columns(): + """Invented return incidence bridge; no physical people or source admission.""" + reduced = _puf59_donor() + _, returns, _, _ = _puf59_columns() + for name in full.PUF59.person_outputs: + returns[name] = ( + np.array([0.0, 1.0, 1.0, 0.0]) + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + else reduced[name] + ) + returns["puf_person_incidence_capacity"] = 1.0 + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *full.PUF59.person_outputs, + *full.PUF59.tax_unit_outputs, + *full.PUF59.predictors[:2], + ), + ) + return returns, known + + +def _puf59_return_only_donor(): + returns, known = _puf59_return_only_columns() + return full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def _puf59_frame(): + frame = _complete_frame() + tax_unit = frame.table("tax_unit") + tax_unit[full.PUF59.predictors[0]] = np.resize( + np.array([2.0, 4.0, 1.0, 3.0]), len(tax_unit) + ) + tax_unit[full.PUF59.predictors[1]] = np.resize( + np.array([5.0, 4.0, 3.0, 2.0]), len(tax_unit) + ) + return frame + + +def _puf59_known(frame): + return _known(frame).set_axis(full.PUF59.predictors, axis="columns") + + +class InventedProfileMatrix(InventedMatrix): + """Real ordinary projection of the declared selected predictor columns.""" + + ref = "test.puf_profile.matrix@1" + + def run(self, context): + table = context.tables["tax_unit"] + ids = table.tax_unit_id.to_numpy() + features = table.loc[:, list(context.node.inputs[0].columns)].copy() + features.index = pd.Index(ids, name="tax_unit_id") + return KernelResult( + artifacts={ + "matrix": model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=ids + ) + } + ) + + +def _finalizer_arguments(case, result): + artifacts = result.artifacts + return dict( + predictor_known=case.binding.predictor_known, + matrix=artifacts["matrix"].payload, + matrix_producer_key=artifacts["matrix"].producer_key, + raw_draws={ + target: artifacts[f"raw_{index:03d}"].payload + for index, target in enumerate(case.binding.profile.targets) + }, + apply_state=artifacts["apply_state"].payload, + training_state=artifacts["training_state"].payload, + last_model=artifacts["last_model"].payload, + seed=578, + ) + + +def _independent_puf59_population(binding, nodes, artifacts): + """Actual finalizer and graph patch, independent of attachment result code.""" + case = SimpleNamespace(binding=binding) + arguments = _finalizer_arguments(case, SimpleNamespace(artifacts=artifacts)) + candidate, receipt = full.finalize_full_puf( + binding.expected_population.frame, + binding.donor, + **arguments, + profile=full.PUF59, + ) + expected = binding.expected_population + masks = {} + for entity in ("person", "tax_unit"): + table = expected.frame.table(entity) + id_column = expected.frame.schema.entity_id_column(entity) + masks[entity] = pd.Series( + support.puf_tax_detail_clone_mask(table, entity=entity), + index=pd.Index(table[id_column], name=id_column), + dtype=bool, + ) + expected = population_ops.patch( + expected, + nodes[0], + KernelResult( + columns={ + (entity, placement.MASKS[entity]): mask + for entity, mask in masks.items() + } + ), + ) + columns = {} + for owned in nodes[1].outputs: + table = candidate.table(owned.entity) + mask = masks[owned.entity].to_numpy() + id_column = candidate.schema.entity_id_column(owned.entity) + columns[(owned.entity, owned.column)] = pd.Series( + table.loc[mask, owned.column].array, + index=pd.Index(table.loc[mask, id_column], name=id_column), + dtype=dtype_for_token(owned.dtype), + ) + return population_ops.patch( + expected, nodes[1], KernelResult(columns=columns) + ), receipt + + +@pytest.fixture(scope="module", params=("absent", "incumbent")) +def attached_puf59(tmp_path_factory, request): + root = tmp_path_factory.mktemp("puf59_" + request.param) + frame = _puf59_frame() + donor = _puf59_return_only_donor() if request.param == "absent" else _puf59_donor() + if request.param == "incumbent": + # The actual Frame contract requires global column-name uniqueness. + # These six details belong to tax units; the separate observed person + # home_mortgage_interest destination remains part of the PUF59 profile. + table = frame.table("tax_unit") + for index, name in enumerate(SCF_MORTGAGE_OUTPUTS): + # Independent incumbent bytes, including nulls and signed zero, + # survive on native AND clone rows without entering the PUF model. + table[name] = np.resize( + np.array([-0.0, np.nan, -12.5, 1800.5 + index]), len(table) + ) + assert name not in frame.person + frame.revalidate() + known = _puf59_known(frame) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + decoded = model_input.decode_recipient_matrix(prepared.matrix) + table = decoded.features.copy() + table.insert(0, "tax_unit_id", decoded.entity_ids) + matrix_frame = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": decoded.entity_ids, + "person_tax_unit_id": decoded.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.ones(len(table), dtype="float64"), WeightKind.DESIGN)}, + ) + retained_store = ContentStore( + root / "retained", codecs={"frame-store": load_frame_store} + ) + sources, nodes = {}, [] + for name, value in ( + ("survey", frame), + ("donor", prepared.donor_frame), + ("matrix_input", matrix_frame), + ): + source = "invented." + name + sources[source] = retained_store.put_frame( + full.codec.sha(("ordinary-puf59:" + request.param + ":" + name).encode()), + value, + ) + outputs = ( + frame_column_declarations(value) + if name == "survey" + else tuple( + Owned("tax_unit", name, "float64") + for name in value.table("tax_unit") + if name != "tax_unit_id" + ) + ) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(source,), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + ) + boundary = Node( + "complete_upstream", + InventedCompleteBoundary.ref, + base="survey", + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + ) + matrix_node = Node( + "matrix", + InventedProfileMatrix.ref, + population="matrix_input", + inputs=(Slice("tax_unit", full.PUF59.predictors),), + artifact_outputs=(ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),), + ) + base_graph = Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, boundary, matrix_node), + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + observed = {} + retained_manifest = run_graph( + compile_graph(base_graph), + sources=sources, + store=retained_store, + kernels=kernels, + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + upstream = observed[boundary.id] + matrix = _load_artifact( + retained_manifest, + retained_store, + ArtifactInput( + "matrix", matrix_node.id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + assert matrix.payload == prepared.matrix + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population=boundary.id, + matrix_producer=matrix_node.id, + seed=578, + n_estimators=2, + zero_atol=0, + profile=full.PUF59, + ) + retention_arguments = dict( + population=upstream, + expected_population=_copy_population(upstream), + population_node=boundary, + input_owners=dict(upstream.owners), + donor=donor, + predictor_known=known, + matrix=matrix, + fit_nodes=fits, + apply_nodes=applies, + profile=full.PUF59, + ) + binding = placement.retain_full_puf_attachment(**retention_arguments) + attachment_nodes = placement.full_puf_attachment_nodes(binding) + kernels.register(placement.FullPufMaskKernel(binding)) + kernels.register(placement.FullPufAttachKernel(binding)) + compiled = compile_graph( + replace( + base_graph, nodes=(*base_graph.nodes, *fits, *applies, *attachment_nodes) + ) + ) + store = ContentStore(root / "application", codecs={"frame-store": load_frame_store}) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=lambda name, value, target=observed: ( + target.__setitem__(name, value) + ), + ) + assert all( + record.hit is (resume == "require") for record in manifest.nodes.values() + ) + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + binding, compiled=compiled, manifest=manifest, store=store + ) + actual = observed[attachment_nodes[-1].id] + evidence = placement.verify_materialized_full_puf_attachment( + binding, + upstream_population=observed[boundary.id], + population=actual, + artifacts=artifacts, + producer_keys=producer_keys, + ) + expected, receipt = _independent_puf59_population( + binding, attachment_nodes, artifacts + ) + replay.same_replayed_population(expected, actual) + frame_key = full.codec.sha( + ( + "puf59-materialized:" + manifest.node(attachment_nodes[-1].id).key + ).encode() + ) + store.put_frame(frame_key, actual.frame) + replay.same_replayed_frame(expected.frame, store.load_frame(frame_key)) + results.append( + SimpleNamespace( + manifest=manifest, + population=_copy_population(actual), + upstream=_copy_population(observed[boundary.id]), + artifacts=artifacts, + producer_keys=producer_keys, + evidence=evidence, + finalizer_receipt=receipt, + ) + ) + replay.same_replayed_population(results[0].population, results[1].population) + assert results[0].artifacts == results[1].artifacts + assert results[0].manifest.key == results[1].manifest.key + return SimpleNamespace( + binding=binding, + retention_arguments=retention_arguments, + nodes=attachment_nodes, + fits=fits, + applies=applies, + results=results, + compiled=compiled, + sources=sources, + store=store, + kernels=kernels, + mortgage_mode=request.param, + ) + + +def test_closed_profiles_keep_observed_mortgage_and_original_conditioning_order(): + assert ( + full.FULL65.person_outputs == full.PUF59.person_outputs == full.PERSON_OUTPUTS + ) + assert full.FULL65.tax_unit_outputs == full.TAX_UNIT_OUTPUTS + assert full.FULL65.targets == full.TARGETS + assert full.PUF59.tax_unit_outputs == PUF59_TAX_UNIT_OUTPUTS + assert len(full.PUF59.person_outputs) == 56 and len(full.PUF59.targets) == 59 + assert full.PUF59.targets == tuple( + t for t in full.TARGETS if t not in SCF_MORTGAGE_OUTPUTS + ) + assert "home_mortgage_interest" in full.PUF59.person_outputs + assert not {"prior_year_wages", "employment_income_last_year"} & set( + full.PUF59.targets + ) + assert full.FULL65.predictors == full.PREDICTORS + assert full.FULL65.donor_auxiliary_columns == () + assert full.PUF59.donor_auxiliary_columns == ("puf_person_incidence_capacity",) + assert full.PUF59.predictors == ( + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", + *full.PREDICTORS[2:], + ) + + +@pytest.mark.parametrize("invalid", (None, "puf59", "full65", 59, True, ("puf59",))) +def test_profiles_require_explicit_enum_members(invalid): + with pytest.raises(ValueError, match="PUF_OUTPUT_PROFILE"): + full.require_puf_output_profile(invalid) + + +def test_puf59_canonical_donor_accepts_truly_absent_scf_columns_and_keeps_observed_interest(): + person, tax_unit, pk, tk = _puf59_columns() + # Actual canonical arithmetic after an invented QBI allocation: entirely + # SSTB positive, ordinary positive, entirely SSTB loss, ordinary positive. + person["self_employment_income_before_lsr"] = [ + 0.0, + 0.0, + 50.0, + 25.0, + 0.0, + 0.0, + 700.0, + 300.0, + ] + person["sstb_self_employment_income_before_lsr"] = [ + 900.0, + 100.0, + 0.0, + 0.0, + -250.0, + -50.0, + 0.0, + 0.0, + ] + before = tuple(value.copy(deep=True) for value in (person, tax_unit, pk, tk)) + donor = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert tuple(donor) == ( + *full.PUF59.predictors, + *full.PUF59.targets, + "weight", + "puf_person_incidence_capacity", + ) + assert not set(SCF_MORTGAGE_OUTPUTS) & set(donor) + np.testing.assert_array_equal( + donor.home_mortgage_interest, + person.home_mortgage_interest.groupby(person.person_tax_unit_id).sum(), + ) + assert donor.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + np.testing.assert_array_equal( + donor[full.PUF59.predictors[3]], [1000.0, 75.0, -300.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.self_employment_income_before_lsr, [0.0, 75.0, 0.0, 1000.0] + ) + np.testing.assert_array_equal( + donor.sstb_self_employment_income_before_lsr, [1000.0, 0.0, -300.0, 0.0] + ) + for actual, expected in zip((person, tax_unit, pk, tk), before, strict=True): + pd.testing.assert_frame_equal(actual, expected) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_puf59_donor_does_not_validate_excluded_years_or_consume_scf_payloads(): + person, tax_unit, pk, tk = _puf59_columns() + expected = _puf59_donor() + for name in SCF_MORTGAGE_OUTPUTS: + tax_unit[name] = ["independent SCF", None, False, 1800.5] + actual = full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + pd.testing.assert_frame_equal(actual, expected) + person["business_is_sstb"] = person.business_is_sstb.astype("float64") + with pytest.raises(ValueError, match="PUF_PHYSICAL_TYPE:person.business_is_sstb"): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + + +def test_puf59_return_only_donor_uses_explicit_leaves_without_inventing_persons(): + expected = _puf59_donor() + returns, known = _puf59_return_only_columns() + before = returns.copy(deep=True), known.copy(deep=True) + assert "tax_unit_person_count" not in returns + assert "tax_unit_person_count" not in known + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert returns[name].dtype == np.dtype("float64") + assert set(returns[name]) == {0.0, 1.0} + expected[name] = returns[name] + expected["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + pd.testing.assert_frame_equal(actual, expected) + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.PUF59.predictors[3]], + returns.self_employment_income_before_lsr + + returns.sstb_self_employment_income_before_lsr, + ) + pd.testing.assert_frame_equal(returns, before[0]) + pd.testing.assert_frame_equal(known, before[1]) + + +@pytest.mark.parametrize( + ("failure", "error"), + ( + ("absent", "PUF_DONOR_COLUMNS:tax_unit"), + ("physical_count_alias", "PUF_DONOR_COLUMNS:tax_unit"), + ("unknown", "PUF_UNKNOWN:tax_unit.puf_person_incidence_capacity"), + ("missing_knownness", "PUF_KNOWNNESS_AXIS:tax_unit"), + ), +) +def test_puf59_return_only_requires_exact_known_incidence_capacity(failure, error): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + if failure == "absent": + returns.drop(columns=name, inplace=True) + elif failure == "physical_count_alias": + returns.rename(columns={name: "tax_unit_person_count"}, inplace=True) + known.rename(columns={name: "tax_unit_person_count"}, inplace=True) + elif failure == "unknown": + known.loc[0, name] = False + else: + known.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +@pytest.mark.parametrize( + ("value", "error"), + ( + (0.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (-1.0, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + (0.5, "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN"), + ("1", "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (True, "PUF_PHYSICAL_TYPE:puf_person_incidence_capacity"), + (None, "PUF_UNKNOWN:puf_person_incidence_capacity"), + (np.inf, "PUF_NONFINITE:puf_person_incidence_capacity"), + ), +) +def test_puf59_return_only_incidence_capacity_requires_physical_positive_integer( + value, error +): + returns, known = _puf59_return_only_columns() + name = "puf_person_incidence_capacity" + returns[name] = returns[name].astype(object) + returns.loc[0, name] = value + with pytest.raises(ValueError, match=error): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + + +def test_puf59_return_only_rejects_boolean_incidence_above_source_capacity(): + returns, known = _puf59_return_only_columns() + returns.loc[0, "business_is_sstb"] = 2.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.PUF59.person_outputs, + profile=full.PUF59, + ) + donor = _puf59_return_only_donor() + donor.loc[0, "business_is_sstb"] = 2.0 + frame = _puf59_frame() + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize("predictor_index", range(8)) +def test_puf59_requires_all_eight_known_predictors(predictor_index): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + known.iloc[0, predictor_index] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + known.iloc[0, predictor_index] = True + with pytest.raises(ValueError, match="PUF_KNOWNNESS_AXIS:recipient_predictor"): + full.prepare_full_puf_inputs( + frame, + donor, + predictor_known=known.drop(columns=known.columns[predictor_index]), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("predictor_index", (0, 1)) +def test_puf59_requires_new_leaves_and_never_falls_back_to_generic_measurements( + predictor_index, +): + frame, donor = _puf59_frame(), _puf59_donor() + known = _puf59_known(frame) + before = frame.table("tax_unit").filing_status_input.copy(deep=True) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + assert prepared.profile is full.PUF59 + decoded = model_input.decode_recipient_matrix(prepared.matrix) + assert tuple(decoded.features) == full.PUF59.predictors + tax_unit = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + for name in full.PUF59.predictors[:2]: + np.testing.assert_array_equal(decoded.features[name], tax_unit.loc[mask, name]) + pd.testing.assert_series_equal(tax_unit.filing_status_input, before) + name = full.PUF59.predictors[predictor_index] + tax_unit.drop(columns=name, inplace=True) + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + # A same-named person column is also insufficient for the explicit return-grain leaf. + frame.table("person")[name] = 1.0 + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE:" + name): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=full.PUF59 + ) + person, returns, pk, tk = _puf59_columns() + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + person, + returns.drop(columns=name), + person_known=pk, + tax_unit_known=tk.drop(columns=name), + profile=full.PUF59, + ) + + +@pytest.mark.parametrize("value", (0.5, 6.0)) +def test_puf59_prepare_still_rejects_invalid_selected_boolean_counts(value): + frame, donor = _puf59_frame(), _puf59_donor() + donor.loc[0, "business_is_sstb"] = value + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + ("predictor_index", "value", "error"), + ( + (0, "1", "PUF_PHYSICAL_TYPE"), + (1, "2", "PUF_PHYSICAL_TYPE"), + (0, True, "PUF_PHYSICAL_TYPE"), + (1, False, "PUF_PHYSICAL_TYPE"), + (0, None, "missing values before coercion"), + (1, np.inf, "PUF_NONFINITE"), + (0, 5.0, "PUF_PROFILE_FILING_STATUS_DOMAIN"), + (1, 1.5, "PUF_PROFILE_RETURN_SIZE_DOMAIN"), + ), +) +def test_puf59_measured_predictor_leaves_require_physical_values_and_domains( + predictor_index, value, error +): + frame, donor = _puf59_frame(), _puf59_donor() + tax_unit = frame.table("tax_unit") + selected = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + name = full.PUF59.predictors[predictor_index] + tax_unit[name] = tax_unit[name].astype(object) + tax_unit.loc[tax_unit.index[selected][0], name] = value + with pytest.raises(ValueError, match=error): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +def test_puf59_person_donor_derives_incidence_capacity_from_actual_membership(): + person, returns, pk, tk = _puf59_columns() + person.loc[:1, "business_is_sstb"] = True + returns.loc[0, full.PUF59.predictors[1]] = 1.0 + # The bound is derived from membership here, not read as a return source fact. + returns["puf_person_incidence_capacity"] = "not a source fact" + returns["tax_unit_person_count"] = "not a source fact" + donor = full.canonical_full_puf_donor( + person, returns, person_known=pk, tax_unit_known=tk, profile=full.PUF59 + ) + assert ( + donor.loc[0, "business_is_sstb"] + == donor.loc[0, "puf_person_incidence_capacity"] + == 2 + ) + assert donor.loc[0, full.PUF59.predictors[1]] == 1 + frame = _puf59_frame() + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + for auxiliary in ("puf_person_incidence_capacity", "tax_unit_person_count"): + assert auxiliary not in prepared.donor + assert auxiliary not in prepared.donor_frame.table("tax_unit") + donor.loc[0, "puf_person_incidence_capacity"] = 1.0 + with pytest.raises(ValueError, match="PUF_BOOLEAN_COUNT_DOMAIN:business_is_sstb"): + full.prepare_full_puf_inputs( + frame, donor, predictor_known=_puf59_known(frame), profile=full.PUF59 + ) + + +@pytest.mark.parametrize( + "year", + ("first_home_mortgage_origination_year", "second_home_mortgage_origination_year"), +) +def test_default_full65_retains_selected_discrete_year_checks(year): + person, tax_unit, pk, tk = _donor_columns() + tax_unit.loc[0, year] = 2010.5 + with pytest.raises(ValueError, match="PUF_YEAR_DOMAIN:" + year): + full.canonical_full_puf_donor( + person, tax_unit, person_known=pk, tax_unit_known=tk + ) + + +def test_default_full65_return_only_retains_legacy_person_count_contract(): + person, returns, person_known, tax_unit_known = _donor_columns() + for name in full.FULL65.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + person.loc[:1, name] = True + expected = full.canonical_full_puf_donor( + person, + returns, + person_known=person_known, + tax_unit_known=tax_unit_known, + ) + returns["tax_unit_person_count"] = 2.0 + for name in full.FULL65.person_outputs: + returns[name] = expected[name] + known = pd.DataFrame( + True, + index=returns.index, + columns=( + "weight", + "filing_status_code", + "tax_unit_person_count", + *full.FULL65.person_outputs, + *full.FULL65.tax_unit_outputs, + ), + ) + assert "puf_person_incidence_capacity" not in returns + for arguments in ({}, {"profile": full.FULL65}): + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + **arguments, + ) + pd.testing.assert_frame_equal(actual, expected) + assert actual.loc[0, "business_is_sstb"] == 2.0 + # FULL65 compatibility is explicit: its historical alias excludes the + # nonzero SSTB component rather than silently changing old evidence. + assert bool((returns.sstb_self_employment_income_before_lsr != 0).any()) + np.testing.assert_array_equal( + actual[full.FULL65.predictors[3]], returns.self_employment_income_before_lsr + ) + returns["puf_person_incidence_capacity"] = 1.0 + actual = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + pd.testing.assert_frame_equal(actual, expected) + returns.drop(columns="tax_unit_person_count", inplace=True) + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=full.FULL65.person_outputs, + ) + + +def test_real_puf59_chain_finalization_attachment_and_required_replay(attached_puf59): + case = attached_puf59 + assert case.binding.profile is full.PUF59 + assert len(case.fits) == len(case.applies) == 59 + assert [ + edge.name + for edge in case.applies[-1].artifact_inputs + if edge.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(58)] + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.PUF59.targets + for node in (*case.fits, *case.applies): + assert node.params["phase"] == full.PUF59.phase + assert not { + *SCF_MORTGAGE_OUTPUTS, + "puf_person_incidence_capacity", + "tax_unit_person_count", + } & {column for item in node.inputs for column in item.columns} + assert all(node.params["predictors"] == full.PUF59.predictors for node in case.fits) + assert all(node.params["targets"] == full.PUF59.targets for node in case.fits) + assert "raw_058" in case.results[0].artifacts + assert "raw_059" not in case.results[0].artifacts + if case.mortgage_mode == "absent": + assert case.binding.donor.puf_person_incidence_capacity.eq(1.0).all() + for name in full.PUF59.person_outputs: + if name in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS: + assert set(case.binding.donor[name]) == {0.0, 1.0} + for result in case.results: + assert result.finalizer_receipt["output_profile"] == "puf59" + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.finalizer_receipt["predictor_order"] == list( + full.PUF59.predictors + ) + assert result.finalizer_receipt["donor_auxiliary_columns"] == [ + "puf_person_incidence_capacity" + ] + assert result.finalizer_receipt["target_order"] == list(full.PUF59.targets) + assert result.finalizer_receipt["person_target_count"] == 56 + assert result.finalizer_receipt["tax_unit_target_count"] == 3 + assert result.finalizer_receipt["target_count"] == 59 + assert result.evidence["output_profile"] == "puf59" + assert result.evidence["source_admission_issued"] is False + assert result.evidence["release_eligible"] is False + for entity in ("person", "tax_unit"): + before, after = ( + result.upstream.frame.table(entity), + result.population.frame.table(entity), + ) + for name in SCF_MORTGAGE_OUTPUTS: + if case.mortgage_mode == "absent" or entity != "tax_unit": + assert name not in before and name not in after + else: + assert population_ops.storage_equal( + before[name], after[name], np.ones(len(before), dtype=bool) + ) + assert ( + result.population.owners[(entity, name)] + == result.upstream.owners[(entity, name)] + ) + for owned in case.nodes[-1].outputs: + before = result.upstream.frame.table(owned.entity) + after = result.population.frame.table(owned.entity) + mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) + assert after.loc[mask, owned.column].notna().all() + if owned.column in before: + assert population_ops.storage_equal( + before[owned.column], after[owned.column], ~mask + ) + else: + assert after.loc[~mask, owned.column].isna().all() + + +def test_puf59_incidence_capacity_remains_bound_to_attachment(attached_puf59): + case = attached_puf59 + donor = case.binding.donor.copy(deep=True) + donor["puf_person_incidence_capacity"] += 1.0 + stale = replace(case.binding, donor=donor) + with pytest.raises(ValueError, match="FULL_PUF_DONOR_CHANGED"): + placement.full_puf_attachment_nodes(stale) + rebound = placement.retain_full_puf_attachment( + **{**case.retention_arguments, "donor": donor} + ) + assert rebound.fit_nodes == case.binding.fit_nodes + assert rebound.apply_nodes == case.binding.apply_nodes + assert rebound.matrix == case.binding.matrix + nodes = placement.full_puf_attachment_nodes(rebound) + for original, changed in zip(case.nodes, nodes, strict=True): + assert original.params["donor_values"] != changed.params["donor_values"] + + +def test_default_full65_matches_explicit_full65_and_runs_actual_complete_chain( + attached_full65, +): + case = attached_full65 + kwargs = dict( + donor_population="donor", + recipient_population="complete_upstream", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + ) + default = full.full_puf_train_apply_nodes(**kwargs) + explicit = full.full_puf_train_apply_nodes(**kwargs, profile=full.FULL65) + assert default == explicit == (case.fits, case.applies) + assert len(case.fits) == len(case.applies) == 65 + assert tuple(owned.column for owned in case.nodes[-1].outputs) == full.TARGETS + assert case.binding.profile is full.FULL65 + assert set(SCF_MORTGAGE_OUTPUTS) <= set(full.FULL65.tax_unit_outputs) + for result in case.results: + assert "raw_064" in result.artifacts + assert result.finalizer_receipt["output_profile"] == "full65" + assert result.finalizer_receipt["target_order"] == list(full.TARGETS) + + +def test_actual_wrong_profile_chains_and_manifests_are_refused( + attached_puf59, attached_full65 +): + for correct, wrong in ( + (attached_puf59, attached_full65), + (attached_full65, attached_puf59), + ): + with pytest.raises(ValueError, match="FULL_PUF_CHAIN_ROSTER"): + placement.retain_full_puf_attachment( + **{**correct.retention_arguments, "profile": wrong.binding.profile} + ) + result = correct.results[-1] + arguments = _finalizer_arguments(correct, result) + arguments.pop("predictor_known") + arguments.pop("last_model") + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**arguments, profile=wrong.binding.profile) + # Feed the real other-profile compiled graph, manifest and ContentStore; + # no fabricated manifest, source issuer or artifact readiness is involved. + with pytest.raises(ValueError, match="FULL_PUF_"): + placement.load_full_puf_attachment_artifacts( + correct.binding, + compiled=wrong.compiled, + manifest=wrong.results[-1].manifest, + store=wrong.store, + ) + + +def test_puf59_wrong_profile_model_and_trimmed_full65_history_are_refused( + attached_puf59, attached_full65 +): + case, result = attached_puf59, attached_puf59.results[-1] + arguments = _finalizer_arguments(case, result) + arguments["last_model"] = ( + attached_full65.results[-1].artifacts["last_model"].payload + ) + with pytest.raises(ValueError, match="Legacy QRF artifact content digest mismatch"): + full.finalize_full_puf( + case.binding.expected_population.frame, + case.binding.donor, + **arguments, + profile=full.PUF59, + ) + full_arguments = _finalizer_arguments(attached_full65, attached_full65.results[-1]) + full_arguments.pop("predictor_known") + full_arguments.pop("last_model") + full_arguments["raw_draws"] = { + target: full_arguments["raw_draws"][target] for target in full.PUF59.targets + } + with pytest.raises(ValueError, match="PUF_"): + full.decode_full_puf_draws(**full_arguments, profile=full.PUF59) + + +def test_puf59_rebuilt_binding_survives_nullable_frame_store_normalization( + attached_puf59, tmp_path +): + case = attached_puf59 + retained = _copy_population(case.binding.expected_population) + nullable = retained.frame.table("person").unrelated_nullable.array + missing = nullable._mask.copy() + assert missing.any() + nullable._data[missing] = True + arguments = {**case.retention_arguments, "expected_population": retained} + physical_binding = placement.retain_full_puf_attachment(**arguments) + assert placement.full_puf_attachment_nodes(physical_binding) == case.nodes + + directory = tmp_path / "puf59-new-session" + key = full.codec.sha(b"invented-puf59-retained-upstream") + ContentStore(directory).put_frame(key, retained.frame) + # Fresh ContentStore decoding performs the actual v2 masked-byte normalization. + restored_frame = ContentStore(directory).load_frame(key) + assert ( + not restored_frame.table("person").unrelated_nullable.array._data[missing].any() + ) + replay.same_replayed_frame(retained.frame, restored_frame) + restored = replace(_copy_population(retained), frame=restored_frame) + upstream = _copy_population(case.results[-1].upstream) + rebuilt = placement.retain_full_puf_attachment( + **{ + **arguments, + "population": upstream, + "expected_population": restored, + "input_owners": dict(upstream.owners), + } + ) + assert physical_binding.expected_stamp != rebuilt.expected_stamp + nodes = placement.full_puf_attachment_nodes(rebuilt) + assert nodes == case.nodes + replacements = {node.id: node for node in nodes} + compiled = compile_graph( + replace( + case.compiled.graph, + nodes=tuple( + replacements.get(node.id, node) for node in case.compiled.graph.nodes + ), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + placement.FullPufMaskKernel(rebuilt), + placement.FullPufAttachKernel(rebuilt), + ): + registry.register(kernel) + observed = {} + manifest = run_graph( + compiled, + sources=case.sources, + store=case.store, + kernels=registry, + resume="require", + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + assert all(record.hit for record in manifest.nodes.values()) + assert manifest.key == case.results[-1].manifest.key + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + rebuilt, compiled=compiled, manifest=manifest, store=case.store + ) + evidence = placement.verify_materialized_full_puf_attachment( + rebuilt, + upstream_population=observed[rebuilt.population_node.id], + population=observed[nodes[-1].id], + artifacts=artifacts, + producer_keys=producer_keys, + ) + assert evidence["output_profile"] == "puf59" + replay.same_replayed_population(case.results[-1].population, observed[nodes[-1].id]) diff --git a/packages/microcosm-build/tests/test_us_graph_survey_population.py b/packages/microcosm-build/tests/test_us_graph_survey_population.py index 212518d4d..76d10506b 100644 --- a/packages/microcosm-build/tests/test_us_graph_survey_population.py +++ b/packages/microcosm-build/tests/test_us_graph_survey_population.py @@ -314,12 +314,12 @@ def test_allocation_transport_streams_canonical_exact_rows_and_refuses_before_ov graph._allocation_payload(instructions, **kwargs) -def authenticated_arguments(tmp_path, monkeypatch): +def authenticated_arguments(tmp_path, monkeypatch, **fixture_kwargs): # This helper creates closed, privately pinned invented source bytes and # executes the actual ASEC parent, both catalogues and both native issuers. from test_us_survey_population_preparation import fixture - arguments = fixture(tmp_path, monkeypatch) + arguments = fixture(tmp_path, monkeypatch, **fixture_kwargs) return {**arguments, "store_root": tmp_path / "graph-store"} diff --git a/packages/microcosm-build/tests/test_us_puf59_canonical.py b/packages/microcosm-build/tests/test_us_puf59_canonical.py new file mode 100644 index 000000000..3585e4ff2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf59_canonical.py @@ -0,0 +1,517 @@ +"""Invented source/control coverage; no native files or tax engine.""" + + +def test_puf59_canonical_source_and_artifact_controls(): + import hashlib + from dataclasses import replace + + import numpy as np + import pandas as pd + + from microcosm.build.us_runtime import full_puf_enrichment as enrichment + from microcosm.build.us_runtime import puf59_canonical as canonical + from microcosm.build.us_runtime import puf59_canonical_artifact as artifact + from microcosm.build.us_runtime import puf_full_source as full + from microcosm.build.us_runtime import puf_interest_components as interest + from microcosm.build.us_runtime import puf_raw_source as raw + from microcosm.build.us_runtime import puf_target2024_growth as growth + + checks = [] + + def check(name, value): + assert bool(value), name + checks.append(name) + + def refuses(name, fn, code): + try: + fn() + except ValueError as e: + check(name, str(e).startswith(code)) + else: + raise AssertionError(name + " accepted") + + def fixture(n=2048, override=None): + definition = raw.packaged_definition() + rows = [] + for i in range(n + 4): + agg = i >= n + recid = 999996 + i - n if agg else 501000 + i + row = {k: "0" for k in definition.main.delivered_header} + mars = 0 if agg else 1 + i % 4 + secondary = int(mars == 2) + xfpt = int(not agg and i % 13 != 0) + row.update( + RECID=str(recid), + FLPDYR="2014" if i % 7 == 0 else "2015", + FLPDMO="12", + MARS=str(mars), + DSI="0", + S006=str(125 + i % 200), + XFPT=str(xfpt), + XFST=str(secondary), + XTOT=str(xfpt + secondary), + ) + if not agg: + row.update( + E00200=str(40000 + i), + E00900=str(-300 if i % 5 == 0 else 2000 + i), + E00600="120", + E00650="30", + E25940="700", + E25980="500", + E25920="100", + E25960="50", + E26110="20", + E26170="200", + E26190="300", + E26160="10", + E26180="20", + E26100="5", + E26270="1495", + E00100=str(500 * i - 10000), + E19200=str(1000 + i), + E02400="50", + E03230="100", + E87530="300", + ) + else: + row["E19200"] = "" + if override and i == 0: + row.update(override) + rows.append(row) + mh = definition.main.delivered_header + dh = definition.demographic.delivered_header + main = ( + ",".join(mh) + + "\n" + + "\n".join(",".join(row[k] for k in mh) for row in rows) + + "\n" + ).encode() + d = dict( + RECID="501000", + AGEDP1="0", + AGEDP2="0", + AGEDP3="0", + AGERANGE="2", + EARNSPLIT="0", + GENDER="1", + ) + demo = (",".join(dh) + "\n" + ",".join(d[k] for k in dh) + "\n").encode() + doc = raw.definition_document_json(definition) + doc.update(route="test_fixture", authority="invented_fixture_nonauthority") + for name, payload, count in (("main", main, n + 4), ("demographic", demo, 1)): + doc["sources"][name].update( + bytes=len(payload), + data_records=count, + sha256=hashlib.sha256(payload).hexdigest(), + git_blob_sha1=hashlib.sha1( + b"blob " + str(len(payload)).encode() + b"\0" + payload + ).hexdigest(), + ) + return full.decode_full_puf_source(main, demo, raw.fixture_definition(doc)) + + asset = interest.puf_e19200_interest_components_asset_identity() + check( + "exact_public_interest_asset", + asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256, + ) + + def build(source, **kwargs): + return canonical.construct_canonical_puf59( + source, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=asset["asset_sha256"], + **kwargs, + ) + + small = fixture(64) + small_result = build(small) + check( + "small64_actual_raw_to59", + len(small_result.recids) == 64 and len(small_result.columns) == 59, + ) + source = fixture() + before = full.encode_full_puf_source(source) + result = build(source) + check( + "larger2048_actual_raw_to59", + len(result.recids) == 2048 and len(result.columns) == 59, + ) + check("all_source_bytes_unchanged", full.encode_full_puf_source(source) == before) + check( + "all_ordinary_ids_and_missing_demographics_retained", + np.array_equal(result.recids, source.status["RECID"][source.ordinary]) + and len(result.recids) == 2048, + ) + check( + "source_s006_divided_once", + np.array_equal( + result.design_weight, + source.status["S006"][source.ordinary].astype(float) / 100, + ), + ) + check( + "capacity_is_constant_one_not_return_size", + np.all(result.person_incidence_capacity == 1) + and np.any(result.source_predictors["puf_2015_capped_return_size"] > 1), + ) + check( + "no_six_mortgage_destinations_or_invented_persons", + not any( + k.startswith(("first_home_mortgage_", "second_home_mortgage_")) + for k in result.columns + ) + and not hasattr(result, "person"), + ) + check( + "raw_non2015_fields_not_repriced_again", + result.receipt["source_statistical_year"] == 2015 + and result.receipt["growth"]["no_extra_raw_row_year_cpi"], + ) + selected = tuple(int(x) for x in result.recids[:64]) + replay = build( + source, + selected_recids=selected, + qbi_employment_calibration=result.qbi_calibration, + ) + check( + "full_cohort_fit_prefix_replays_all59", + all( + np.array_equal(replay.columns[k], v[:64]) for k, v in result.columns.items() + ), + ) + cpi = build( + source, + qbi_employment_calibration=result.qbi_calibration, + growth_scheme="cpi_only", + ) + check( + "cpi_sensitivity_preserves_ids_weight_capacity", + np.array_equal(cpi.recids, result.recids) + and np.array_equal(cpi.design_weight, result.design_weight) + and np.array_equal( + cpi.person_incidence_capacity, result.person_incidence_capacity + ), + ) + out, aux, status = full.observed_and_derived_return_columns(source) + modeled, receipt = canonical.baseline_modeled_columns( + aux, + n=2048, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=asset["asset_sha256"], + ) + check( + "all11_baseline_modeled_outputs_explicit", + len(modeled) == 11 and receipt["origin"] == "modeled_not_observed", + ) + check( + "interest_decomposition_bit_conserves_e19200", + np.array_equal( + modeled["home_mortgage_interest"] + modeled["investment_interest_expense"], + aux["raw_total_interest_deduction"].astype(float), + ), + ) + check( + "ss_carrier_preserves_observed_total", + np.array_equal( + sum( + modeled[k] + for k in ( + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + ) + ), + aux["raw_total_social_security"], + ), + ) + check( + "tuition_proxy_is_explicit_maximum", + np.all(modeled["qualified_tuition_expenses"] == 300), + ) + refuses( + "negative_interest_is_not_clipped", + lambda: build(fixture(64, {"E19200": "-1"})), + "PUF59_MODEL_NEGATIVE_SOURCE:", + ) + refuses( + "wrong_interest_asset_identity_refused", + lambda: canonical.baseline_modeled_columns( + aux, + n=2048, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256="0" * 64, + ), + "PUF59_INTEREST_ASSET_IDENTITY", + ) + bad = list(interest.US_PUF_E19200_AGI_BANDS) + bad[3] = replace(bad[3], lower_bound=bad[3].lower_bound + 1) + refuses( + "band_gap_refused", + lambda: canonical.baseline_modeled_columns( + aux, n=2048, interest_bands=bad, interest_asset_sha256=asset["asset_sha256"] + ), + "PUF59_INTEREST_CONTIGUOUS", + ) + refuses( + "aggregate_selection_refused", + lambda: build(source, selected_recids=(999996,)), + "FULL_SELECTION_UNIVERSE", + ) + payload = artifact.encode_canonical_puf59(result) + arrays, receipt = artifact.decode_canonical_puf59(payload) + check( + "canonical59_typed_envelope_byte_replay", + artifact.reencode_canonical_puf59(arrays, receipt) == payload, + ) + check( + "canonical59_typed_envelope_all_values_replay", + all(np.array_equal(arrays[k], v) for k, v in result.columns.items()), + ) + changed = payload[:-1] + bytes([payload[-1] ^ 1]) + refuses( + "canonical59_body_corruption_refused", + lambda: artifact.decode_canonical_puf59(changed), + "PUF59_ARTIFACT_BODY_IDENTITY", + ) + refuses( + "canonical59_truncation_refused", + lambda: artifact.decode_canonical_puf59(payload[:-8]), + "PUF59_ARTIFACT_BODY_IDENTITY", + ) + refuses( + "canonical59_unknown_magic_refused", + lambda: artifact.decode_canonical_puf59(b"BADMAGIC" + payload[8:]), + "PUF59_ARTIFACT_MAGIC", + ) + bad_arrays = { + **arrays, + "puf_person_incidence_capacity": np.full(2048, 2, dtype=np.int64), + } + refuses( + "canonical59_incidence_capacity_not_person_count", + lambda: artifact.reencode_canonical_puf59(bad_arrays, receipt), + "PUF59_ARTIFACT_CAPACITY", + ) + try: + arrays["employment_income_before_lsr"][0] = 2 + except ValueError: + checks.append("canonical59_decoded_arrays_immutable") + else: + raise AssertionError("decoded arrays mutable") + tax_unit = pd.DataFrame( + { + "tax_unit_id": arrays["RECID"], + "weight": arrays["weight"], + "filing_status_code": arrays["puf_2015_filing_status_code"], + "puf_person_incidence_capacity": arrays["puf_person_incidence_capacity"], + **{k: arrays[k] for k in enrichment.PUF59.targets}, + **{k: arrays[k] for k in enrichment.PUF59.source_predictors}, + } + ) + known = pd.DataFrame( + True, + index=tax_unit.index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *enrichment.PUF59.person_outputs, + *enrichment.PUF59.tax_unit_outputs, + *enrichment.PUF59.source_predictors, + ), + ) + donor = enrichment.canonical_full_puf_donor( + None, + tax_unit, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=enrichment.PUF59.person_outputs, + profile=enrichment.PUF59, + ) + check( + "actual_profile59_canonical_donor_ordered_contract", + tuple(donor.columns) + == ( + *enrichment.PUF59.predictors, + *enrichment.PUF59.targets, + "weight", + "puf_person_incidence_capacity", + ), + ) + check( + "actual_profile59_conditioning_uses_total_schedule_c", + np.array_equal( + donor[enrichment.PUF59.predictors[3]].to_numpy(), + arrays["self_employment_income_before_lsr"] + + arrays["sstb_self_employment_income_before_lsr"], + ), + ) + check( + "actual_profile59_retains_exact_source_size_measurement", + np.array_equal( + donor.puf_2015_capped_return_size.to_numpy(), + result.source_predictors["puf_2015_capped_return_size"], + ), + ) + check( + "actual_profile59_weight_unchanged", + np.array_equal(donor.weight.to_numpy(), result.design_weight), + ) + + # Integrity successor: domain-valid mutations cannot inherit old provenance. + import json + import struct + + check( + "canonical_v2_and_exact_band_fingerprint", + result.receipt["schema"] == canonical.VERSION + and result.receipt["model_assumptions"]["interest_band_facts_sha256"] + == canonical.INTEREST_BAND_FACTS_SHA256, + ) + for field, replacement in ( + ("RECID", arrays["RECID"][::-1]), + ("weight", arrays["weight"] * 2), + ("puf_2015_filing_status_code", arrays["puf_2015_filing_status_code"] % 4 + 1), + ("puf_2015_capped_return_size", np.ones(2048, dtype=np.int64)), + ): + refuses( + "valid_domain_prefix_mutation_refused_" + field, + lambda field=field, replacement=replacement: ( + artifact.reencode_canonical_puf59( + {**arrays, field: replacement}, receipt + ) + ), + "PUF59_ARTIFACT_PREFIX_BINDING", + ) + for field in ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "business_is_sstb", + ): + changed_array = arrays[field].copy() + if field == "business_is_sstb": + changed_array = 1 - changed_array + else: + changed_array = changed_array[::-1] + refuses( + "valid_domain_output_mutation_refused_" + field, + lambda field=field, changed_array=changed_array: ( + artifact.reencode_canonical_puf59( + {**arrays, field: changed_array}, receipt + ) + ), + "PUF59_ARTIFACT_GROWTH_BINDING", + ) + prefix = len(artifact.MAGIC) + 4 + header_length = struct.unpack(" None: + # A fresh interpreter catches the partially initialized module path that a + # previously collected sibling test can otherwise hide in sys.modules. + script = """ +import importlib +import sys + +importlib.import_module('microcosm.build.us_runtime.' + sys.argv[1]) +pool = importlib.import_module('microcosm.build.us_runtime.multispine_pool') +agreement = importlib.import_module('microcosm.build.us_runtime.spine_agreement') +assert pool.default_spine_agreement_registry is agreement.default_spine_agreement_registry +assert pool.spine_agreement_gate is agreement.spine_agreement_gate +assert pool.POOL_SPINE_AGREEMENT_REGISTRY is not None +assert agreement.US_SPINE_AGREEMENT_REGISTRY is not None +assert callable(pool.pool_remaining_stage_input_manifest) +assert callable(pool.pool_remaining_stage_input_manifest_receipt) +""" + completed = subprocess.run( + [sys.executable, "-I", "-c", script, first], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/packages/microcosm-build/tests/test_us_survey_population_preparation.py b/packages/microcosm-build/tests/test_us_survey_population_preparation.py index b03ebefb8..3715c3db2 100644 --- a/packages/microcosm-build/tests/test_us_survey_population_preparation.py +++ b/packages/microcosm-build/tests/test_us_survey_population_preparation.py @@ -32,6 +32,7 @@ def fixture( zero=True, zero_key=None, missing_asec_money=None, + current_predictor_money=None, ): asec_root = tmp_path / "asec-original" asec_root.mkdir() @@ -40,6 +41,7 @@ def fixture( monkeypatch, extra_household=True, missing_money=missing_asec_money, + current_predictor_money=current_predictor_money, tokens=("2", "1", "2", "1", "2", "1"), household_rows=None if zero and zero_key is None diff --git a/pyproject.toml b/pyproject.toml index f7ebf3c6c..fd8df0613 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ testpaths = ["packages"] # importlib mode lets shards share test-file basenames (each has its own # test_compat.py) without the prepend-mode rootdir collision. addopts = "-q --import-mode=importlib" +# Build tests share named fixture helpers across CI shards. This test-only +# path also works for installed-wheel runs without exposing any package/src. +pythonpath = ["packages/microcosm-build/tests"] markers = [ "slow: end-to-end tests that require large local source artifacts", "requires_us: needs policyengine-us (the us extra); auto-skipped when absent", From 78d84f9c44ab8e8209cd52a2c912138afe2b6e25 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 12:49:43 -0400 Subject: [PATCH 021/255] Specify declarative geography with shared operators --- docs/geography-assignment.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index b6bee7cf5..ea9e6938a 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -9,6 +9,32 @@ separate declared population operation with explicit identities and weights. This document records the intended contract and the implementation gap. It does not certify a new geographic source, assignment run or population file. +## Declarative country capability + +Geography is a declarative country capability backed by shared graph operators. +A country declares its atomic area type, code system and vintage, source mapping +artifacts, observed-geography constraints and sampling-weight convention. The UK +declaration can select an area system by nation. Adding a country should not +require writing a country-specific assignment or derivation kernel. + +The shared assignment operator selects the atomic area using the declared +support and stable household identity. Shared lookup/join operations derive +larger geographies from identified mappings. If an input already has a qualified +atomic area, validate and retain it. Source acquisition and normalization may +still require publisher-specific adapters; they produce the common support and +mapping contracts, rather than owning a separate assignment algorithm. + +Sampling and derivation remain executable graph operations with typed inputs, +outputs and replay identity. A country configuration does not itself execute +them. Mapping edges retain their relation type, including exact nesting or an +explicit best-fit convention. Prefer existing graph lookup/join primitives where +they support these contracts; introduce only the shared behavior they lack. + +The existing country-spec geography declaration is a starting point, but its +legacy clone-and-assign contract and country-specific runtime references do not +yet implement this shared atomic-area contract. Preserve those compatibility +paths while the shared operators and country adapters receive their own checks. + ```mermaid flowchart LR S[Combined survey spine] --> A[Assign one small area per household] From adba8035c6b93c5ec3a9e623ad2fa41c10e9e022 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 13:55:31 -0400 Subject: [PATCH 022/255] Fix stored JSON corruption and grouped calibration validation --- ...s-launch-store-grouped-validation.fixed.md | 1 + docs/geography-assignment.md | 10 ++ docs/us-launch-review.md | 33 ++++++- .../src/microcosm/calibrate/solve.py | 40 ++++---- .../tests/test_group_bounds.py | 77 +++++++++++++++ .../src/microcosm/graph/store.py | 26 ++++- .../tests/test_frame_metadata_store.py | 97 +++++++++++++++++++ 7 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 changelog.d/us-launch-store-grouped-validation.fixed.md diff --git a/changelog.d/us-launch-store-grouped-validation.fixed.md b/changelog.d/us-launch-store-grouped-validation.fixed.md new file mode 100644 index 000000000..c341c76e2 --- /dev/null +++ b/changelog.d/us-launch-store-grouped-validation.fixed.md @@ -0,0 +1 @@ +Reject nonfinite stored JSON as a store-corruption error and normalize the deprecated calibration method alias before validating grouped constraints. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index ea9e6938a..785311696 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -69,6 +69,16 @@ approximation, with its source and vintage. A mapping convention cannot be presented as observed household location. See the [ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). +For US congressional districts, the Census Bureau's CD119 block equivalency +file assigns whole original 2020 tabulation blocks to districts. Colorado's +block `080010096072000` crosses the enacted CD07/CD08 boundary but is assigned +to CD08 for tabulation. Represent this as an official tabulation mapping, +not exact spatial containment. The same source distinguishes original 2020 +blocks from subsequently adjusted geometry and includes undefined `ZZ` areas. +See the [Census Bureau's CD119 documentation](https://www.census.gov/geographies/mapping-files/2025/dec/rdo/119-congressional-district-bef.html). +By contrast, the same-vintage tract-to-PUMA relation is nested, as described in +the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/guidance/geo-areas/pumas.html). + ## Required behavior - Respect observed source constraints, such as ACS PUMA and FRS region. Record diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 1e5036a14..b0b491dea 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -68,7 +68,13 @@ Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. Fable's earlier replay review closed seven findings. A separate review of the published consolidation identified the checkpoint test gap and additional -grouped-bound, metadata-store and facade coverage work, which remains open. +grouped-bound, metadata-store and facade coverage work. The portable grouped-bound, +fixed-support and metadata-store selection now passes all 117 tests with no skips. +The fixes reject nonfinite stored JSON as `StoreCorrupt` and normalize the deprecated +`apg` alias before grouped-mode validation. Source/control, resource and provider +bytes match before, after and in an external postcheck. Receipt: +`3674af9d610417897539da6eae766726d0f582ae6dd3bc2c3e550fefa45bab68`. +The historical platform fixture and complete facade import coverage remain separate. Environment-specific prepatch byte fixtures do not establish portable CI byte parity. Existing v1 graph stores remain preserved; v2 execution uses a new store rather than silently promoting old frames. @@ -76,6 +82,16 @@ rather than silently promoting old frames. Private runtime records and generated population artifacts stay outside this source PR. Receipt identifiers are audit references, not reproducibility inputs. +The composed 156-case test run against source commit `8157300` at documentation +head `78d84f9` passed 152 cases and failed four current-survey cases with +`PRODUCER_CHANGED`. Its execution guard also recorded unexpected test-bytecode +and installed-model cache-directory probes. Source/control, model-source and +resource bytes remained unchanged. This is a failed integration run, not a +replacement for the earlier passing component evidence. Its receipt is +`67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. +Broader CI also identifies missing helpers and stale schema, seed and runtime +classification expectations. These failures remain open. + ## Component evidence and remaining work Separate component work has verified the national Census joint-geography source @@ -85,6 +101,21 @@ four disclosure records. A corrected SCF wage-code interpretation passed source preparation and model-mechanism checks. Those results belong to their respective component revisions; they do not establish a calibrated result on this branch. +The corrected canonical-donor codec v2 was rebuilt from the same accepted typed +PUF source. All 207,692 returns and 59 outputs passed selected-cohort replay and +byte/value serialization checks in 24.37 seconds, using 1.95 GB peak RSS. The 481 +source/control files and four resources matched their before/after/current pins. +Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. +This run constructs the donor; it does not fit or place it onto survey recipients. + +The proposed identity encoder passed two 40-case preflights and four 40-case +timing runs in both execution orders. The 36 tiny invented cases retain exact +identity bytes and mutation checks. Nine of the 72 case/trial comparisons were +slower, including the small Python-string Frame in both orders; larger numeric +fixtures improved substantially. These timings do not establish genuine-build +speed. Integration must preserve the newer current-wage projection in the +published source, which was absent from the benchmark's accepted baseline. + The following work remains open: 1. Finish current survey predictors and demographic graph bindings, then connect diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index 762eccda6..93452d304 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -1797,6 +1797,28 @@ def calibrate( ``l2_lambda``, or if no targets compile (from the matrix build). """ _validate_grouped_zero_option(grouped_upper_bounds, grouped_preserve_zeros) + # Normalize and value-check method/mass before grouped-mode validation: + # grouped bounds are documented as Adam with free mass, so the deprecated + # 'apg' alias must reach that check as 'adam', and an invalid mass string + # must report itself rather than be read as a mass-conserving request. + if method == "apg": + warnings.warn( + "method='apg' is deprecated and now aliases method='adam'; result " + "options record the normalized method='adam'.", + DeprecationWarning, + stacklevel=2, + ) + method = "adam" + if method not in ("adam", "prox"): + raise ValueError( + f"Unknown method {method!r}; supported: 'adam' (Adam on log-weights), " + "'prox' (proximal gradient on raw weights, for the l1_lambda penalty), " + "and deprecated alias 'apg' -> 'adam'." + ) + if mass not in (FREE_MASS, CONSERVE_MASS): + raise ValueError( + f"mass must be {FREE_MASS!r} or {CONSERVE_MASS!r}, got {mass!r}." + ) if grouped_upper_bounds is not None: _validate_grouped_mode( grouped_upper_bounds, @@ -1830,24 +1852,6 @@ def grouped_observer(payload: dict[str, object]) -> None: _post_projection_observer(payload) _check_grouped_household_ids(frame, actual_ids) - if method == "apg": - warnings.warn( - "method='apg' is deprecated and now aliases method='adam'; result " - "options record the normalized method='adam'.", - DeprecationWarning, - stacklevel=2, - ) - method = "adam" - if method not in ("adam", "prox"): - raise ValueError( - f"Unknown method {method!r}; supported: 'adam' (Adam on log-weights), " - "'prox' (proximal gradient on raw weights, for the l1_lambda penalty), " - "and deprecated alias 'apg' -> 'adam'." - ) - if mass not in (FREE_MASS, CONSERVE_MASS): - raise ValueError( - f"mass must be {FREE_MASS!r} or {CONSERVE_MASS!r}, got {mass!r}." - ) if mass_reason is not None: if mass != FREE_MASS: raise ValueError( diff --git a/packages/microcosm-calibrate/tests/test_group_bounds.py b/packages/microcosm-calibrate/tests/test_group_bounds.py index 0ad218d1d..4b73715d7 100644 --- a/packages/microcosm-calibrate/tests/test_group_bounds.py +++ b/packages/microcosm-calibrate/tests/test_group_bounds.py @@ -506,3 +506,80 @@ def alter(payload): _post_projection_observer=alter, ) assert actual.tobytes() == expected.tobytes() + + +def _grouped_run(**overrides): + options = dict(epochs=6, learning_rate=0.05, grouped_upper_bounds=_groups()) + options.update(overrides) + return calibrate(_frame(), _targets(), **options) + + +def test_public_grouped_apg_reaches_the_adam_path_and_warns(monkeypatch): + """The deprecated alias is normalized before the grouped mode check.""" + entered = [] + grouped = solve._optimize_grouped + + def recorded(*args, **kwargs): + entered.append(True) + return grouped(*args, **kwargs) + + monkeypatch.setattr(solve, "_optimize_grouped", recorded) + with pytest.warns(DeprecationWarning, match="apg"): + aliased = _grouped_run(method="apg") + plain = _grouped_run(method="adam") + assert entered == [True, True] + assert aliased.options["method"] == "adam" == plain.options["method"] + assert aliased.weights.tobytes() == plain.weights.tobytes() + assert ( + aliased.options["grouped_upper_bounds"] == plain.options["grouped_upper_bounds"] + ) + + +def test_ungrouped_apg_alias_is_unchanged_by_the_reordering(): + """Control: the ungrouped alias behaviour the move must not disturb.""" + with pytest.warns(DeprecationWarning, match="apg"): + aliased = calibrate(_frame(), _targets(), epochs=6, method="apg") + plain = calibrate(_frame(), _targets(), epochs=6, method="adam") + assert aliased.options["method"] == "adam" + assert aliased.weights.tobytes() == plain.weights.tobytes() + + +@pytest.mark.parametrize("mass", ["conserved", "Free", "", "none", None, 0, True]) +def test_public_grouped_invalid_mass_is_reported_as_mass(monkeypatch, mass): + """An unknown mass must not be silently read as a conserve-mass request.""" + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match="mass must be") as caught: + _grouped_run(epochs=2, mass=mass) + assert "grouped upper bounds" not in str(caught.value) + assert not calls + + +@pytest.mark.parametrize("method", ["sgd", "APG", "", "adam "]) +def test_public_grouped_unknown_method_is_reported_as_method(monkeypatch, method): + """An unknown method is a method error, not a grouped-mode refusal.""" + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match="Unknown method") as caught: + _grouped_run(epochs=2, method=method) + assert "grouped upper bounds" not in str(caught.value) + assert not calls + + +@pytest.mark.parametrize( + "kwargs,expected", + [ + ({"method": "prox"}, "grouped upper bounds"), + ({"mass": "conserve"}, "grouped upper bounds"), + ({"method": "apg", "max_weight_ratio": 2.0}, "grouped upper bounds"), + ({"method": "apg", "target_records": 2}, "grouped upper bounds"), + ({"method": "apg", "mass": "conserve"}, "grouped upper bounds"), + ], +) +def test_grouped_mode_constraints_survive_the_reordering(monkeypatch, kwargs, expected): + """Valid-valued but unsupported grouped modes still refuse as before.""" + calls = [] + monkeypatch.setattr(torch.optim, "Adam", lambda *a, **k: calls.append(1)) + with pytest.raises(ValueError, match=expected): + _grouped_run(epochs=2, **kwargs) + assert not calls diff --git a/packages/microcosm-graph/src/microcosm/graph/store.py b/packages/microcosm-graph/src/microcosm/graph/store.py index 5d0fa17fd..cccb197d0 100644 --- a/packages/microcosm-graph/src/microcosm/graph/store.py +++ b/packages/microcosm-graph/src/microcosm/graph/store.py @@ -194,10 +194,32 @@ def _require_key(key: str) -> str: return key +def _reject_non_finite_constant(token: str) -> float: + """Refuse the ``NaN``/``Infinity`` literals ``json`` accepts by default.""" + raise ValueError(f"Stored JSON carries the non-finite constant {token}.") + + +def _finite_json_number(token: str) -> float: + """Refuse numeric literals that overflow to an infinity (e.g. ``1e999``).""" + value = float(token) + if not math.isfinite(value): + raise ValueError(f"Stored JSON carries the non-finite number {token}.") + return value + + def _load_json_file(path: Path, *, label: str) -> Any: + # ``_canonical_json`` writes every store JSON with ``allow_nan=False``, so + # no valid payload can carry a non-finite value and the decode boundary is + # the right place to refuse one. Without these hooks a corrupt frame + # manifest survives the load and only fails later inside the canonical + # re-encode, which escapes as TypeError instead of StoreCorrupt. try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + return json.loads( + path.read_text(encoding="utf-8"), + parse_constant=_reject_non_finite_constant, + parse_float=_finite_json_number, + ) + except (OSError, UnicodeDecodeError, ValueError) as error: raise StoreCorrupt(f"Stored {label} is not readable canonical JSON.") from error diff --git a/packages/microcosm-graph/tests/test_frame_metadata_store.py b/packages/microcosm-graph/tests/test_frame_metadata_store.py index 29aaa9116..917119d77 100644 --- a/packages/microcosm-graph/tests/test_frame_metadata_store.py +++ b/packages/microcosm-graph/tests/test_frame_metadata_store.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import json import math import struct @@ -99,3 +101,98 @@ def test_v1_frame_is_unavailable_instead_of_silently_losing_metadata( def test_malformed_metadata_refuses(value): with pytest.raises(StoreCorrupt, match="metadata"): store_module._decode_frame_metadata(value) + + +def _restated_manifest(store, key, *, literal=None): + """Rewrite one stored frame manifest, keeping the store's own gate honest. + + The manifest is re-registered with its true size and SHA-256 so + :func:`_verified_meta` still runs and still passes; the refusal under test + therefore comes from the decode boundary, not from a skipped checksum. + Passing ``literal=None`` restates identical content, which is the control + proving valid bytes are unaffected. + """ + object_path = store.object_path(key) + manifest_path = object_path / "frame.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if literal is not None: + manifest["metadata"] = [ + "mapping", + [["us_spine_assembly_manifest", ["float64", "@NON_FINITE@"]]], + ] + text = json.dumps(manifest, separators=(",", ":"), sort_keys=True) + manifest_path.write_text( + text.replace('"@NON_FINITE@"', literal or ""), encoding="utf-8" + ) + meta_path = object_path / "meta.json" + meta = json.loads(meta_path.read_text(encoding="utf-8")) + record = meta["payloads"]["frame.json"] + record["sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + record["size"] = manifest_path.stat().st_size + meta_path.write_text( + json.dumps(meta, separators=(",", ":"), sort_keys=True), encoding="utf-8" + ) + return manifest_path + + +@pytest.mark.parametrize( + "literal", ["NaN", "Infinity", "-Infinity", "1e999", "-1e999", "1E9999"] +) +def test_non_finite_stored_manifest_is_store_corrupt_not_type_error(tmp_path, literal): + """A checksum-consistent but non-finite manifest keeps the store taxonomy.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("d" * 64, _frame({"source": "invented"})) + manifest_path = _restated_manifest(store, "d" * 64, literal=literal) + assert literal in manifest_path.read_text(encoding="utf-8") + with pytest.raises(StoreCorrupt, match="frame manifest") as caught: + store.load_frame("d" * 64) + # The escape this pins is a bare TypeError from the canonical re-encode. + assert type(caught.value) is StoreCorrupt + + +def test_restating_the_same_manifest_still_loads_the_identical_frame(tmp_path): + """Control: the stricter decoder does not reject any valid stored bytes.""" + store = ContentStore(tmp_path / "invented-store") + original = _frame({"source": "invented", "share": 0.25, "count": 3}) + store.put_frame("e" * 64, original) + _restated_manifest(store, "e" * 64) + restored = store.load_frame("e" * 64) + assert store_module._encode_frame_metadata( + restored.metadata + ) == store_module._encode_frame_metadata(original.metadata) + + +@pytest.mark.parametrize("literal", ["NaN", "-Infinity", "1e999"]) +def test_non_finite_object_metadata_is_store_corrupt(tmp_path, literal): + """The same boundary covers meta.json, which no payload checksum guards.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("f" * 64, _frame({"source": "invented"})) + meta_path = store.object_path("f" * 64) / "meta.json" + meta = json.loads(meta_path.read_text(encoding="utf-8")) + meta["invented_non_finite"] = "@NON_FINITE@" + meta_path.write_text( + json.dumps(meta, separators=(",", ":"), sort_keys=True).replace( + '"@NON_FINITE@"', literal + ), + encoding="utf-8", + ) + with pytest.raises(StoreCorrupt, match="canonical JSON"): + store.load_frame("f" * 64) + + +def test_non_finite_json_never_reaches_the_canonical_encoder(tmp_path, monkeypatch): + """The refusal happens at decode, before any canonical re-encode runs.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("0" * 64, _frame({"source": "invented"})) + _restated_manifest(store, "0" * 64, literal="NaN") + seen = [] + canonical = store_module._canonical_json + + def recorded(value): + seen.append(value) + return canonical(value) + + monkeypatch.setattr(store_module, "_canonical_json", recorded) + with pytest.raises(StoreCorrupt): + store.load_frame("0" * 64) + assert seen == [], "malformed JSON reached the canonical encoder" From 8b076b1fa8672725c6f41a801b0ef68aa8af005e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 14:28:23 -0400 Subject: [PATCH 023/255] Fix survey producer replay and accelerate frame identity checks --- .../us-survey-copy-and-identity.fixed.md | 1 + docs/current-survey-puf59-progress.md | 4 +- docs/us-launch-review.md | 33 +- .../us_runtime/acs_native_coverage_binding.py | 4 +- .../survey_population_preparation.py | 59 +- .../test_us_current_survey_predictors.py | 39 + .../test_us_survey_frame_identity_encoding.py | 868 ++++++++++++++++++ .../src/microcosm/frame/bundle.py | 7 + 8 files changed, 999 insertions(+), 16 deletions(-) create mode 100644 changelog.d/us-survey-copy-and-identity.fixed.md create mode 100644 packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py diff --git a/changelog.d/us-survey-copy-and-identity.fixed.md b/changelog.d/us-survey-copy-and-identity.fixed.md new file mode 100644 index 000000000..76d9520fa --- /dev/null +++ b/changelog.d/us-survey-copy-and-identity.fixed.md @@ -0,0 +1 @@ +Keep frame metadata copying and pickling from changing producer identity on Python 3.14. Refresh ACS source pins after verified formatting-only changes, and reduce per-cell survey identity encoding overhead while preserving canonical bytes, mutation checks, and the current-wage projection. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index 3d8fe71d1..d5c086f0e 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -10,4 +10,6 @@ The source projection records the distinct survey observation, income and price Social Security handling remains a launch blocker. The PUF donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. The current reconciliation code can treat unknown recipient components as zero and use equal component shares. PUF59 requires a source-qualified recipient basis, including observed ASEC component/type information and an explicit ACS completion path, before those outputs can support release acceptance. -These source tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. The integrated checkout still needs its own checks after source formatting and composition. +The current consolidated source now passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. All source, model-source and resource pins survived external postchecks. See the [review guide](us-launch-review.md) for exact receipt and exclusions. A separate seven-node age-development check exposed a diagnostic reconstruction mismatch that remains under repair. + +These tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. Genuine survey replay and complete calibration/export acceptance remain pending. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index b0b491dea..de8baf4bb 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -89,8 +89,25 @@ and installed-model cache-directory probes. Source/control, model-source and resource bytes remained unchanged. This is a failed integration run, not a replacement for the earlier passing component evidence. Its receipt is `67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. -Broader CI also identifies missing helpers and stale schema, seed and runtime -classification expectations. These failures remain open. +The successor now passes all 314 selected invented integration and identity +checks with no skips or unexpected refusals. This includes the original 156 +cases, five metadata-copy/mutation regressions and 153 encoder controls; seven +synthetic-issuance controls remain explicitly outside this selection. Shallow +copy, deepcopy and pickle previously added a class cache entry captured by +Python 3.14 annotation closures. An explicit immutable-metadata representation +avoids that mutation without weakening producer checks. Two stale ACS pins were +updated only after verifying AST equality with their accepted source revisions. +All 476 source/control, 5,983 model-source and 12 resource entries matched before, +after and in the external postcheck. The run took 253.5 seconds and 738.7 MB peak +RSS. Receipt: +`d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. + +A separate 16-case current seven-node age-development run passed 13 cases and +failed three on exact diagnostic reconstruction. The checker omitted two +solver-option fields added by the consolidated solver; its correction is being +checked before another genuine run. Broader CI also identifies missing helpers +and stale schema, seed and runtime classification expectations. These remain +separate from the passing 314-case selection. ## Component evidence and remaining work @@ -108,13 +125,14 @@ source/control files and four resources matched their before/after/current pins. Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. This run constructs the donor; it does not fit or place it onto survey recipients. -The proposed identity encoder passed two 40-case preflights and four 40-case +The identity encoder passed two 40-case preflights and four 40-case timing runs in both execution orders. The 36 tiny invented cases retain exact identity bytes and mutation checks. Nine of the 72 case/trial comparisons were slower, including the small Python-string Frame in both orders; larger numeric fixtures improved substantially. These timings do not establish genuine-build -speed. Integration must preserve the newer current-wage projection in the -published source, which was absent from the benchmark's accepted baseline. +speed. It is now integrated and passes the current 314-case selection, with the +newer current-wage projection preserved byte for byte. That projection was absent +from the benchmark's accepted baseline. The following work remains open: @@ -134,10 +152,11 @@ The following work remains open: the build must retain one assigned Census block and derive its larger geographies. Prior wages remain excluded. A source operator's existence is not complete cell coverage. -4. Resolve repeated per-cell identity encoding in the survey verifier. One genuine +4. Verify the optimized identity encoder on genuine survey replay. One genuine successor run reached persisted-artifact verification but exhausted its two-hour CPU limit. That run failed; it is not a verified build. Performance - changes must preserve the identity bytes and pass small controls first. + changes now pass the small identity controls; a new genuine run must also pass + the current seven-node calibration/diagnostics checks before acceptance. 5. Run real model evaluation, national/CD calibration, holdout checks and complete export/replay verification, followed by a dashboard bound to that exact file. Check every pruned analysis file separately. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py index 5e853585b..569fe84a8 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -30,9 +30,9 @@ # Exact accepted direct AGEP -> A_AGE and AGEP -> age implementation, and owners. # A new transform/owner version requires explicit review of this successor. _ACCEPTED = { - "acs_pums.py": "3a80468c947775826516acebb51c28fba1bd35261cc9668ca72fdafe481a6690", + "acs_pums.py": "6ecf79f0dfb0c0bc0ad0af6be5fa65bd8c2d1e1968009402e4c8dee347de70dc", "acs_inputs.py": "aa4a8aeaba63dfef2f3e04fb89de59766deb088ed7f4d290aeba0425739916da", - "acs_housing_universe_source.py": "dfc23754587523edb68fb38d050852184b9b841559f15f1adc706f77b5aa9abc", + "acs_housing_universe_source.py": "beb46a4a05a13580a868be423809a77946441dcafc3a0f57160561157e93e9a3", "acs_person_coverage_authentication.py": "475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff", } _TOKEN = object() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 6145dfc2f..0d223ecdf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -49,6 +49,7 @@ MAX_REQUEST_BYTES = 4096 MAX_PAYLOAD_BYTES = 64 * 1024**2 _MAX_SCALAR_BYTES = 1024**2 +_FRAME_FAST_STRING_CHARS = 4096 _SOURCE_ROSTER = ( "selection-request.json", "acs/csv_hus.zip", @@ -476,8 +477,9 @@ def _live(): ) result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) result["csv"] = (csv.reader, _csv.reader) - # The bounded catalogue path also uses JSONEncoder.encode and its C - # provider. Bind their actual live identities through this existing seal. + # The bounded catalogue path uses JSONEncoder.encode and its C provider; + # frame cells also use the same encode_basestring provider as _chunks. + # Bind their actual live identities through this existing seal. result["json_catalogue"] = ( json.JSONEncoder, json.encoder, @@ -500,6 +502,7 @@ def _live(): MAX_REQUEST_BYTES, MAX_PAYLOAD_BYTES, _MAX_SCALAR_BYTES, + _FRAME_FAST_STRING_CHARS, _SOURCE_ROSTER, domains.DECLARATION, domains.MAX_MEMBERS, @@ -559,6 +562,46 @@ def _cell(value): return value +def _frame_cell_encode(value, maximum=MAX_PAYLOAD_BYTES): + """Encode one normalized cell with the exact _encode preimage and limits. + + _cell admits only None, exact bool/int/str, or its own freshly constructed + ["float", finite_float.hex()] list. Keep normalization and the scalar walk + before encoding; large strings/integers retain the original slow path. + No Frame, Series, index, or normalized value is retained between calls. + """ + value = _cell(value) + kind = type(value) + if not ( + value is None + or kind is bool + or (kind is int and -(2**63) <= value < 2**64) + or (kind is str and len(value) <= _FRAME_FAST_STRING_CHARS) + or kind is list + ): + return _encode(value, maximum) + + _check_scalars(value) + if value is None: + encoded = b"null" + elif kind is bool: + encoded = b"true" if value else b"false" + elif kind is int: + # Match JSON's exact-int formatter, including its decimal digit policy. + encoded = int.__repr__(value).encode("ascii") + elif kind is str: + # _chunks selects this very provider with ensure_ascii=False. UTF-8 + # conversion still precedes the payload check, including surrogates. + # At most 6 * 4096 + 2 encoded bytes, even for all control characters. + encoded = json.encoder.encode_basestring(value).encode("utf-8") + else: + # Only _cell can construct this list: both strings are ASCII without + # JSON escapes. Keep hex spelling (especially -0.0), never decimalize. + encoded = b'["float","' + value[1].encode("ascii") + b'"]' + _require(len(encoded) <= maximum, "PAYLOAD_LIMIT") + return encoded + + def _frame_identity(frame): _require( isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, @@ -570,6 +613,10 @@ def update(value): digest.update(_encode(value)) digest.update(b"\n") + def update_cell(value): + digest.update(_frame_cell_encode(value)) + digest.update(b"\n") + update([list(frame.entities), list(frame.weighted_entities)]) for entity in frame.entities: table = frame.table(entity) @@ -590,7 +637,7 @@ def update(value): ] ) for value in table.index: - update(_cell(value)) + update_cell(value) for column in table: series = table[column] dtype = series.dtype @@ -603,7 +650,7 @@ def update(value): ] ) for value in series: - update(_cell(value)) + update_cell(value) update( [ str(frame.strata.dtype), @@ -615,9 +662,9 @@ def update(value): ] ) for value in frame.strata.index: - update(_cell(value)) + update_cell(value) for value in frame.strata: - update(_cell(value)) + update_cell(value) for entity in frame.weighted_entities: weights = frame.weights_for(entity) update( diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictors.py b/packages/microcosm-build/tests/test_us_current_survey_predictors.py index b8b51549b..d342aaaec 100644 --- a/packages/microcosm-build/tests/test_us_current_survey_predictors.py +++ b/packages/microcosm-build/tests/test_us_current_survey_predictors.py @@ -5,7 +5,9 @@ fixture establishes held-out fit quality. """ +import copy import json +import pickle from dataclasses import replace from types import MappingProxyType @@ -20,14 +22,51 @@ from microcosm.build.us_runtime import graph_current_survey_predictors as graph from microcosm.build.us_runtime import graph_survey_population as survey from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_preparation as preparation from microcosm.fit import qrf, qrf_target from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel from microcosm.frame import Frame, WeightKind +from microcosm.frame.bundle import _freeze_metadata from microcosm.graph import compile_graph, run_graph from microcosm.graph.keys import opaque_artifact_key +@pytest.mark.parametrize("operation", ["copy", "deepcopy", "pickle"]) +def test_metadata_copy_preserves_producer_identity(monkeypatch, operation): + metadata = _freeze_metadata({"origin": {"stages": ["survey", "clone"]}}) + # Exercise a cold class even if a preceding test already copied metadata. + # Python 3.14 annotation closures capture this mutable class namespace. + monkeypatch.delattr(type(metadata), "__slotnames__", raising=False) + namespace = dict(vars(type(metadata))) + producer = preparation._live() + + if operation == "pickle": + restored = pickle.loads(pickle.dumps(metadata)) + else: + restored = getattr(copy, operation)(metadata) + + assert restored == metadata + assert restored is not metadata + assert restored["origin"]["stages"] == ("survey", "clone") + with pytest.raises(TypeError): + restored["origin"]["stages"] = () + assert dict(vars(type(metadata))) == namespace + assert preparation._live() == producer + + +@pytest.mark.parametrize("method", ["__getitem__", "__reduce__"]) +def test_metadata_method_replacement_still_invalidates_producer(monkeypatch, method): + metadata = _freeze_metadata({"origin": "survey"}) + producer = preparation._live() + monkeypatch.setattr(type(metadata), method, lambda *args: None) + assert preparation._live() != producer + with pytest.raises( + preparation.SurveyPopulationPreparationError, match="PRODUCER_CHANGED" + ): + preparation._producer() + + def test_survey_source_ids_preserve_zero_without_relaxing_physical_or_unique_axis(): original = pd.Series([0, 2, 5], dtype="int64") result = values._ids(original) diff --git a/packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py b/packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py new file mode 100644 index 000000000..e1db605b7 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_frame_identity_encoding.py @@ -0,0 +1,868 @@ +"""Frame identity byte parity, refusal behavior, and mutation detection. + +Fixtures are invented and never acquire survey files. The legacy oracle retains +an independent implementation of the previous frame traversal. +""" + +from __future__ import annotations + +import datetime +import hashlib +import sys +import weakref +from decimal import Decimal +from fractions import Fraction +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import survey_population_preparation as owner +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord, WeightKind, Weights + + +def _legacy_cell(value, maximum=owner.MAX_PAYLOAD_BYTES): + return owner._encode(owner._cell(value), maximum) + + +def _outcome(call): + """Compare bytes or the exact exception type/message, including precedence.""" + try: + return ("value", call()) + except Exception as error: + return ("error", type(error), str(error)) + + +@pytest.mark.parametrize( + "value", + [ + None, + pd.NA, + False, + True, + -(2**63) - 1, + -(2**63), + -1, + 0, + 1, + 2**63 - 1, + 2**63, + 2**64 - 1, + 2**64, + 10**100, + 0.0, + -0.0, + 1.0, + -1.25, + float.fromhex("0x0.0000000000001p-1022"), + float.fromhex("0x1.fffffffffffffp+1023"), + float("nan"), + -float("nan"), + "", + 'quote" backslash\\ slash/', + "\x00\b\t\n\f\r\x1f\x7f", + "é e\u0301 中文 🧪 \u2028\u2029", + "null true false 17 -0.0", + "x" * 4095, + "x" * 4096, + "x" * 4097, + "\x00" * 4096, + "🧪" * 4096, + ], +) +def test_scalar_exact_bytes_and_payload_boundaries(value): + expected = _legacy_cell(value) + assert owner._frame_cell_encode(value) == expected + for maximum in (-1, 0, 1, len(expected) - 1, len(expected), len(expected) + 1): + assert _outcome( + lambda maximum=maximum: owner._frame_cell_encode(value, maximum) + ) == _outcome(lambda maximum=maximum: _legacy_cell(value, maximum)) + + +@pytest.mark.parametrize("dtype", [np.int8, np.int16, np.int32, np.int64]) +def test_native_signed_integer_extremes(dtype): + bounds = np.iinfo(dtype) + for raw in (bounds.min, -1, 0, 1, bounds.max): + value = dtype(raw) + assert owner._frame_cell_encode(value) == _legacy_cell(value) + + +@pytest.mark.parametrize("dtype", [np.uint8, np.uint16, np.uint32, np.uint64]) +def test_native_unsigned_integer_extremes(dtype): + for raw in (0, 1, np.iinfo(dtype).max): + value = dtype(raw) + assert owner._frame_cell_encode(value) == _legacy_cell(value) + + +@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64, np.longdouble]) +def test_numpy_floating_conversion_and_platform_wide_refusals(dtype): + # longdouble.item() can remain a NumPy scalar. The existing _cell decides + # admission; neither the candidate nor this test coerces it to Python float. + for raw in ("0", "-0", "1.25", "nan", "inf", "-inf"): + value = dtype(raw) + assert _outcome( + lambda value=value: owner._frame_cell_encode(value) + ) == _outcome(lambda value=value: _legacy_cell(value)) + for raw in (np.finfo(dtype).smallest_subnormal, np.finfo(dtype).max): + value = dtype(raw) + assert _outcome( + lambda value=value: owner._frame_cell_encode(value) + ) == _outcome(lambda value=value: _legacy_cell(value)) + + +def test_signed_zero_and_missing_spellings_are_pinned(): + assert owner._frame_cell_encode(0.0) == b'["float","0x0.0p+0"]' + assert owner._frame_cell_encode(-0.0) == b'["float","-0x0.0p+0"]' + assert owner._frame_cell_encode(0.0) != owner._frame_cell_encode(-0.0) + nan_payloads = np.array( + [0x7FF8000000000001, 0x7FF8000000000002, 0xFFF8000000000001], + dtype=np.uint64, + ).view(np.float64) + for value in [None, pd.NA, float("nan"), *nan_payloads]: + assert owner._frame_cell_encode(value) == b"null" + assert owner._frame_cell_encode(False) == b"false" + assert owner._frame_cell_encode(0) == b"0" + assert owner._frame_cell_encode("0") == b'"0"' + + +@pytest.mark.parametrize( + "value", + [ + np.bool_(True), + np.bool_(False), + np.str_("é\n🧪"), + np.bytes_(b"invented"), + np.complex64(1 + 2j), + np.complex128(1 + 2j), + np.datetime64("2001-02-03", "D"), + np.datetime64("2001-02-03", "ns"), + np.datetime64("NaT"), + np.timedelta64(2, "D"), + np.timedelta64(2, "ns"), + np.void(b"invented"), + np.array(1), + np.array([1]), + pd.NaT, + pd.Timestamp("2001-02-03"), + datetime.date(2001, 2, 3), + b"invented", + bytearray(b"invented"), + Decimal("1.25"), + Fraction(1, 2), + 1 + 2j, + ["float", "0x0.0p+0"], + (1,), + {"value": 1}, + {1}, + object(), + ], +) +def test_numpy_and_unsupported_scalar_admission_matches_legacy(value): + assert _outcome(lambda value=value: owner._frame_cell_encode(value)) == _outcome( + lambda value=value: _legacy_cell(value) + ) + + +def test_scalar_subclasses_remain_refused(): + class Text(str): + pass + + class Integer(int): + pass + + class Floating(float): + pass + + for value in (Text("x"), Integer(1), Floating(1.25)): + for encode in (_legacy_cell, owner._frame_cell_encode): + with pytest.raises( + owner.SurveyPopulationPreparationError, match="^FRAME_CELL_TYPE$" + ): + encode(value) + # The predecessor's early missing check also accepts a float-subclass NaN. + assert owner._frame_cell_encode(Floating("nan")) == _legacy_cell(Floating("nan")) + + +@pytest.mark.parametrize( + "value, maximum, error_type, code", + [ + (object(), -1, owner.SurveyPopulationPreparationError, "FRAME_CELL_TYPE"), + (float("inf"), -1, owner.SurveyPopulationPreparationError, "FRAME_NONFINITE"), + (-float("inf"), -1, owner.SurveyPopulationPreparationError, "FRAME_NONFINITE"), + ("\ud800", 0, UnicodeEncodeError, None), + ("prefix\udfff", 0, UnicodeEncodeError, None), + ("x" * 4097 + "\ud800", 0, UnicodeEncodeError, None), + ( + "x" * (owner._MAX_SCALAR_BYTES + 1), + 0, + owner.SurveyPopulationPreparationError, + "SCALAR_LIMIT", + ), + ( + "\ud800" + "x" * owner._MAX_SCALAR_BYTES, + 0, + owner.SurveyPopulationPreparationError, + "SCALAR_LIMIT", + ), + ("x", 0, owner.SurveyPopulationPreparationError, "PAYLOAD_LIMIT"), + ], +) +def test_refusal_type_code_and_precedence(value, maximum, error_type, code): + expected = _outcome(lambda maximum=maximum: _legacy_cell(value, maximum)) + assert expected[0:2] == ("error", error_type) + if code is not None: + assert expected[2] == code + assert ( + _outcome(lambda maximum=maximum: owner._frame_cell_encode(value, maximum)) + == expected + ) + + +def test_scalar_check_retained_before_fast_encoding(monkeypatch): + monkeypatch.setattr(owner, "_MAX_SCALAR_BYTES", 4) + for value in ("12345", 1.25, -0.0): + expected = _outcome(lambda value=value: _legacy_cell(value, 0)) + assert expected == ( + "error", + owner.SurveyPopulationPreparationError, + "SCALAR_LIMIT", + ) + assert ( + _outcome(lambda value=value: owner._frame_cell_encode(value, 0)) == expected + ) + + +def test_large_integer_decimal_policy_is_legacy_fallback(): + limit = sys.get_int_max_str_digits() + if limit == 0: + pytest.skip("Interpreter decimal conversion limit is disabled") + if limit > 10_000: + pytest.skip("Decimal refusal case exceeds the invented 10000-digit cap") + value = 10**limit + expected = _outcome(lambda value=value: _legacy_cell(value, 0)) + assert expected[0:2] == ("error", ValueError) + assert _outcome(lambda value=value: owner._frame_cell_encode(value, 0)) == expected + + +def test_large_scalar_fallback_remains_bounded_and_called(monkeypatch): + original = owner._encode + calls = [] + + def record(value, maximum=owner.MAX_PAYLOAD_BYTES): + calls.append((value, maximum)) + return original(value, maximum) + + monkeypatch.setattr(owner, "_encode", record) + for value in ("x" * 4097, -(2**63) - 1, 2**64): + assert owner._frame_cell_encode(value) == original(owner._cell(value)) + assert calls[-1] == (value, owner.MAX_PAYLOAD_BYTES) + assert len(calls) == 3 + for value in (None, True, -1, 2**64 - 1, "x" * 4096, -0.0): + assert owner._frame_cell_encode(value) == original(owner._cell(value)) + assert len(calls) == 3 + + +@pytest.mark.parametrize("length_delta", [-1, 0, 1]) +def test_maximum_scalar_length_fallback_equality(length_delta): + value = "x" * (owner._MAX_SCALAR_BYTES + length_delta) + expected = _outcome(lambda value=value: _legacy_cell(value)) + if length_delta <= 0: + assert expected[0] == "value" + else: + assert expected == ( + "error", + owner.SurveyPopulationPreparationError, + "SCALAR_LIMIT", + ) + assert _outcome(lambda value=value: owner._frame_cell_encode(value)) == expected + + +@pytest.mark.parametrize( + "change", ["helper_code", "helper_defaults", "bound", "string_provider"] +) +def test_live_producer_seal_binds_fast_encoder(change, monkeypatch): + # Restrict the owner scan to in-memory source; the refusing producer must + # short-circuit before its source-file hashing callback can run. + monkeypatch.setattr(owner, "_modules", lambda: (owner,)) + reads = [] + + def refuse_source_reads(): + reads.append(True) + pytest.fail("Changed live code must be refused before source hashing") + + monkeypatch.setattr(owner, "_code_bytes", refuse_source_reads) + original = owner._live() + monkeypatch.setattr(owner, "_LIVE", original) + if change == "helper_code": + + def changed(value, maximum=owner.MAX_PAYLOAD_BYTES): + return b"changed" + + monkeypatch.setattr(owner._frame_cell_encode, "__code__", changed.__code__) + elif change == "helper_defaults": + monkeypatch.setattr( + owner._frame_cell_encode, + "__defaults__", + (owner.MAX_PAYLOAD_BYTES - 1,), + ) + elif change == "bound": + monkeypatch.setattr( + owner, "_FRAME_FAST_STRING_CHARS", owner._FRAME_FAST_STRING_CHARS - 1 + ) + else: + monkeypatch.setattr(owner.json.encoder, "encode_basestring", object()) + assert owner._live() != original + with pytest.raises( + owner.SurveyPopulationPreparationError, match="^PRODUCER_CHANGED$" + ): + owner._producer() + assert reads == [] + + +def _frame(): + index = pd.Index([101, 103, 107], dtype="int64", name="invented_row") + person = pd.DataFrame( + { + "person_id": np.array([1, 2, 3], dtype=np.int64), + "native_integer": np.array([-9, 0, 11], dtype=np.int32), + "native_unsigned": np.array([0, 2**63, 2**64 - 1], dtype=np.uint64), + "native_boolean": np.array([True, False, True], dtype=np.bool_), + "native_float": np.array([-0.0, np.nan, 1.25], dtype=np.float64), + "nullable_integer": pd.array([11, pd.NA, -7], dtype="Int64"), + "nullable_boolean": pd.array([True, pd.NA, False], dtype="boolean"), + "nullable_float": pd.array([-0.0, pd.NA, 1.25], dtype="Float64"), + "text": pd.array( + ["", "é\x00🧪", pd.NA], dtype=pd.StringDtype(storage="python") + ), + "object_value": np.array([None, True, "invented\nlabel"], dtype=object), + }, + index=index, + ) + tables = {US_SCHEMA.person_entity: person} + for ordinal, group in enumerate(US_SCHEMA.group_entities, 1): + first, second = 10 * ordinal, 10 * ordinal + 1 + person[US_SCHEMA.membership_column(group)] = np.array( + [first, first, second], dtype=np.int64 + ) + tables[group] = pd.DataFrame( + {US_SCHEMA.entity_id_column(group): [first, second]} + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([-0.0, 2.5]), WeightKind.DESIGN)}, + pd.Series( + ["invented_a", "invented_a", "invented_b"], + index=index, + name="stratum", + dtype=object, + ), + mass_log=(MassChangeRecord("household", 1.25, 2.5, 2.0, "invented"),), + metadata={"us_spine_assembly_manifest": {"invented_marker": "initial"}}, + ) + + +def _rebuild(frame, **changes): + arguments = { + "tables": {entity: frame.table(entity) for entity in frame.entities}, + "schema": frame.schema, + "weights": { + entity: frame.weights_for(entity) for entity in frame.weighted_entities + }, + "strata": frame.strata, + "mass_log": frame.mass_log, + "metadata": frame.metadata, + } + arguments.update(changes) + return Frame(**arguments) + + +def _legacy_frame_preimage(frame): + """Explicit predecessor traversal, including every newline and raw weight byte.""" + owner._require( + isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, + "FRAME_TYPE", + ) + chunks = [] + + def append(value): + chunks.extend((owner._encode(value), b"\n")) + + append([list(frame.entities), list(frame.weighted_entities)]) + for entity in frame.entities: + table = frame.table(entity) + owner._require( + type(table) is pd.DataFrame and not table.columns.has_duplicates, + "FRAME_TABLE", + ) + append( + [ + entity, + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ] + ) + for value in table.index: + append(owner._cell(value)) + for column in table: + series = table[column] + dtype = series.dtype + append( + [ + column, + str(dtype), + getattr(dtype, "storage", None), + str(getattr(dtype, "na_value", "")), + ] + ) + for value in series: + append(owner._cell(value)) + append( + [ + str(frame.strata.dtype), + type(frame.strata.index).__name__, + str(frame.strata.index.dtype), + list(frame.strata.index.names), + getattr(frame.strata.dtype, "storage", None), + frame.strata.name, + ] + ) + for value in frame.strata.index: + append(owner._cell(value)) + for value in frame.strata: + append(owner._cell(value)) + for entity in frame.weighted_entities: + weights = frame.weights_for(entity) + append( + [ + entity, + weights.kind.value, + str(weights.values.dtype), + list(weights.values.shape), + ] + ) + chunks.append(weights.values.tobytes()) + chunks.append(owner.graph_context.encode_us_frame_context(frame)) + return b"".join(chunks) + + +def _assert_exact_frame_preimage(frame, monkeypatch): + expected = _legacy_frame_preimage(frame) + original_sha256 = hashlib.sha256 + expected_digest = original_sha256(expected).hexdigest() + captures = [] + + class RecordingSHA: + def __init__(self, *args, **kwargs): + self.digest = original_sha256(*args, **kwargs) + self.parts = [args[0]] if args else [] + captures.append(self) + + def update(self, data): + self.parts.append(bytes(data)) + self.digest.update(data) + + def hexdigest(self): + return self.digest.hexdigest() + + # graph_context also calls hashlib; its separate seeded hashers are recorded + # independently. The frame hasher is always the first, unseeded instance. + with monkeypatch.context() as local: + local.setattr(owner.hashlib, "sha256", RecordingSHA) + actual = owner._frame_identity(frame) + assert actual == expected_digest + assert b"".join(captures[0].parts) == expected + + +def test_complete_frame_exact_preimage(monkeypatch): + _assert_exact_frame_preimage(_frame(), monkeypatch) + + +def test_empty_table_and_strata_preimage_after_mutation(monkeypatch): + frame = _frame() + original = owner._frame_identity(frame) + # Frame construction requires nonempty weights. This deliberately corrupts + # an invented borrow to test the identity traversal's empty-loop semantics; + # it makes no claim that the mutated object passes Frame.revalidate(). + for entity in frame.entities: + frame._tables[entity] = frame.table(entity).iloc[:0].copy() + frame._strata = frame.strata.iloc[:0].copy() + assert owner._frame_identity(frame) != original + _assert_exact_frame_preimage(frame, monkeypatch) + + +def test_multiple_weighted_entities_and_weight_order(monkeypatch): + frame = _frame() + weights = { + "person": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.DESIGN), + "household": frame.weights_for("household"), + } + multiple = _rebuild(frame, weights=weights) + _assert_exact_frame_preimage(multiple, monkeypatch) + assert owner._frame_identity(multiple) != owner._frame_identity(frame) + reversed_weights = _rebuild(frame, weights=dict(reversed(list(weights.items())))) + _assert_exact_frame_preimage(reversed_weights, monkeypatch) + assert multiple.weighted_entities == reversed_weights.weighted_entities + assert owner._frame_identity(reversed_weights) == owner._frame_identity(multiple) + + +@pytest.mark.parametrize( + "dtype, values", + [ + ("int8", [-128, 0, 127]), + ("int16", [-32768, 0, 32767]), + ("int32", [-(2**31), 0, 2**31 - 1]), + ("int64", [-(2**63), 0, 2**63 - 1]), + ("uint8", [0, 1, 255]), + ("uint16", [0, 1, 65535]), + ("uint32", [0, 1, 2**32 - 1]), + ("uint64", [0, 2**63, 2**64 - 1]), + ("float16", [-0.0, np.nan, 1.25]), + ("float32", [-0.0, np.nan, 1.25]), + ("float64", [-0.0, np.nan, 1.25]), + ("Int8", [-128, pd.NA, 127]), + ("Int16", [-32768, pd.NA, 32767]), + ("Int32", [-(2**31), pd.NA, 2**31 - 1]), + ("Int64", [-(2**63), pd.NA, 2**63 - 1]), + ("UInt8", [0, pd.NA, 255]), + ("UInt16", [0, pd.NA, 65535]), + ("UInt32", [0, pd.NA, 2**32 - 1]), + ("UInt64", [0, pd.NA, 2**64 - 1]), + ("Float32", [-0.0, pd.NA, 1.25]), + ("Float64", [-0.0, pd.NA, 1.25]), + ("boolean", [True, pd.NA, False]), + ("category", ["invented", None, "é"]), + ("object", [2**80, "x" * 4097, -0.0]), + ], +) +def test_representative_column_dtypes_exact_preimage(dtype, values, monkeypatch): + frame = _frame() + frame.person["invented_variant"] = pd.array(values, dtype=dtype) + _assert_exact_frame_preimage(frame, monkeypatch) + + +@pytest.mark.parametrize("storage", ["python", "pyarrow"]) +def test_string_storage_descriptor_and_values_are_preserved(storage, monkeypatch): + if storage == "pyarrow": + pytest.importorskip("pyarrow") + frame = _frame() + frame.person["text"] = pd.array( + ["a", pd.NA, "é\x00🧪"], dtype=pd.StringDtype(storage=storage) + ) + _assert_exact_frame_preimage(frame, monkeypatch) + + +@pytest.mark.parametrize( + "index", + [ + pd.RangeIndex(3, name="range"), + pd.Index([9, 7, 5], dtype="int32", name="integer"), + pd.Index([0, 2**63, 2**64 - 1], dtype="uint64", name="unsigned"), + pd.Index([-0.0, np.nan, 1.25], dtype="float64", name="float"), + pd.Index(["", "é", "\x00🧪"], dtype=object, name="text"), + pd.Index([11, pd.NA, 13], dtype="Int64", name="nullable"), + pd.CategoricalIndex(["a", "b", "a"], name="category"), + ], +) +def test_person_and_strata_index_preimages(index, monkeypatch): + frame = _frame() + # Mutate both borrowed objects deliberately; the identity owner does not + # normalize index order, index type, missing labels, or duplicate labels. + frame.person.index = index.copy() + frame.strata.index = index.copy() + _assert_exact_frame_preimage(frame, monkeypatch) + + +@pytest.mark.parametrize( + "change", + [ + "value", + "dtype", + "index_value", + "index_name", + "column_order", + "column_name", + "column_index_dtype", + "strata_value", + "strata_name", + "strata_index", + "strata_dtype", + ], +) +def test_each_repeated_call_rechecks_mutable_frame(change): + frame = _frame() + original = owner._frame_identity(frame) + assert owner._frame_identity(frame) == original + if change == "value": + frame.person.loc[101, "native_integer"] = 17 + elif change == "dtype": + frame.person["native_integer"] = frame.person["native_integer"].astype("int64") + elif change == "index_value": + frame.person.index = pd.Index([109, 103, 107], name="invented_row") + elif change == "index_name": + frame.person.index.name = "changed" + elif change == "column_order": + frame.person.insert(0, "moved", frame.person.pop("native_integer")) + frame.person.rename(columns={"moved": "native_integer"}, inplace=True) + elif change == "column_name": + frame.person.columns.name = "changed" + elif change == "column_index_dtype": + frame.person.columns = pd.Index( + frame.person.columns, dtype=pd.StringDtype(storage="python") + ) + elif change == "strata_value": + frame.strata.iloc[0] = "changed" + elif change == "strata_name": + frame.strata.name = "changed" + elif change == "strata_index": + frame.strata.index = pd.Index([109, 103, 107], name="invented_row") + else: + frame._strata = frame.strata.astype(pd.StringDtype(storage="python")) + changed = owner._frame_identity(frame) + assert changed != original + assert changed == hashlib.sha256(_legacy_frame_preimage(frame)).hexdigest() + assert owner._frame_identity(frame) == changed + + +def test_same_object_mutate_restore_mutate_has_no_identity_cache(): + frame = _frame() + original = owner._frame_identity(frame) + for value in (17, 19, 23): + frame.person.loc[101, "native_integer"] = value + assert owner._frame_identity(frame) != original + frame.person.loc[101, "native_integer"] = -9 + assert owner._frame_identity(frame) == original + + +def test_raw_weight_signed_zero_kind_metadata_and_mass_context(monkeypatch): + frame = _frame() + original = owner._frame_identity(frame) + variants = [ + _rebuild( + frame, + weights={"household": Weights(np.array([0.0, 2.5]), WeightKind.DESIGN)}, + ), + _rebuild( + frame, + weights={"household": Weights(np.array([-0.0, 2.75]), WeightKind.DESIGN)}, + ), + _rebuild( + frame, + weights={ + "household": Weights(np.array([-0.0, 2.5]), WeightKind.IMPORTANCE) + }, + ), + _rebuild( + frame, + metadata={"us_spine_assembly_manifest": {"invented_marker": "changed"}}, + ), + _rebuild( + frame, mass_log=(MassChangeRecord("household", 1.25, 2.5, 2.0, "changed"),) + ), + ] + for changed in variants: + assert owner._frame_identity(changed) != original + _assert_exact_frame_preimage(changed, monkeypatch) + values = frame.weights_for("household").values + values.setflags(write=True) + try: + values[1] = 3.0 + finally: + values.setflags(write=False) + assert owner._frame_identity(frame) != original + + +@pytest.mark.parametrize("place", ["index", "column", "strata_index", "strata"]) +def test_repeated_identity_refuses_new_unsupported_cell(place): + frame = _frame() + owner._frame_identity(frame) + if place == "index": + frame.person.index = pd.Index([b"bad", 103, 107], dtype=object) + elif place == "column": + frame.person["object_value"] = [b"bad", "fine", None] + elif place == "strata_index": + frame.strata.index = pd.Index([b"bad", 103, 107], dtype=object) + else: + frame.strata.iloc[0] = b"bad" + expected = _outcome(lambda: _legacy_frame_preimage(frame)) + assert expected == ( + "error", + owner.SurveyPopulationPreparationError, + "FRAME_CELL_TYPE", + ) + assert _outcome(lambda: owner._frame_identity(frame)) == expected + + +def test_table_and_traversal_error_precedence(): + frame = _frame() + frame.person["bad_later"] = [float("inf"), 1.0, 2.0] + frame.person.index = pd.Index([b"bad_first", 103, 107], dtype=object) + expected = _outcome(lambda: _legacy_frame_preimage(frame)) + assert expected == ( + "error", + owner.SurveyPopulationPreparationError, + "FRAME_CELL_TYPE", + ) + assert _outcome(lambda: owner._frame_identity(frame)) == expected + frame.person.columns = ["duplicate"] * len(frame.person.columns) + expected = _outcome(lambda: _legacy_frame_preimage(frame)) + assert expected == ("error", owner.SurveyPopulationPreparationError, "FRAME_TABLE") + assert _outcome(lambda: owner._frame_identity(frame)) == expected + with pytest.raises(owner.SurveyPopulationPreparationError, match="^FRAME_TYPE$"): + owner._frame_identity(object()) + + +@pytest.mark.parametrize("early", ["surrogate", "scalar_bound"]) +@pytest.mark.parametrize("later", ["unsupported", "nonfinite"]) +def test_earlier_series_encoding_error_precedes_later_cell_error(early, later): + frame = _frame() + first = "\ud800" if early == "surrogate" else "x" * (owner._MAX_SCALAR_BYTES + 1) + second = b"unsupported" if later == "unsupported" else float("inf") + frame.person["object_value"] = pd.Series( + [first, second, None], index=frame.person.index, dtype=object + ) + expected = _outcome(lambda: _legacy_frame_preimage(frame)) + if early == "surrogate": + assert expected[0:2] == ("error", UnicodeEncodeError) + else: + assert expected == ( + "error", + owner.SurveyPopulationPreparationError, + "SCALAR_LIMIT", + ) + assert _outcome(lambda: owner._frame_identity(frame)) == expected + + +@pytest.mark.parametrize( + "change, code", + [ + ("schema", "FRAME_TYPE"), + ("links", "FRAME_TYPE"), + ("table_subclass", "FRAME_TABLE"), + ], +) +def test_frame_structure_refusals_still_precede_cells(change, code): + frame = _frame() + frame.person["bad_later"] = [float("inf"), 1.0, 2.0] + if change == "schema": + frame._schema = object() + elif change == "links": + + class LinkedFrame(Frame): + @property + def links(self): + return ("invented_link",) + + frame = LinkedFrame(frame._tables, frame.schema, frame._weights, frame.strata) + else: + + class Table(pd.DataFrame): + pass + + frame._tables[US_SCHEMA.person_entity] = Table(frame.person) + expected = ("error", owner.SurveyPopulationPreparationError, code) + assert _outcome(lambda: _legacy_frame_preimage(frame)) == expected + assert _outcome(lambda: owner._frame_identity(frame)) == expected + + +def _invented_preparation(frame, monkeypatch): + """Install an in-memory synthetic issuance solely for boundary unit tests.""" + preparation = owner.AuthenticatedSurveyPopulationPreparation( + b"{}", _token=owner._TOKEN + ) + state = SimpleNamespace(frame=frame, identity=owner._frame_identity(frame)) + entry = (weakref.ref(preparation), preparation.payload, state) + monkeypatch.setitem(owner._ISSUED, id(preparation), entry) + return preparation, state, entry + + +def test_checked_boundary_revalidates_each_borrow(monkeypatch): + frame = _frame() + preparation, state, _entry = _invented_preparation(frame, monkeypatch) + calls = [] + + def validate(candidate): + assert candidate is state + calls.append(True) + owner._require( + owner._frame_identity(frame) == state.identity, "PREPARED_FRAME_CHANGED" + ) + + monkeypatch.setattr(owner, "_validate", validate) + assert preparation.frame is frame + assert preparation.frame is frame + assert calls == [True, True] + frame.person.loc[101, "native_integer"] = 17 + with pytest.raises( + owner.SurveyPopulationPreparationError, match="^PREPARED_FRAME_CHANGED$" + ): + _ = preparation.frame + assert calls == [True, True, True] + + +@pytest.mark.parametrize("change", ["entry", "payload"]) +def test_checked_boundary_post_validation_issuance_guard(change, monkeypatch): + preparation, _state, entry = _invented_preparation(_frame(), monkeypatch) + + def validate(_candidate): + if change == "entry": + # Equal tuple contents do not satisfy the required same-entry guard. + monkeypatch.setitem(owner._ISSUED, id(preparation), tuple(list(entry))) + else: + object.__setattr__(preparation, "payload", b'{"changed":true}') + + monkeypatch.setattr(owner, "_validate", validate) + with pytest.raises( + owner.SurveyPopulationPreparationError, match="^UNISSUED_OR_CHANGED$" + ): + preparation._checked() + + +@pytest.mark.parametrize("mutation", ["none", "frame", "payload", "frame_and_payload"]) +def test_materialized_frame_two_pass_recheck_and_toctou(mutation, monkeypatch): + frame = _frame() + preparation, state, entry = _invented_preparation(frame, monkeypatch) + calls = [] + original_identity = owner._frame_identity + + def checked(candidate): + assert candidate is preparation + calls.append("checked") + return entry + + def identity(candidate): + assert candidate is frame + calls.append("identity") + return original_identity(candidate) + + def validate(candidate): + assert candidate is state + calls.append("validate") + if mutation in ("frame", "frame_and_payload"): + frame.person.loc[101, "native_integer"] = 17 + if mutation in ("payload", "frame_and_payload"): + object.__setattr__(preparation, "payload", b'{"changed":true}') + + # Isolate this API's ordered rechecks from acquisition/ancestry validation. + monkeypatch.setattr( + owner.AuthenticatedSurveyPopulationPreparation, "_checked", checked + ) + monkeypatch.setattr(owner, "_frame_identity", identity) + monkeypatch.setattr(owner, "_validate", validate) + if mutation != "none": + code = ( + "UNISSUED_OR_CHANGED" + if mutation == "payload" + else "MATERIALIZED_FRAME_CHANGED" + ) + with pytest.raises(owner.SurveyPopulationPreparationError, match=f"^{code}$"): + owner.verify_materialized_survey_population(preparation, frame) + else: + owner.verify_materialized_survey_population(preparation, frame) + assert calls == ["checked", "identity", "validate", "identity"] diff --git a/packages/microcosm-frame/src/microcosm/frame/bundle.py b/packages/microcosm-frame/src/microcosm/frame/bundle.py index e16c3a45a..875fd5c82 100644 --- a/packages/microcosm-frame/src/microcosm/frame/bundle.py +++ b/packages/microcosm-frame/src/microcosm/frame/bundle.py @@ -1361,6 +1361,13 @@ def __iter__(self): def __len__(self) -> int: return len(self._items) + def __reduce__(self): + # An explicit representation keeps copy/pickle from populating the + # class's __slotnames__ cache. Python 3.14 annotation closures capture + # that namespace, so the incidental mutation changes producer seals. + # Constructor arguments still undergo normal recursive deepcopy. + return type(self), (self._items,) + def _freeze_metadata(metadata: Mapping[str, Any] | None) -> Mapping[str, Any]: """Return a recursively immutable copy of frame metadata.""" From a9895d8a50886470f6dfe4b2b9d20c352c9871cf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 14:31:25 -0400 Subject: [PATCH 024/255] Keep survey diagnostics aligned with grouped solver options --- .../us-survey-diagnostic-options.fixed.md | 1 + docs/current-survey-puf59-progress.md | 2 +- docs/us-launch-review.md | 17 +++++++++++------ .../survey_calibration_diagnostics.py | 4 ++++ .../tests/test_us_survey_age_development.py | 2 ++ 5 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 changelog.d/us-survey-diagnostic-options.fixed.md diff --git a/changelog.d/us-survey-diagnostic-options.fixed.md b/changelog.d/us-survey-diagnostic-options.fixed.md new file mode 100644 index 000000000..e8668b7e0 --- /dev/null +++ b/changelog.d/us-survey-diagnostic-options.fixed.md @@ -0,0 +1 @@ +Reconstruct the grouped calibration solver's closing-state selection options during survey diagnostic verification, preserving exact diagnostic comparison and late-mutation checks. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index d5c086f0e..0da23af28 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -10,6 +10,6 @@ The source projection records the distinct survey observation, income and price Social Security handling remains a launch blocker. The PUF donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. The current reconciliation code can treat unknown recipient components as zero and use equal component shares. PUF59 requires a source-qualified recipient basis, including observed ASEC component/type information and an explicit ACS completion path, before those outputs can support release acceptance. -The current consolidated source now passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. All source, model-source and resource pins survived external postchecks. See the [review guide](us-launch-review.md) for exact receipt and exclusions. A separate seven-node age-development check exposed a diagnostic reconstruction mismatch that remains under repair. +The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. These tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. Genuine survey replay and complete calibration/export acceptance remain pending. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index de8baf4bb..27c909006 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -102,12 +102,17 @@ after and in the external postcheck. The run took 253.5 seconds and 738.7 MB pea RSS. Receipt: `d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. -A separate 16-case current seven-node age-development run passed 13 cases and -failed three on exact diagnostic reconstruction. The checker omitted two -solver-option fields added by the consolidated solver; its correction is being -checked before another genuine run. Broader CI also identifies missing helpers -and stale schema, seed and runtime classification expectations. These remain -separate from the passing 314-case selection. +A separate 16-case current seven-node age-development run initially passed 13 +cases and failed three on exact diagnostic reconstruction. The checker omitted +two solver-option fields added by the consolidated solver. Reconstructing the +grouped solver's closing-state selection and empty selection receipt fixes that +mismatch while retaining exact whole-document comparison. All 16 tests now pass, +including actual seven-node execution and late target/successor mutation checks, +in 170.3 seconds at 541.9 MB peak RSS. All 469 source/control, 5,983 model-source +and 12 resource entries matched before/after/current checks. Receipt: +`5849dd42e914e4d8304a5d290ff451da45910e7ac9a61e76fff36468949ffe3a`. +Broader CI's missing helpers and stale schema, seed and runtime classification +expectations remain separate from these selected passing suites. ## Component evidence and remaining work diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py index 28caa0f09..25d0bb9b7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -146,6 +146,10 @@ def validate_survey_calibration_diagnostics( "method": "adam", "epochs": epochs, "learning_rate": learning_rate, + # This profile always uses grouped bounds, whose accepted weights are + # the closing state. Ungrouped Adam's best-iterate history cannot apply. + "iterate_selection": "closing_state", + "iterate_selection_receipt": {}, "mass": "free", "mass_reason": None, "max_weight_ratio": None, diff --git a/packages/microcosm-build/tests/test_us_survey_age_development.py b/packages/microcosm-build/tests/test_us_survey_age_development.py index 7ca662a8b..337a634f1 100644 --- a/packages/microcosm-build/tests/test_us_survey_age_development.py +++ b/packages/microcosm-build/tests/test_us_survey_age_development.py @@ -177,6 +177,8 @@ def test_actual_invented_population_with_captured_age_profile_preserves_seven_no assert receipt["scope"] == "source_documented_survey_age_calibration_numbers_only" assert receipt["release_eligible"] is False assert result.diagnostics["verification"]["optimizer_rerun"] is False + assert result.diagnostics["options"]["iterate_selection"] == "closing_state" + assert result.diagnostics["options"]["iterate_selection_receipt"] == {} activation.verify_survey_age_targets( tmp_path / "age", declaration=declaration, registry=registry ) From ebb8db8c1e9f2f0422ef5f4cf5a0954bb99a4e4b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 15:22:08 -0400 Subject: [PATCH 025/255] Add shared atomic geography assignment and graph operators --- changelog.d/shared-atomic-geography.added.md | 1 + docs/geography-assignment.md | 44 ++ .../src/microcosm/build/atomic_geography.py | 590 ++++++++++++++++++ .../microcosm/build/graph_atomic_geography.py | 255 ++++++++ .../tests/test_atomic_geography.py | 443 +++++++++++++ tools/ci_test_groups.py | 5 +- 6 files changed, 1337 insertions(+), 1 deletion(-) create mode 100644 changelog.d/shared-atomic-geography.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/atomic_geography.py create mode 100644 packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py create mode 100644 packages/microcosm-build/tests/test_atomic_geography.py diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md new file mode 100644 index 000000000..a269c08ca --- /dev/null +++ b/changelog.d/shared-atomic-geography.added.md @@ -0,0 +1 @@ +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index 785311696..2d68ee34d 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -99,6 +99,50 @@ the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/gui ## Existing implementation and next changes +The shared implementation now lives in `microcosm.build.atomic_geography` and +`microcosm.build.graph_atomic_geography`. It provides four country-neutral nodes: +`geography.support_import@1`, `geography.assign_atomic@1`, +`geography.derive@1` and `geography.gate@1`. Countries supply declarations and +normalized, pinned support files. The graph interfaces and legacy country +operators are unchanged. Country configuration and native support integration +remain necessary before these nodes can assign locations in a release build. + +The support contract uses deterministic NPZ bytes with no object arrays. It has +one unique string code per atomic area; every mapping column identifies its +source, vintage and relation, and every integer sampling-weight column identifies +its source and basis. Nonnegative weights have a bounded exact total. A source +adapter must resolve any nonfunctional crosswalk before admission; the shared +lookup never multiplies households through a many-to-many join. + +Assignment conditions on all nonmissing observed constraints jointly. Declared +stages can use different weights—for example, household counts to select a +constituency followed by population counts to select its small area. Each draw +uses the existing `keyed_uniform` protocol with stable source identity, the +declared stream, system, stage, assignment definition and support digest. +Selection uses an integer inverse CDF. Reordering, subsetting or adding unrelated +households preserves draws. The numerical declaration remains `platform_bitwise`; +cross-platform equivalence has not been established. + +The version-1 declaration lists `identity`, `stream`, three assignment `outputs` +(`area`, `system`, `basis`), and `systems`. Each system supplies its atomic +identity and source, a selector, observed constraints, an optional observed-area +column, sampling stages and derived layers. Geographic codes use nullable +strings throughout, retaining leading zeroes. An engine that needs integer +storage must declare that conversion separately. A system without a particular +layer produces missing values for that layer, which the integrity gate checks. +The graph stores the canonical declaration in its normative `definition` +parameter and exposes the stream separately; no graph parameter grammar changed. + +The 34 invented-data controls include real graph execution, cold and warm reuse, +source-change invalidation, exact lookup metadata, independent sampling-boundary +checks, three-system routing, observed-area retention, row/subset stability and +clone/prune validation. All passed under the isolated control runner on +9 September 2026. These checks establish shared operator behavior; they do not +admit a Census/ONS source or certify any population's geographic fit. The gate +checks mapping integrity, while the executor and build own structural lineage +and population-quality acceptance. The country's chosen identity still needs +verification across actual sampling rungs. + The US development composed graph attaches geography after harmonization, but `us_runtime/graph_geography.py` selects a joint tract/congressional-district cell and emits PUMA, county and district. It does not assign a Census block. Its prior diff --git a/packages/microcosm-build/src/microcosm/build/atomic_geography.py b/packages/microcosm-build/src/microcosm/build/atomic_geography.py new file mode 100644 index 000000000..81ce87fb9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/atomic_geography.py @@ -0,0 +1,590 @@ +"""Country-neutral atomic geography support, assignment and functional lookups. + +Publisher adapters supply one row per atomic area. Country declarations choose +constraints, sampling stages and output names. Observed geography is never +overwritten. This module does not acquire sources or certify their authority. +""" + +from __future__ import annotations + +import bisect +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from io import BytesIO +from types import MappingProxyType +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import numpy as np +import pandas as pd + +from microcosm.graph.canonical import canonical_json +from microcosm.graph.randomness import keyed_uniform + +RELATIONS = frozenset({"exact", "best_fit", "official_tabulation", "inferred_modal"}) +_U53 = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("Atomic geography: " + message) + + +def _text(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +def _unique_json(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "duplicate metadata key") + result[key] = value + return result + + +@dataclass(frozen=True) +class AtomicSupport: + """Validated, immutable arrays decoded from a pinned support artifact.""" + + metadata: Mapping + arrays: Mapping[str, np.ndarray] + sha256: str + + +def encode_atomic_support(metadata: Mapping, arrays: Mapping[str, np.ndarray]) -> bytes: + """Create deterministic NPZ bytes; validate through the same decoder as readers.""" + _require("metadata_json" not in arrays, "reserved metadata array") + output = BytesIO() + members = {**arrays, "metadata_json": np.asarray(canonical_json(metadata).decode())} + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + for name, array in sorted(members.items()): + _require(_text(name) and name.isidentifier(), "invalid array name") + member = BytesIO() + np.lib.format.write_array(member, np.asarray(array), allow_pickle=False) + info = ZipInfo(name + ".npy", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = ZIP_DEFLATED + archive.writestr(info, member.getvalue()) + payload = output.getvalue() + decode_atomic_support(payload) + return payload + + +def decode_atomic_support(payload: bytes) -> AtomicSupport: + """Refuse ambiguous mappings, object arrays, untyped columns and invalid weights.""" + _require(type(payload) is bytes, "support requires immutable bytes") + with np.load(BytesIO(payload), allow_pickle=False) as archive: + _require(len(set(archive.files)) == len(archive.files), "duplicate array") + meta_array = archive["metadata_json"] + _require( + meta_array.shape == () and meta_array.dtype.kind == "U", "metadata encoding" + ) + metadata = json.loads(str(meta_array.item()), object_pairs_hook=_unique_json) + _require( + isinstance(metadata, dict) + and set(metadata) + == {"version", "system", "level", "code_system", "vintage", "columns"} + and type(metadata["version"]) is int + and metadata["version"] == 1, + "metadata schema", + ) + _require( + all( + _text(metadata[k]) + for k in ("system", "level", "code_system", "vintage") + ), + "area identity", + ) + columns = metadata["columns"] + _require( + isinstance(columns, dict) and "area" in columns and bool(columns), + "column metadata", + ) + _require(set(archive.files) == {"metadata_json", *columns}, "undeclared array") + arrays = {} + for name, description in columns.items(): + _require( + _text(name) and name.isidentifier() and name != "metadata_json", + "column name", + ) + _require(isinstance(description, dict), "column description") + array = archive[name] + _require( + array.ndim == 1 and len(array) > 0, "nonempty one-dimensional support" + ) + kind = description.get("kind") + if kind == "code": + _require( + set(description) == {"kind", "source", "vintage", "relation"}, + "code metadata", + ) + _require( + array.dtype.kind == "U" and np.all(np.char.str_len(array) > 0), + "nonempty string codes", + ) + _require(np.all(np.char.strip(array) == array), "whitespace in codes") + _require( + _text(description["vintage"]) + and description["relation"] in RELATIONS, + "mapping relation or vintage", + ) + else: + _require( + kind == "weight" + and set(description) == {"kind", "source", "basis"}, + "weight metadata", + ) + _require( + array.dtype.kind in "iu" and np.all(array >= 0), + "nonnegative integer weights", + ) + _require(_text(description["basis"]), "sampling basis") + # Python accumulation cannot wrap before the bound is checked. + total = sum(int(value) for value in array) + _require(0 < total <= _U53, "weight total outside exact sampling range") + array = array.astype(np.int64) + _require(_text(description["source"]), "source identity") + # Bytes-backed arrays cannot be made writable by changing flags. + arrays[name] = np.frombuffer(array.tobytes(), dtype=array.dtype) + area = arrays["area"] + _require(columns["area"]["kind"] == "code", "atomic area must be a code") + _require( + columns["area"]["vintage"] == metadata["vintage"], "atomic vintage mismatch" + ) + _require( + columns["area"]["relation"] == "exact", "atomic identity must be exact" + ) + _require( + all(len(a) == len(area) for a in arrays.values()), "array lengths differ" + ) + _require(len(np.unique(area)) == len(area), "atomic codes must be unique") + _require( + any(c["kind"] == "weight" for c in columns.values()), + "missing sampling weights", + ) + frozen = { + **metadata, + "columns": MappingProxyType( + {k: MappingProxyType(v) for k, v in columns.items()} + ), + } + return AtomicSupport( + MappingProxyType(frozen), + MappingProxyType(arrays), + hashlib.sha256(payload).hexdigest(), + ) + + +def validate_assignment_spec(spec: Mapping) -> dict: + """Normalize a graph parameter mapping without accepting undeclared options. + + All geography code columns use strings, preserving leading zeroes. A later + engine adapter may declare a separate storage conversion when needed. + """ + value = json.loads(canonical_json(spec)) + _require( + set(value) == {"version", "identity", "stream", "outputs", "systems"} + and value["version"] == 1 + and type(value["version"]) is int, + "assignment schema", + ) + identity, outputs, systems = value["identity"], value["outputs"], value["systems"] + _require( + isinstance(identity, list) + and bool(identity) + and all(_text(c) for c in identity) + and len(set(identity)) == len(identity), + "stable identity columns", + ) + _require( + isinstance(outputs, dict) + and set(outputs) == {"area", "system", "basis"} + and all(_text(c) for c in outputs.values()) + and len(set(outputs.values())) == 3, + "assignment outputs", + ) + keyed_uniform(stream=tuple(value["stream"]), keys=[]) + _require(isinstance(systems, list) and bool(systems), "area systems") + ids, layers = set(), {} + for system in systems: + _require( + isinstance(system, dict) + and set(system) + == { + "id", + "level", + "code_system", + "vintage", + "source", + "selector", + "constraints", + "observed_area", + "stages", + "layers", + }, + "system schema", + ) + _require( + all( + _text(system[k]) + for k in ("id", "level", "code_system", "vintage", "source") + ), + "system identity", + ) + _require(system["id"] not in ids, "duplicate system") + ids.add(system["id"]) + selector = system["selector"] + _require( + isinstance(selector, dict) + and all( + _text(c) + and isinstance(v, list) + and bool(v) + and all(_text(x) for x in v) + and len(set(v)) == len(v) + for c, v in selector.items() + ), + "system selector", + ) + _require( + system["observed_area"] is None or _text(system["observed_area"]), + "observed atomic column", + ) + constraints = system["constraints"] + _require(isinstance(constraints, list), "constraints") + seen = set() + for c in constraints: + _require( + isinstance(c, dict) + and set(c) == {"input", "support", "required"} + and _text(c["input"]) + and _text(c["support"]) + and type(c["required"]) is bool, + "constraint schema", + ) + _require( + c["support"] not in seen and c["support"] != "area", + "duplicate or atomic constraint", + ) + seen.add(c["support"]) + stages = system["stages"] + _require(isinstance(stages, list) and bool(stages), "sampling stages") + _require( + all( + isinstance(s, dict) + and set(s) == {"level", "weight"} + and _text(s["level"]) + and _text(s["weight"]) + for s in stages + ), + "stage schema", + ) + _require( + stages[-1]["level"] == "area" + and len({s["level"] for s in stages}) == len(stages), + "stages must end at the unique atomic level", + ) + _require(isinstance(system["layers"], list), "layers") + local_layers = set() + for layer in system["layers"]: + _require( + isinstance(layer, dict) + and set(layer) == {"input", "output", "vintage", "relation", "source"} + and all(_text(x) for x in layer.values()) + and layer["relation"] in RELATIONS, + "layer schema", + ) + name = layer["output"] + _require( + name not in outputs.values() and name not in local_layers, + "duplicate layer output", + ) + local_layers.add(name) + layers.setdefault(name, []).append(layer) + required = set(identity) + for system in systems: + required.update(system["selector"]) + required.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + required.add(system["observed_area"]) + _require( + not required.intersection({*outputs.values(), *layers}), + "observed or identity columns would be overwritten", + ) + return value + + +def _bind(spec: Mapping, supports: Mapping[str, AtomicSupport]): + spec = validate_assignment_spec(spec) + _require( + set(supports) == {s["id"] for s in spec["systems"]}, "support systems differ" + ) + for system in spec["systems"]: + support = supports[system["id"]] + _require( + all( + support.metadata[k] == system[v] + for k, v in ( + ("system", "id"), + ("level", "level"), + ("code_system", "code_system"), + ("vintage", "vintage"), + ) + ), + "support identity differs from declaration", + ) + columns = support.metadata["columns"] + for c in system["constraints"]: + _require( + c["support"] in columns and columns[c["support"]]["kind"] == "code", + "unknown constraint", + ) + for s in system["stages"]: + _require( + s["level"] in columns + and columns[s["level"]]["kind"] == "code" + and s["weight"] in columns + and columns[s["weight"]]["kind"] == "weight", + "unknown stage column", + ) + for layer in system["layers"]: + description = columns.get(layer["input"], {}) + _require( + description.get("kind") == "code" + and all( + description[k] == layer[k] + for k in ("source", "vintage", "relation") + ), + "layer source, relation or vintage differs", + ) + return spec + + +def _route(households: pd.DataFrame, spec: Mapping) -> dict[str, np.ndarray]: + masks, claimed = {}, np.zeros(len(households), dtype=np.int64) + for system in spec["systems"]: + mask = np.ones(len(households), dtype=bool) + for name, accepted in system["selector"].items(): + mask &= households[name].isin(accepted).to_numpy() + masks[system["id"]] = np.flatnonzero(mask) + claimed += mask + _require(np.all(claimed == 1), "every household must match exactly one area system") + return masks + + +def _constraints(row, system) -> tuple: + values = [] + for c in system["constraints"]: + value = row[c["input"]] + if pd.isna(value): + _require(not c["required"], "missing required observed geography") + else: + _require(_text(value), "observed codes require nonempty strings") + values.append((c["support"], value)) + return tuple(sorted(values)) + + +class _SupportIndex: + """Index each observed-column pattern once, rather than scanning per household.""" + + def __init__(self, support): + self.support = support + self.groupings = {} + self.area_index = pd.Index(support.arrays["area"]) + self.draw_cells = {} + + def rows(self, constraints): + columns = tuple(k for k, _ in constraints) + if columns not in self.groupings: + if columns: + table = pd.DataFrame({c: self.support.arrays[c] for c in columns}) + # Always group by a list: normalize pandas' single-column key below. + groups = table.groupby(list(columns), sort=False, observed=True).indices + self.groupings[columns] = { + (k,) if len(columns) == 1 and not isinstance(k, tuple) else k: v + for k, v in groups.items() + } + else: + self.groupings[columns] = {(): np.arange(len(self.area_index))} + key = tuple(v for _, v in constraints) + _require( + key in self.groupings[columns], + "observed constraints have no common support", + ) + return self.groupings[columns][key] + + def pick(self, constraints, stage, u): + key = (constraints, stage["level"], stage["weight"]) + if key not in self.draw_cells: + rows = self.rows(constraints) + levels, inverse = np.unique( + self.support.arrays[stage["level"]][rows], return_inverse=True + ) + weights = np.zeros(len(levels), dtype=np.int64) + np.add.at(weights, inverse, self.support.arrays[stage["weight"]][rows]) + cumulative = np.cumsum(weights).tolist() + _require(cumulative[-1] > 0, "constraint cell has zero sampling mass") + self.draw_cells[key] = levels, cumulative + levels, cumulative = self.draw_cells[key] + # Integer inverse CDF: no floating-point rounding across a cell boundary. + threshold = (int(u * _U53) * cumulative[-1]) // _U53 + return str(levels[bisect.bisect_right(cumulative, threshold)]) + + +def assign_atomic( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Select atomic areas without changing row order, ids or observed geography.""" + spec = _bind(spec, supports) + _require( + not set(spec["outputs"].values()).intersection(households.columns), + "assignment output already exists", + ) + _require( + not households[spec["identity"]].isna().any().any() + and not households.duplicated(spec["identity"]).any(), + "null or duplicate draw identity", + ) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in spec["outputs"].values(): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + definition = hashlib.sha256(canonical_json(spec)).hexdigest() + for system in spec["systems"]: + positions = routing[system["id"]] + support = supports[system["id"]] + index = _SupportIndex(support) + identities = list( + households.iloc[positions][spec["identity"]].itertuples( + index=False, name=None + ) + ) + draws = [ + keyed_uniform( + stream=tuple(spec["stream"]), + keys=[ + ( + *identity, + system["id"], + stage["level"], + definition, + support.sha256, + ) + for identity in identities + ], + ) + for stage in system["stages"] + ] + for local, position in enumerate(positions): + row = households.iloc[position] + constraints = _constraints(row, system) + area = row[system["observed_area"]] if system["observed_area"] else None + observed = area is not None and not pd.isna(area) + if observed: + _require(_text(area), "invalid observed atomic code") + match = index.area_index.get_indexer([area])[0] + _require( + match >= 0 + and all( + support.arrays[c][match] == value for c, value in constraints + ), + "observed atomic code is unsupported or conflicts", + ) + else: + for stage, uniforms in zip(system["stages"], draws, strict=True): + area = index.pick(constraints, stage, uniforms[local]) + constraints = tuple( + sorted({**dict(constraints), stage["level"]: area}.items()) + ) + for key, value in ( + ("area", area), + ("system", system["id"]), + ("basis", "observed" if observed else "assigned"), + ): + result.iloc[position, result.columns.get_loc(spec["outputs"][key])] = ( + value + ) + return result + + +def derive_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Functional area lookups, including explicitly identified non-nesting conventions.""" + spec = _bind(spec, supports) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in sorted( + {layer["output"] for s in spec["systems"] for layer in s["layers"]} + ): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + for system in spec["systems"]: + positions = routing[system["id"]] + rows = households.iloc[positions] + support = supports[system["id"]] + assigned_system = rows[spec["outputs"]["system"]] + _require( + assigned_system.notna().all() and assigned_system.eq(system["id"]).all(), + "assigned system disagrees with routing", + ) + locations = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + _require(np.all(locations >= 0), "missing atomic mapping") + for layer in system["layers"]: + result.iloc[positions, result.columns.get_loc(layer["output"])] = ( + support.arrays[layer["input"]][locations] + ) + return result + + +def validate_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> dict: + """Recheck constraints and functional mappings on a full or pruned population. + + Draws are not repeated: enrichment clones have new identities and must inherit + their parent's location. Structural lineage verification remains the executor's + responsibility. This gate alone is not a population-quality certificate. + """ + spec = _bind(spec, supports) + expected = derive_geography(households, spec, supports) + for name in expected: + _require( + households[name].astype("string").equals(expected[name]), + "derived geography differs from atomic mapping", + ) + routing = _route(households, spec) + for system in spec["systems"]: + support = supports[system["id"]] + rows = households.iloc[routing[system["id"]]] + positions = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + for (_, row), position in zip(rows.iterrows(), positions, strict=True): + _require( + all( + support.arrays[c][position] == v + for c, v in _constraints(row, system) + ), + "assigned area violates observed geography", + ) + original = row[system["observed_area"]] if system["observed_area"] else None + observed = original is not None and not pd.isna(original) + _require( + row[spec["outputs"]["basis"]] + == ("observed" if observed else "assigned"), + "assignment basis differs", + ) + if observed: + _require( + row[spec["outputs"]["area"]] == original, + "observed atomic area changed", + ) + return { + "outcome": "pass", + "scope": "atomic_geography_mapping_integrity", + "households": len(households), + "support_sha256": {k: v.sha256 for k, v in sorted(supports.items())}, + "definition_sha256": hashlib.sha256(canonical_json(spec)).hexdigest(), + } diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py new file mode 100644 index 000000000..b507f845d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -0,0 +1,255 @@ +"""Typed graph nodes for the shared atomic-area geography contract. + +Countries supply declarations and normalized source bytes. Import, assignment, +derivation and the integrity gate remain independently inspectable operations. +These nodes do not change weights, rows, memberships or observed columns. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence + +import pandas as pd + +from microcosm.build import atomic_geography as geography +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes +from microcosm.graph.kernel import KernelRole +from microcosm.graph.randomness import keyed_uniform + +ATOMIC_SUPPORT_TYPE = ArtifactType("microcosm.geography.atomic_support_npz", 1) +_DEPENDENCIES = ("numpy", "pandas") + + +class _GeographyKernel(KernelBase): + def implementation_hash(self) -> str: + return source_hash( + type(self), + geography, + canonical_json, + keyed_uniform, + load_source_bytes, + dependencies=_DEPENDENCIES, + ) + + +class AtomicSupportImportKernel(_GeographyKernel): + ref = "geography.support_import@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"source", "system"} or set(context.sources) != { + context.params["source"] + }: + raise ValueError( + "Atomic geography import requires its sole declared source." + ) + payload = load_source_bytes( + "raw-bytes-v1", context.sources[context.params["source"]] + ) + support = geography.decode_atomic_support(payload) + if support.metadata["system"] != context.params["system"]: + raise ValueError("Atomic geography source belongs to a different system.") + return KernelResult( + artifacts={"support": payload}, + receipt={ + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + }, + ) + + +def _inputs(context): + if set(context.params) != {"definition", "stream"}: + raise ValueError( + "Atomic geography requires its canonical definition and stream." + ) + spec = geography.validate_assignment_spec(json.loads(context.params["definition"])) + if ( + canonical_json(spec).decode() != context.params["definition"] + or tuple(spec["stream"]) != context.params["stream"] + ): + raise ValueError("Atomic geography definition or stream differs.") + if set(context.artifacts) != {s["id"] for s in spec["systems"]}: + raise ValueError( + "Atomic geography requires exactly its declared support artifacts." + ) + supports = {} + for name, value in context.artifacts.items(): + if value.type != ATOMIC_SUPPORT_TYPE: + raise ValueError("Atomic geography artifact type differs.") + supports[name] = geography.decode_atomic_support(value.payload) + return context.tables["household"], spec, supports + + +def _result(context, frame, receipt): + if {(o.entity, o.column, o.dtype) for o in context.node.outputs} != { + ("household", c, "string") for c in frame + }: + raise ValueError("Atomic geography owned outputs differ from the declaration.") + index = pd.Index(context.tables["household"]["household_id"], name="household_id") + return KernelResult( + columns={ + ("household", c): pd.Series(frame[c].array, index=index) for c in frame + }, + receipt=receipt, + ) + + +class AtomicAssignKernel(_GeographyKernel): + ref = "geography.assign_atomic@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.KEYED, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.assign_atomic(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": {k: v.sha256 for k, v in supports.items()}, + }, + ) + + +class AtomicDeriveKernel(_GeographyKernel): + ref = "geography.derive@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.derive_geography(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_functional_lookup", + "layers": {s["id"]: s["layers"] for s in spec["systems"]}, + }, + ) + + +class AtomicGeographyGateKernel(_GeographyKernel): + ref = "geography.gate@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + return KernelResult( + receipt=geography.validate_geography(households, spec, supports) + ) + + +def atomic_geography_nodes( + spec: Mapping, columns: Sequence[Owned], *, base: str, prefix: str = "geography" +) -> tuple[Node, ...]: + """Append shared nodes to an existing population, refusing column rewrites. + + SourceRef declarations use each system's ``source`` and ``raw-bytes-v1``. + ``identity`` must be stable across the country's sampling rungs; this builder + cannot establish that property merely from a column name. + """ + spec = geography.validate_assignment_spec(spec) + inventory = {(o.entity, o.column): o for o in columns} + if len(inventory) != len(columns): + raise ValueError("Atomic geography input inventory repeats columns.") + inputs = set(spec["identity"]) + for system in spec["systems"]: + inputs.update(system["selector"]) + inputs.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + inputs.add(system["observed_area"]) + assignment = tuple(spec["outputs"].values()) + layers = tuple( + sorted({layer["output"] for s in spec["systems"] for layer in s["layers"]}) + ) + if any(("household", c) not in inventory for c in inputs): + raise ValueError("Atomic geography is missing a declared household input.") + if any(("household", c) in inventory for c in (*assignment, *layers)): + raise ValueError("Atomic geography refuses to overwrite existing columns.") + imports, artifacts = [], [] + for i, system in enumerate(spec["systems"]): + name = f"{prefix}.support.{i}" + imports.append( + Node( + id=name, + kernel=AtomicSupportImportKernel.ref, + population=base, + sources=(system["source"],), + params={"source": system["source"], "system": system["id"]}, + artifact_outputs=(ArtifactOutput("support", ATOMIC_SUPPORT_TYPE),), + ) + ) + artifacts.append( + ArtifactInput(system["id"], name, "support", ATOMIC_SUPPORT_TYPE) + ) + + def node(suffix, kernel, read, write=()): + return Node( + id=f"{prefix}.{suffix}", + kernel=kernel.ref, + population=base, + inputs=(Slice("household", tuple(sorted(read))),), + outputs=tuple(Owned("household", c, "string") for c in write), + params={ + "definition": canonical_json(spec).decode(), + "stream": tuple(spec["stream"]), + }, + artifact_inputs=tuple(artifacts), + ) + + return ( + *imports, + node("assign", AtomicAssignKernel, inputs, assignment), + node("derive", AtomicDeriveKernel, inputs | set(assignment), layers), + node("gate", AtomicGeographyGateKernel, inputs | set(assignment) | set(layers)), + ) + + +def register_atomic_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + AtomicSupportImportKernel(), + AtomicAssignKernel(), + AtomicDeriveKernel(), + AtomicGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/tests/test_atomic_geography.py b/packages/microcosm-build/tests/test_atomic_geography.py new file mode 100644 index 000000000..84745e434 --- /dev/null +++ b/packages/microcosm-build/tests/test_atomic_geography.py @@ -0,0 +1,443 @@ +"""Invented support controls; no survey, Census or administrative files are read.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.graph_atomic_geography import ( + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def support_parts(system="invented_block", *, district=True): + columns = { + name: { + "kind": "code", + "source": "invented-lookup-v1", + "vintage": "invented-2020", + "relation": "official_tabulation" if name == "district" else "exact", + } + for name in ("area", "region", "puma", *(("district",) if district else ())) + } + columns.update( + { + name: {"kind": "weight", "source": "invented-counts-v1", "basis": basis} + for name, basis in (("population", "persons"), ("households", "households")) + } + ) + arrays = { + "area": np.asarray(["0001", "0002", "0003", "0004"]), + "region": np.asarray(["R", "R", "R", "S"]), + "puma": np.asarray(["A", "A", "B", "C"]), + "population": np.asarray([0, 2, 6, 1]), + "households": np.asarray([1, 3, 4, 1]), + } + if district: + arrays["district"] = np.asarray(["01", "02", "02", "03"]) + return { + "version": 1, + "system": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "columns": columns, + }, arrays + + +def fixture(system="invented_block", *, district=True): + metadata, arrays = support_parts(system, district=district) + payload = geo.encode_atomic_support(metadata, arrays) + support = geo.decode_atomic_support(payload) + spec = { + "version": 1, + "identity": ["source_record"], + "stream": ["sha256-u53-v1", "invented-geography", 0, 21], + "outputs": {"area": "block", "system": "area_system", "basis": "area_basis"}, + "systems": [ + { + "id": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "source": system + "_source", + "selector": {}, + "constraints": [ + {"input": "observed_region", "support": "region", "required": True}, + {"input": "observed_puma", "support": "puma", "required": False}, + ], + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": ( + [ + { + "input": "district", + "output": "district_code", + "vintage": "invented-2020", + "relation": "official_tabulation", + "source": "invented-lookup-v1", + } + ] + if district + else [] + ), + } + ], + } + households = pd.DataFrame( + { + "household_id": np.arange(1, 9, dtype=np.int64), + "source_record": pd.array( + ["source-" + str(i) for i in range(8)], dtype="string" + ), + "observed_region": pd.array(["R"] * 7 + ["S"], dtype="string"), + "observed_puma": pd.array( + [None, "A", "B", None, "A", None, "B", "C"], dtype="string" + ), + } + ) + return households, spec, {system: support}, payload + + +def complete(households, spec, supports): + assigned = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + return pd.concat([assigned, geo.derive_geography(assigned, spec, supports)], axis=1) + + +def test_codec_is_deterministic_and_decoded_values_are_immutable(): + metadata, arrays = support_parts() + first = geo.encode_atomic_support(metadata, arrays) + assert first == geo.encode_atomic_support( + metadata, dict(reversed(list(arrays.items()))) + ) + support = geo.decode_atomic_support(first) + assert support.arrays["area"].tolist() == ["0001", "0002", "0003", "0004"] + with pytest.raises(ValueError): + support.arrays["area"].setflags(write=True) + with pytest.raises(TypeError): + support.metadata["columns"]["area"]["source"] = "changed" + + +def test_integer_inverse_cdf_respects_zero_mass_and_boundaries(): + _, _, supports, _ = fixture() + index = geo._SupportIndex(supports["invented_block"]) + stage = {"level": "area", "weight": "population"} + cell = (("region", "R"),) + assert index.pick(cell, stage, 0.0) == "0002" + assert index.pick(cell, stage, 0.25 - 1 / 2**53) == "0002" + assert index.pick(cell, stage, 0.25) == "0003" + assert index.pick(cell, stage, 1 - 1 / 2**53) == "0003" + + +def test_draws_are_stable_under_order_subset_extra_columns_and_unrelated_rows(): + households, spec, supports, _ = fixture() + before = households.copy(deep=True) + expected = geo.assign_atomic(households, spec, supports) + reordered = households.iloc[[7, 2, 0, 5]] + actual = geo.assign_atomic(reordered.assign(unrelated=42), spec, supports) + pd.testing.assert_frame_equal(actual, expected.loc[reordered.index]) + extra = households.iloc[[0]].copy() + extra.index = [100] + extra["source_record"] = "unrelated-record" + grown = geo.assign_atomic(pd.concat([households, extra]), spec, supports) + pd.testing.assert_frame_equal(grown.loc[households.index], expected) + pd.testing.assert_frame_equal(households, before) + + +def test_stage_law_uses_declared_weights_at_each_level(): + households, spec, supports, _ = fixture() + spec["systems"][0]["stages"] = [ + {"level": "puma", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + result = complete(households, spec, supports) + assert result.loc[1, "block"] == "0002" + assert result.loc[2, "block"] == "0003" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + # A/B household mass is 4/4, but population mass is 2/6. + index = geo._SupportIndex(supports["invented_block"]) + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.49) == "A" + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.5) == "B" + + +def test_qualified_observed_area_is_retained_even_with_zero_sampling_mass(): + households, spec, supports, _ = fixture() + households["observed_block"] = pd.array(["0001"] + [None] * 7, dtype="string") + spec["systems"][0]["observed_area"] = "observed_block" + result = complete(households, spec, supports) + assert result.loc[0, "block"] == "0001" + assert result.loc[0, "area_basis"] == "observed" + assert result.loc[0, "district_code"] == "01" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + + +@pytest.mark.parametrize( + "defect", + [ + "missing_region", + "wrong_region", + "inconsistent_puma", + "duplicate_identity", + "null_identity", + "existing_output", + "unknown_observed_area", + "conflicting_observed_area", + ], +) +def test_assignment_refuses_invalid_or_conflicting_households(defect): + households, spec, supports, _ = fixture() + if defect == "missing_region": + households.loc[0, "observed_region"] = pd.NA + elif defect == "wrong_region": + households.loc[0, "observed_region"] = "missing" + elif defect == "inconsistent_puma": + households.loc[0, "observed_puma"] = "C" + elif defect == "duplicate_identity": + households.loc[0, "source_record"] = households.loc[1, "source_record"] + elif defect == "null_identity": + households.loc[0, "source_record"] = pd.NA + elif defect == "existing_output": + households["block"] = "untouched" + else: + households["observed_block"] = pd.array( + ["unknown" if defect == "unknown_observed_area" else "0004"] + [None] * 7, + dtype="string", + ) + spec["systems"][0]["observed_area"] = "observed_block" + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_area", + "negative_weight", + "float_weight", + "bool_weight", + "overflow_weight", + "all_zero_weight", + "unknown_relation", + "wrong_area_vintage", + "empty_code", + "wrong_length", + ], +) +def test_support_refuses_ambiguous_or_invalid_artifacts(defect): + metadata, arrays = support_parts() + if defect == "duplicate_area": + arrays["area"][1] = arrays["area"][0] + elif defect == "negative_weight": + arrays["population"][0] = -1 + elif defect == "float_weight": + arrays["population"] = arrays["population"].astype(float) + elif defect == "bool_weight": + arrays["population"] = arrays["population"].astype(bool) + elif defect == "overflow_weight": + arrays["population"] = np.asarray([2**63] * 4, dtype=np.uint64) + elif defect == "all_zero_weight": + arrays["population"] *= 0 + elif defect == "unknown_relation": + metadata["columns"]["district"]["relation"] = "exact_enough" + elif defect == "wrong_area_vintage": + metadata["columns"]["area"]["vintage"] = "different" + elif defect == "empty_code": + arrays["area"][0] = "" + elif defect == "wrong_length": + arrays["region"] = arrays["region"][:-1] + with pytest.raises(ValueError): + geo.encode_atomic_support(metadata, arrays) + + +@pytest.mark.parametrize("field", ["source", "relation", "vintage"]) +def test_layer_metadata_must_match_the_pinned_support(field): + households, spec, supports, _ = fixture() + spec["systems"][0]["layers"][0][field] = ( + "exact" if field == "relation" else "different" + ) + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +def test_three_nation_systems_use_the_same_operations_and_mark_absent_layers(): + households, spec, supports, _ = fixture() + households["nation"] = pd.array( + ["one"] * 3 + ["two"] * 3 + ["three"] * 2, dtype="string" + ) + spec["systems"][0]["selector"] = {"nation": ["one"]} + for name, selector, district in ( + ("invented_oa", "two", True), + ("invented_dz", "three", False), + ): + _, other, added, _ = fixture(name, district=district) + system = other["systems"][0] + system["selector"] = {"nation": [selector]} + spec["systems"].append(system) + supports.update(added) + result = complete(households, spec, supports) + assert ( + result["area_system"].tolist() + == ["invented_block"] * 3 + ["invented_oa"] * 3 + ["invented_dz"] * 2 + ) + assert result.loc[[6, 7], "district_code"].isna().all() + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + spec["systems"][1]["selector"] = {"nation": ["one", "two"]} + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) + + +def test_clone_inheritance_and_pruned_mapping_checks_do_not_redraw(): + households, spec, supports, _ = fixture() + original = complete(households, spec, supports) + clone = original.iloc[[2, 0, 5]].copy() + clone["household_id"] += 100 + clone["source_record"] += "-clone" + assert geo.validate_geography(clone, spec, supports)["households"] == 3 + clone.iloc[0, clone.columns.get_loc("district_code")] = "wrong" + with pytest.raises(ValueError, match="derived geography differs"): + geo.validate_geography(clone, spec, supports) + + +@pytest.mark.parametrize("defect", ["atomic", "basis", "observed", "system"]) +def test_final_integrity_gate_refuses_changed_geography(defect): + households, spec, supports, _ = fixture() + result = complete(households, spec, supports) + if defect == "atomic": + result.loc[0, "block"] = "missing" + elif defect == "basis": + result.loc[0, "area_basis"] = "observed" + elif defect == "observed": + result.loc[0, "observed_region"] = "wrong" + elif defect == "system": + result.loc[0, "area_system"] = pd.NA + with pytest.raises(ValueError): + geo.validate_geography(result, spec, supports) + + +class _InventedSpine(KernelBase): + ref = "invented.atomic_spine@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + households, _, _, _ = fixture() + person = pd.DataFrame( + { + "person_id": np.arange(1, 9, dtype=np.int64), + "person_household_id": households["household_id"], + "age": np.arange(21, 29, dtype=np.int64), + } + ) + return KernelResult( + frame=Frame( + {"person": person, "household": households}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(8), WeightKind.DESIGN)}, + pd.Series(["invented"] * 8, name="stratum"), + ) + ) + + +def test_real_executor_cold_warm_and_changed_source_identity(tmp_path): + households, spec, supports, payload = fixture() + columns = ( + Owned("person", "age", "int64"), + *( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ), + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes(spec, columns, base=base.id) + graph = compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(_InventedSpine()) + register_atomic_geography_kernels(registry) + raw_spine = tmp_path / "invented-spine.bin" + raw_spine.write_bytes(b"invented test source declaration") + raw_support = tmp_path / "invented-support.npz" + raw_support.write_bytes(payload) + sources = {"invented_spine": raw_spine, "invented_block_source": raw_support} + store = ContentStore(tmp_path / "store") + cold = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph(graph, sources=sources, store=store, kernels=registry) + actual = cold.population(base.id).table("household") + expected = complete(households, spec, supports) + for column in expected: + left, right = actual[column], expected[column] + if isinstance(expected[column].dtype, pd.StringDtype): + # Source checkpoints and emitted columns may use different string + # backends. Check the nullable-string contract, then compare every + # value and missing cell using one explicit physical representation. + assert isinstance(left.dtype, pd.StringDtype) + assert left.dtype.na_value is pd.NA + left = left.astype(pd.StringDtype(storage="python")) + right = right.astype(pd.StringDtype(storage="python")) + pd.testing.assert_series_equal(left, right, check_index_type=False) + assert cold.nodes["geography.assign"].key == warm.nodes["geography.assign"].key + assert cold.nodes["geography.gate"].receipt["outcome"] == "pass" + np.testing.assert_array_equal( + cold.population(base.id).weights_for("household").values, np.ones(8) + ) + metadata, arrays = support_parts() + arrays["population"][1] += 1 + raw_support.write_bytes(geo.encode_atomic_support(metadata, arrays)) + changed = run_graph(graph, sources=sources, store=store, kernels=registry) + assert changed.nodes["geography.assign"].key != cold.nodes["geography.assign"].key + + +def test_graph_builder_refuses_input_overwrites_and_duplicate_inventory(): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", c, "string") + for c in ("source_record", "observed_region", "observed_puma") + ) + with pytest.raises(ValueError, match="overwrite"): + atomic_geography_nodes( + spec, (*columns, Owned("household", "block", "string")), base="spine" + ) + with pytest.raises(ValueError, match="repeats"): + atomic_geography_nodes(spec, (*columns, columns[0]), base="spine") + with pytest.raises(ValueError, match="missing"): + atomic_geography_nodes(spec, columns[1:], base="spine") + altered = deepcopy(spec) + altered["outputs"]["area"] = "observed_region" + with pytest.raises(ValueError, match="overwritten"): + atomic_geography_nodes(altered, columns, base="spine") diff --git a/tools/ci_test_groups.py b/tools/ci_test_groups.py index bc737de4b..70c586519 100644 --- a/tools/ci_test_groups.py +++ b/tools/ci_test_groups.py @@ -30,7 +30,10 @@ # Shared-engine behavioral contracts that deliberately do not carry a country # prefix or the spec-engine ``test_spec_*`` prefix. Listing them makes their # shared-lane placement reviewed rather than a silent classifier default. -EXPLICIT_SHARED_SPEC = ("packages/microcosm-build/tests/test_cross_grain.py",) +EXPLICIT_SHARED_SPEC = ( + "packages/microcosm-build/tests/test_atomic_geography.py", + "packages/microcosm-build/tests/test_cross_grain.py", +) PROCESSES = { "trade": ("main",), From 4aaaeb9ab527d6089e84cb86aa2f6ee3c8c1923e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 15:55:50 -0400 Subject: [PATCH 026/255] Qualify survey Social Security totals and reporting semantics --- .../survey-social-security-source.added.md | 1 + docs/current-survey-puf59-progress.md | 6 +- docs/survey-social-security-source.md | 74 +++ .../current_social_security_source.py | 420 ++++++++++++++++++ .../us_runtime/survey_social_security.py | 176 ++++++++ .../test_us_survey_population_preparation.py | 9 + .../tests/test_us_survey_social_security.py | 401 +++++++++++++++++ 7 files changed, 1086 insertions(+), 1 deletion(-) create mode 100644 changelog.d/survey-social-security-source.added.md create mode 100644 docs/survey-social-security-source.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py create mode 100644 packages/microcosm-build/tests/test_us_survey_social_security.py diff --git a/changelog.d/survey-social-security-source.added.md b/changelog.d/survey-social-security-source.added.md new file mode 100644 index 000000000..11d76e2dd --- /dev/null +++ b/changelog.d/survey-social-security-source.added.md @@ -0,0 +1 @@ +Add current survey Social Security source qualification with explicit reporting units, question-universe unknownness, allocation literals and component mapping judgments. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index 0da23af28..9c752900e 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -8,7 +8,11 @@ Current survey financial completion passed 26 ordinary tests, including 15 ASEC The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. -Social Security handling remains a launch blocker. The PUF donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. The current reconciliation code can treat unknown recipient components as zero and use equal component shares. PUF59 requires a source-qualified recipient basis, including observed ASEC component/type information and an explicit ACS completion path, before those outputs can support release acceptance. +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The intended successor excludes these four outcomes from PUF enrichment, retains the survey's source-report totals, and conditions the remaining 55 outcomes on an additional Social Security total predictor. That successor is not yet wired into the native build. + +The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. + +The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md new file mode 100644 index 000000000..ae2b4d202 --- /dev/null +++ b/docs/survey-social-security-source.md @@ -0,0 +1,74 @@ +# Survey Social Security source semantics + +The source-report Social Security total precedes PUF enrichment. The current +qualifier verifies the original observations and preserves their limits; it +does not fit a completion model, assign individual beneficiaries, or certify a +population. The intended PUF successor has 55 outputs and uses the retained +total as a ninth conditioning input. Historical full65 and PUF59 behavior is +unchanged until a caller explicitly selects that successor. + +## Source measurements + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +describes `SS_YN` and `SS_VAL` on PDF page 49. The question covers people aged +15 or older and permits combined family payments on a person's report. A +positive report is therefore not necessarily one individual beneficiary's +payment. The qualifier preserves the source report's grain. + +The retained ASEC original member supplies `PERIDNUM`, household/line/age +coordinates, `SS_VAL`, `SS_YN`, `RESNSS1`, `RESNSS2`, `RESNSSA`, `I_SSVAL` +and `I_SSYN`. Source keys, the complete current cohort, raw member size/hash, +live source issuance and exact current money values must all agree. Source +year and income year are 2024; interview year is 2025. Carried reason fields +are not a substitute for these original literals. + +The [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines combined `SSP`, its age-15 question universe, and allocation flag +`FSSP`. The qualifier binds the retained 2024 ACS issuance and verifies +`SSP * (ADJINC / 1000000)` against the prepared amount. This remains a rolling +income window expressed in the survey's price basis. `FSSP` is not in the +current native projection, so publisher allocation provenance stays explicitly +unresolved; no observed-value claim is inferred from a missing flag. + +## Mapping judgments and unknownness + +The reason mapping is an explicit judgment at source-report grain. Retirement +and disability map to their corresponding component. Widowed and surviving +child reasons map to survivors; spouse and dependent child reasons map to +dependents. A single resolved component receives the report total. Repeated +reasons in the same component do not duplicate the amount. + +Distinct component reasons, missing reasons, code 7's combined child categories +and code 8's residual category leave the component amounts unknown. The +published reasons restrict possible components where they can; there is no +reason priority, age fallback or equal-share default. Separately observed +component amounts and individual beneficiary assignments are not claimed. + +An ASEC under-15 `SS_VAL=0` is a not-in-universe sentinel. Both ASEC and ACS +under-15 beneficiary totals remain unknown in the projection, with the raw +ASEC zero retained in the separate literal table. An in-universe zero remains +a known report total of zero. Contradictory universe/recipiency combinations +and invalid nonempty codes refuse; blank literals remain distinguishable +from invalid codes. + +An eventual component model may normalize nonnegative scores within allowed +components, preserving the source total and already resolved source basis. +It must supply positive allowed mass. The numerical helper does not authenticate +such a model; the graph must bind real fit/apply artifacts. A reporting-unit +convention, allocation to individual beneficiaries where necessary, explicit +income periods, design-weighted training, native-to-clone inheritance and +held-out checks remain separate required work. + +## Verification scope + +The 52-case control run executes 50 numerical/literal tests and two complete +maintained source issuances from invented original CSV/checkpoint fixtures. +It verifies source-identity joins despite reordered rows, repeat qualification, +defensive returned data, exact source totals, unknown components and unchanged +input frames. Source/control files, installed model source and the 12 admitted +code resources were unchanged before, after and in the external postcheck. + +The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: +`2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. +These controls are not native-source execution, a fitted component model, +PUF55 integration, policy-engine validation or release acceptance. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py new file mode 100644 index 000000000..8445b6304 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py @@ -0,0 +1,420 @@ +"""Current survey Social Security totals, reason literals and unknown shares. + +An additive projection from a live authenticated survey preparation. The +current ASEC original member is captured against the maintained closed member +registry; carried RESNSS columns are never source evidence. ACS SSP remains a +combined source amount. This module fits no model and issues no admission. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import source_csv_builtin +from . import survey_population_preparation as source +from . import survey_social_security as basis_owner +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-social-security-source.v1" +READ_COLUMNS = ( + "PERIDNUM", + "PH_SEQ", + "A_LINENO", + "A_AGE", + "SS_VAL", + "SS_YN", + "RESNSS1", + "RESNSS2", + "RESNSSA", + "I_SSVAL", + "I_SSYN", +) +WIDTHS = { + "PH_SEQ": 5, + "A_LINENO": 2, + "A_AGE": 2, + "SS_VAL": 5, + "SS_YN": 1, + "RESNSS1": 1, + "RESNSS2": 1, + "RESNSSA": 1, + "I_SSVAL": 2, + "I_SSYN": 2, +} +DICTIONARY = { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_pages_1based": { + "reasons": 48, + "total_and_recipiency": 49, + "allocation_values": [53, 56], + "allocation_fields": [57, 59], + }, +} + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SOCIAL_SECURITY_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _numeric(series): + require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "PHYSICAL_NUMERIC_TYPE", + ) + return series.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + + +def _read_capture(path, *, rows): + """Bounded literal parser; a path or row count confers no source authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, seen, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + indices = [header.index(name) for name in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + selected = [row[i] for i in indices] + require( + re.fullmatch(r"[0-9]{22}", selected[0], re.ASCII) is not None, + "PERSON_KEY", + ) + record = dict(zip(READ_COLUMNS, selected, strict=True)) + for name, width in WIDTHS.items(): + token = record[name] + # Missing measurements remain literal unknowns. Coordinates + # cannot be missing because they establish the source join. + require( + token == "" + or re.fullmatch(r"[0-9]{1," + str(width) + "}", token, re.ASCII) + is not None, + "TOKEN_WIDTH:" + name, + ) + require( + all(record[n] != "" for n in ("PH_SEQ", "A_LINENO", "A_AGE")), + "COORDINATE_MISSING", + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and selected[0] not in seen, + "COORDINATE_DUPLICATE", + ) + coordinates.add(pair) + seen.add(selected[0]) + records.append(record) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records, columns=READ_COLUMNS).set_index("PERIDNUM", drop=False) + + +def _codes(tokens, allowed): + values = np.full(len(tokens), np.nan, dtype=np.float64) + for i, token in enumerate(tokens): + require(type(token) is str, "CODE_TOKEN_TYPE") + if token != "": + require(re.fullmatch(r"[0-9]+", token, re.ASCII) is not None, "CODE_TOKEN") + require(int(token) in allowed, "CODE_DOMAIN") + values[i] = int(token) + return values + + +def _allocation_labels(raw): + total = _codes(raw.I_SSVAL, {0, 11, 12, 13, 14, 15}) + receipt = _codes(raw.I_SSYN, {0, 10, 11}) + reason = _codes(raw.RESNSSA, set(range(10))) + return pd.DataFrame( + { + "amount_allocation_code": total, + "recipiency_allocation_code": receipt, + "reason_allocation_code": reason, + "allocation_origin": [ + "unresolved_allocation_provenance" + if not np.isfinite([a, b, c]).all() + else "publisher_allocated" + if any(v != 0 for v in (a, b, c)) + else "publisher_no_allocation" + for a, b, c in zip(total, receipt, reason, strict=True) + ], + }, + index=raw.index, + ) + + +@dataclass(frozen=True) +class CurrentSocialSecurityProjection: + """Computed data, requalified by the host before execution and replay.""" + + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def qualify_current_social_security(preparation): + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + acs_owned = source.acs_native._owned(state.native[0]) + acs_receipt = json.loads(acs_owned.payload) + require(acs_receipt["vintage"] == 2024, "ACS_NATIVE_PERIOD") + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + selected_pins = [pin for pin in coverage._MEMBER_PINS if pin[0] == 2024] + require(len(selected_pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = selected_pins[0] + native_sources = issued[2].coverage.receipt["sources"] + retained = [s for s in native_sources if s["source_year"] == year] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-ss-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + # _capture's budget bounds its encoded coverage projection, not + # CSV bytes. Short valid source rows can expand in that format. + # Use the source owner's existing closed projection bound; the + # exact captured CSV size and digest remain separate checks. + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity, + "CAPTURE_CHANGED", + ) + require(_file_sha(captured) == digest, "CAPTURE_DIGEST") + + scope = parent.scope + current_positions = np.flatnonzero(np.asarray(scope.person_years) == 2024) + keys = np.asarray(scope.person_native_keys)[current_positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys].copy() + pp = parent.frame.person.iloc[current_positions] + for native_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[native_name].astype("int64").to_numpy(), + pp[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("SS_VAL") + require((field.validity[current_positions] == 1).all(), "CURRENT_AMOUNT_UNKNOWN") + amount = field.amounts[current_positions].copy() + require( + amount.dtype == np.dtype("float64") + and np.isfinite(amount).all() + and (amount >= 0).all(), + "CURRENT_AMOUNT_DOMAIN", + ) + literal_amount = _codes(ordered.SS_VAL, range(100000)) + require(np.array_equal(literal_amount, amount), "CURRENT_AMOUNT_SOURCE_IDENTITY") + reason_1 = _codes(ordered.RESNSS1, range(9)) + reason_2 = _codes(ordered.RESNSS2, range(9)) + recipiency = _codes(ordered.SS_YN, {0, 1, 2}) + report_amount, components, allowed, labels = basis_owner.asec_reporting_basis( + amount, + ordered.A_AGE.to_numpy(dtype=np.float64), + recipiency, + reason_1, + reason_2, + ) + allocation = _allocation_labels(ordered) + + people = state.frame.person + require( + people.person_id.dtype == np.dtype("int64") + and people.person_id.is_unique + and (people.person_id >= 0).all(), + "PERSON_AXIS", + ) + channel = people[support_channel_column("person")] + require(set(channel) == {"acs", "asec"}, "CHANNEL_ROSTER") + out = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + out["native_person_id"] = people[spine_source_id_column("person")].to_numpy() + out["source"] = channel.to_numpy() + out["social_security_source_total"] = np.nan + out["source_reporting_universe"] = False + out["source_reporting_unit"] = "person_report_record" + for c in basis_owner.COMPONENTS: + out[c] = np.nan + out["allowed_" + c] = True + out["basis_origin"] = "unresolved" + out["allocation_origin"] = "unresolved_allocation_provenance" + original_ids = np.asarray(scope.person_ids)[current_positions] + lookup = {int(pid): i for i, pid in enumerate(original_ids)} + asec_ids = out.index[out.source.eq("asec")] + require( + all(int(v) in lookup for v in out.loc[asec_ids, "native_person_id"]), + "SELECTED_NATIVE_JOIN", + ) + take = np.asarray([lookup[int(v)] for v in out.loc[asec_ids, "native_person_id"]]) + require(len(set(take)) == len(take), "SELECTED_NATIVE_DUPLICATE") + out.loc[asec_ids, "social_security_source_total"] = report_amount[take] + out.loc[asec_ids, "source_reporting_universe"] = ( + ordered.A_AGE.to_numpy(dtype=np.float64)[take] >= 15 + ) + out.loc[asec_ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, c in enumerate(basis_owner.COMPONENTS): + out.loc[asec_ids, c] = components[take, j] + out.loc[asec_ids, "allowed_" + c] = allowed[take, j] + out.loc[asec_ids, "basis_origin"] = np.asarray(labels)[take] + out.loc[asec_ids, "allocation_origin"] = allocation.allocation_origin.to_numpy()[ + take + ] + + acs = people.loc[channel.eq("acs")] + require( + {"SSP", "AGEP", "ADJINC", "acs_social_security_income"} <= set(acs), + "ACS_SOURCE_COLUMNS", + ) + raw_ss = _numeric(acs.SSP) + adjusted = _numeric(acs.acs_social_security_income) + age = _numeric(acs.AGEP) + factor = _numeric(acs.ADJINC) / 1_000_000 + require( + np.isfinite(age).all() + and (age == np.floor(age)).all() + and ((age >= 0) & (age <= 99)).all(), + "ACS_AGE", + ) + require(np.isfinite(factor).all() and (factor > 0).all(), "ACS_PRICE_FACTOR") + eligible = age >= 15 + out.loc[acs.person_id, "source_reporting_universe"] = eligible + require( + np.isnan(raw_ss[~eligible]).all() and np.isnan(adjusted[~eligible]).all(), + "ACS_OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require( + np.isfinite(raw_ss[eligible]).all() + and ( + (raw_ss[eligible] == 0) + | ((raw_ss[eligible] >= 4) & (raw_ss[eligible] <= 55000)) + ).all() + and (raw_ss[eligible] == np.floor(raw_ss[eligible])).all(), + "ACS_SSP_DOMAIN", + ) + require( + np.array_equal(adjusted[eligible], raw_ss[eligible] * factor[eligible]), + "ACS_SSP_ADJUSTMENT_IDENTITY", + ) + # Below-age-15 SSP is explicitly out of the question universe. Keep it + # unknown here; a separate reviewed reporting-unit convention is needed. + out.loc[acs.person_id, "social_security_source_total"] = adjusted + zero_ids = acs.person_id.to_numpy()[eligible & (raw_ss == 0)] + out.loc[zero_ids, list(basis_owner.COMPONENTS)] = 0.0 + out.loc[zero_ids, ["allowed_" + c for c in basis_owner.COMPONENTS]] = False + out.loc[zero_ids, "basis_origin"] = "known_total_zero" + out.loc[acs.person_id.to_numpy()[eligible & (raw_ss > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + out.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(issued[1]), + "acs_native_sha256": _sha(acs_owned.payload), + "acs_survey_year": 2024, + "money_header_sha256": _sha(ready.header), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "asec_income_year": 2024, + "asec_interview_year": 2025, + "price_basis_year": 2024, + "acs_income_window": "rolling_prior_12_months_at_2024_interview", + "acs_price_adjustment": "SSP*(ADJINC/1000000)", + "acs_dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "ssp_pdf_page_1based": 45, + "fssp_pdf_page_1based": 130, + }, + "acs_allocation_flag": "FSSP_not_in_current_native_projection; provenance_unresolved", + "asec_component_mapping": "unique_reported_component_only; no priority or age fallback", + "knownness": "unknown components and outside-universe SSP retained as unknown", + "reporting_grain": "source person report; ASEC may combine family payments", + "individual_beneficiary_assignment_claim": False, + "below15_convention": "both surveys retain unknown beneficiary totals; ASEC zero literal is NIU", + "observed_component_amounts_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + return CurrentSocialSecurityProjection(out, ordered, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py new file mode 100644 index 000000000..adf73f3ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py @@ -0,0 +1,176 @@ +"""Explicit source-report Social Security component basis. + +These are numerical owners, not source admission APIs. A graph host must bind +the live source projection and any actual fitted completion artifact before +using the returned values. Missing components never become zero. Report-record totals are not individual +beneficiary incidence. This module neither replaces totals with PUF draws nor +allocates combined family payments to individual beneficiaries. +""" + +from __future__ import annotations + +import numpy as np + +COMPONENTS = ( + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", +) +PROTOCOL = "microcosm.us.survey-social-security-basis.v1" +REASON_COMPONENTS = { + 1: (0,), + 2: (1,), + 3: (3,), + 4: (2,), + 5: (3,), + 6: (2,), + # The published code mixes surviving, dependent and disabled children. + 7: (1, 2, 3), + 8: (0, 1, 2, 3), +} + + +def require(condition, reason): + if not condition: + raise ValueError("SURVEY_SOCIAL_SECURITY_" + reason) + + +def _float64(values, *, shape, nullable=False): + require(type(values) is np.ndarray, "ARRAY_TYPE") + require(values.dtype == np.dtype("float64") and values.shape == shape, "ARRAY_AXIS") + require(not np.isinf(values).any(), "NONFINITE") + require(nullable or not np.isnan(values).any(), "UNKNOWN") + require((values[~np.isnan(values)] >= 0).all(), "NEGATIVE") + return values.copy() + + +def asec_reason_basis(total, reason_1, reason_2): + """Map a source total and reasons without inventing component amounts. + + Callers qualify the actual money and literal reason observations. Codes + 3/5 share survivors and 4/6 share dependents. Multiple distinct components, + code 7, code 8, or missing reasons leave the positive amount unallocated. + A single component receives the total as an explicit mapping judgment. + No age fallback or priority between reported reasons is applied. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + reasons = [] + for values in (reason_1, reason_2): + arr = _float64(values, shape=total.shape, nullable=True) + require( + ( + (arr[np.isfinite(arr)] == np.floor(arr[np.isfinite(arr)])) + & (arr[np.isfinite(arr)] <= 8) + ).all(), + "REASON_DOMAIN", + ) + reasons.append(arr) + out = np.full((len(amount), 4), np.nan, dtype=np.float64) + allowed = np.ones((len(amount), 4), dtype=np.bool_) + labels = [] + for i, value in enumerate(amount): + if value == 0: + out[i] = 0 + allowed[i] = False + labels.append("known_total_zero") + continue + codes = [arr[i] for arr in reasons] + if any(np.isnan(code) for code in codes): + labels.append("unresolved_missing_reason") + continue + nonzero = [int(code) for code in codes if code != 0] + if not nonzero: + labels.append("unresolved_niu_reason_with_positive_total") + continue + possible = set().union(*(REASON_COMPONENTS[code] for code in nonzero)) + allowed[i] = False + allowed[i, list(possible)] = True + if len(possible) == 1: + out[i] = 0 + out[i, next(iter(possible))] = value + labels.append("source_reason_total_allocation") + else: + labels.append("unresolved_component_split") + return out, allowed, tuple(labels) + + +def asec_reporting_basis(total, age, recipiency, reason_1, reason_2): + """Respect the age-15 reporting universe without inferring beneficiaries. + + SS_VAL=0 outside the question universe is a source sentinel, not observed + absence of benefits. Retain that literal upstream and return unknown total + and shares here. A reporting adult may receive combined family payments; + the returned component basis remains at that source report's grain. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + ages = _float64(age, shape=total.shape) + receipt = _float64(recipiency, shape=total.shape, nullable=True) + require(((ages == np.floor(ages)) & (ages <= 99)).all(), "AGE_DOMAIN") + require( + np.isin(receipt[np.isfinite(receipt)], [0, 1, 2]).all(), "RECIPIENCY_DOMAIN" + ) + eligible = ages >= 15 + require( + ((amount[~eligible] == 0) & (receipt[~eligible] == 0)).all(), + "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require(np.isin(receipt[eligible], [1, 2]).all(), "IN_UNIVERSE_RECIPIENCY_UNKNOWN") + require(not ((amount > 0) & (receipt != 1)).any(), "RECIPIENCY_CONTRADICTION") + basis, allowed, labels = asec_reason_basis(amount, reason_1, reason_2) + amount[~eligible] = np.nan + basis[~eligible] = np.nan + allowed[~eligible] = True + labels = tuple( + label if inside else "asec_below15_outside_reporting_universe" + for label, inside in zip(labels, eligible, strict=True) + ) + return amount, basis, allowed, labels + + +def complete_positive_basis(total, source_basis, allowed, modeled_scores): + """Convert actual model scores to shares only for unresolved positive rows. + + The graph owner must authenticate the fitted artifact. This numerical API + cannot do so. Existing complete rows must have no modeled scores; eligible + rows require a complete nonnegative four-vector with positive allowed mass. + Source totals and excluded component possibilities are retained exactly. + A missing/zero model vector refuses instead of creating equal shares. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + shape = (len(amount), 4) + basis = _float64(source_basis, shape=shape, nullable=True) + scores = _float64(modeled_scores, shape=shape, nullable=True) + require( + type(allowed) is np.ndarray + and allowed.dtype == np.dtype("bool") + and allowed.shape == shape, + "ALLOWED_AXIS", + ) + complete = np.isfinite(basis).all(axis=1) + require((complete | np.isnan(basis).all(axis=1)).all(), "PARTIAL_COMPONENT_BASIS") + require(np.isnan(scores[complete]).all(), "MODELED_SOURCE_OVERWRITE") + require( + np.array_equal(basis[complete].sum(axis=1), amount[complete]), + "SOURCE_TOTAL_IDENTITY", + ) + require((basis[complete][~allowed[complete]] == 0).all(), "SOURCE_SUPPORT") + missing = ~complete + require((amount[missing] > 0).all(), "UNRESOLVED_ZERO_TOTAL") + require(np.isfinite(scores[missing]).all(), "COMPLETION_UNKNOWN") + selected = scores[missing].copy() + selected[~allowed[missing]] = 0 + mass = selected.sum(axis=1) + require(np.isfinite(mass).all() and (mass > 0).all(), "COMPLETION_NO_ALLOWED_MASS") + filled = selected / mass[:, None] * amount[missing, None] + # Put the floating residual on the largest supported component. This keeps + # excluded components exactly zero and avoids systematic lost pennies. + if len(filled): + j = filled.argmax(axis=1) + filled[np.arange(len(filled)), j] += amount[missing] - filled.sum(axis=1) + require(np.isfinite(filled).all() and (filled >= 0).all(), "COMPLETION_ARITHMETIC") + basis[missing] = filled + return basis diff --git a/packages/microcosm-build/tests/test_us_survey_population_preparation.py b/packages/microcosm-build/tests/test_us_survey_population_preparation.py index 3715c3db2..9e7a68558 100644 --- a/packages/microcosm-build/tests/test_us_survey_population_preparation.py +++ b/packages/microcosm-build/tests/test_us_survey_population_preparation.py @@ -33,6 +33,7 @@ def fixture( zero_key=None, missing_asec_money=None, current_predictor_money=None, + acs_ssp_values=None, ): asec_root = tmp_path / "asec-original" asec_root.mkdir() @@ -83,6 +84,14 @@ def fixture( _person("2024GQ0000001", 1, 37, AGEP=40, MIL="4", ESR="6", PWGTP=77), _person("2024GQ0000002", 1, 38, AGEP=50, MIL="4", ESR="6", PWGTP=78), ] + if acs_ssp_values is not None: + seen = set() + for person in people: + key = (person["SERIALNO"], person["SPORDER"]) + if key in acs_ssp_values: + person["SSP"] = acs_ssp_values[key] + seen.add(key) + assert seen == set(acs_ssp_values) acs = build_fixture( tmp_path / "acs-original", monkeypatch, diff --git a/packages/microcosm-build/tests/test_us_survey_social_security.py b/packages/microcosm-build/tests/test_us_survey_social_security.py new file mode 100644 index 000000000..8d8b20533 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_social_security.py @@ -0,0 +1,401 @@ +"""Invented numerical and literal-reader controls; no source issuer substitute.""" + +from __future__ import annotations + +import csv +import hashlib +import shutil + +import numpy as np +import pandas as pd +import pytest +from test_us_asec_coverage_authentication import _changed_parent +from test_us_survey_population_preparation import fixture + +from microcosm.build.frame_checkpoint import load_frame_checkpoint +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_social_security_source as source +from microcosm.build.us_runtime import survey_social_security as ss + + +@pytest.mark.parametrize( + ("reason", "component"), [(1, 0), (2, 1), (3, 3), (4, 2), (5, 3), (6, 2)] +) +def test_unique_reported_component_receives_source_total(reason, component): + total = np.asarray([1200.0]) + first, second = np.asarray([float(reason)]), np.asarray([0.0]) + values, allowed, origin = ss.asec_reason_basis(total, first, second) + expected = np.zeros((1, 4)) + expected[0, component] = 1200 + np.testing.assert_array_equal(values, expected) + np.testing.assert_array_equal(allowed, expected > 0) + assert origin == ("source_reason_total_allocation",) + np.testing.assert_array_equal(total, [1200]) + np.testing.assert_array_equal(first, [reason]) + + +def test_two_reasons_in_same_component_do_not_duplicate_total(): + values, allowed, _ = ss.asec_reason_basis( + np.array([300.0]), np.array([3.0]), np.array([5.0]) + ) + np.testing.assert_array_equal(values, [[0, 0, 0, 300]]) + np.testing.assert_array_equal(allowed, [[False, False, False, True]]) + + +@pytest.mark.parametrize( + ("first", "second", "allowed"), + [ + (1.0, 2.0, [True, True, False, False]), + (7.0, 0.0, [False, True, True, True]), + (8.0, 0.0, [True, True, True, True]), + (np.nan, 1.0, [True, True, True, True]), + (0.0, 0.0, [True, True, True, True]), + ], +) +def test_ambiguous_positive_total_has_no_fabricated_component(first, second, allowed): + values, support, origin = ss.asec_reason_basis( + np.array([100.0]), np.array([first]), np.array([second]) + ) + assert np.isnan(values).all() + np.testing.assert_array_equal(support, [allowed]) + assert origin[0].startswith("unresolved_") + + +def test_known_zero_does_not_need_a_component_reason(): + values, support, origin = ss.asec_reason_basis( + np.array([0.0]), np.array([np.nan]), np.array([np.nan]) + ) + np.testing.assert_array_equal(values, np.zeros((1, 4))) + assert not support.any() + assert origin == ("known_total_zero",) + + +def test_asec_niu_zero_remains_unknown_but_reporting_zero_is_known(): + total = np.array([0.0, 0.0, 1200.0]) + age = np.array([14.0, 15.0, 70.0]) + receipt = np.array([0.0, 2.0, 1.0]) + first, second = np.array([0.0, 0.0, 1.0]), np.zeros(3) + inputs = [a.copy() for a in (total, age, receipt, first, second)] + amount, basis, allowed, labels = ss.asec_reporting_basis( + total, age, receipt, first, second + ) + assert np.isnan(amount[0]) and np.isnan(basis[0]).all() and allowed[0].all() + assert labels[0] == "asec_below15_outside_reporting_universe" + np.testing.assert_array_equal(amount[1:], [0.0, 1200.0]) + np.testing.assert_array_equal(basis[1:], [[0, 0, 0, 0], [1200, 0, 0, 0]]) + for original, current in zip( + inputs, (total, age, receipt, first, second), strict=True + ): + np.testing.assert_array_equal(original, current) + + +@pytest.mark.parametrize( + ("amount", "age", "receipt", "error"), + [ + (100.0, 14.0, 1.0, "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION"), + (0.0, 14.0, 2.0, "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION"), + (0.0, 15.0, 0.0, "IN_UNIVERSE_RECIPIENCY_UNKNOWN"), + (0.0, 15.0, np.nan, "IN_UNIVERSE_RECIPIENCY_UNKNOWN"), + (100.0, 30.0, 2.0, "RECIPIENCY_CONTRADICTION"), + (0.0, 15.5, 2.0, "AGE_DOMAIN"), + (0.0, 100.0, 2.0, "AGE_DOMAIN"), + (0.0, 30.0, 3.0, "RECIPIENCY_DOMAIN"), + ], +) +def test_reporting_universe_refuses_contradictions(amount, age, receipt, error): + with pytest.raises(ValueError, match=error): + ss.asec_reporting_basis( + np.array([amount]), + np.array([age]), + np.array([receipt]), + np.array([0.0]), + np.array([0.0]), + ) + + +@pytest.mark.parametrize("value", [-1.0, 9.0, 1.5, np.inf]) +def test_invalid_reason_is_not_age_fallback(value): + with pytest.raises( + ValueError, match="SURVEY_SOCIAL_SECURITY_(REASON_DOMAIN|NONFINITE|NEGATIVE)" + ): + ss.asec_reason_basis(np.array([100.0]), np.array([value]), np.array([0.0])) + + +def test_completion_preserves_known_basis_and_source_totals(): + total = np.array([120.0, 300.0, 0.0]) + source_basis = np.array([[120.0, 0, 0, 0], [np.nan] * 4, [0.0] * 4]) + allowed = np.array( + [[True, False, False, False], [False, True, True, False], [False] * 4] + ) + scores = np.array([[np.nan] * 4, [1000.0, 1, 2, 500], [np.nan] * 4]) + originals = [x.copy() for x in (total, source_basis, allowed, scores)] + got = ss.complete_positive_basis(total, source_basis, allowed, scores) + np.testing.assert_array_equal(got, [[120, 0, 0, 0], [0, 100, 200, 0], [0, 0, 0, 0]]) + np.testing.assert_array_equal(got.sum(axis=1), total) + for original, current in zip( + originals, (total, source_basis, allowed, scores), strict=True + ): + np.testing.assert_array_equal(original, current) + + +@pytest.mark.parametrize( + ("scores", "message"), + [ + ([0.0, 0, 0, 0], "COMPLETION_NO_ALLOWED_MASS"), + ([5.0, 0, 0, 0], "COMPLETION_NO_ALLOWED_MASS"), + ([np.nan, 1, 1, 0], "COMPLETION_UNKNOWN"), + ([0.0, -1, 1, 0], "NEGATIVE"), + ], +) +def test_completion_refuses_invalid_modeled_basis(scores, message): + with pytest.raises(ValueError, match=message): + ss.complete_positive_basis( + np.array([100.0]), + np.full((1, 4), np.nan), + np.array([[False, True, True, False]]), + np.asarray([scores]), + ) + + +def test_completion_cannot_overwrite_a_reported_basis(): + with pytest.raises(ValueError, match="MODELED_SOURCE_OVERWRITE"): + ss.complete_positive_basis( + np.array([100.0]), + np.array([[100.0, 0, 0, 0]]), + np.array([[True, False, False, False]]), + np.ones((1, 4)), + ) + + +def _literal_row(**changes): + return { + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "70", + "SS_VAL": "12000", + "SS_YN": "1", + "RESNSS1": "1", + "RESNSS2": "0", + "RESNSSA": "0", + "I_SSVAL": "0", + "I_SSYN": "0", + **changes, + } + + +def _write_literals(path, rows): + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=source.READ_COLUMNS) + writer.writeheader() + writer.writerows(rows) + + +def test_literal_reader_preserves_missing_reason_and_allocation_provenance(tmp_path): + path = tmp_path / "invented.csv" + _write_literals(path, [_literal_row(RESNSS1="", I_SSVAL="15", RESNSSA="9")]) + result = source._read_capture(path, rows=1) + assert result.iloc[0].RESNSS1 == "" + assert result.iloc[0].PERIDNUM == "0000000000000000000001" + assert ( + source._allocation_labels(result).iloc[0].allocation_origin + == "publisher_allocated" + ) + + +@pytest.mark.parametrize("token", ["9", "-1", "1.0", "1e0", " 1", "١", 1]) +def test_invalid_nonempty_code_is_never_silently_missing(token): + with pytest.raises(ValueError, match="CURRENT_SOCIAL_SECURITY_CODE_"): + source._codes([token], set(range(9))) + + +def test_blank_code_stays_unknown_separately_from_valid_niu(): + values = source._codes(["", "0", "8"], set(range(9))) + assert np.isnan(values[0]) + np.testing.assert_array_equal(values[1:], [0, 8]) + + +@pytest.mark.parametrize("field,token", [("I_SSVAL", "10"), ("I_SSYN", "9")]) +def test_out_of_domain_allocation_code_refuses(field, token): + with pytest.raises(ValueError, match="CODE_DOMAIN"): + source._allocation_labels(pd.DataFrame([_literal_row(**{field: token})])) + + +@pytest.mark.parametrize( + "changes", + [ + {"RESNSS1": "12"}, + {"SS_VAL": "1e3"}, + {"PH_SEQ": ""}, + {"PERIDNUM": "1"}, + {"I_SSVAL": "-1"}, + ], +) +def test_literal_reader_refuses_noncanonical_source_tokens(tmp_path, changes): + path = tmp_path / "invented.csv" + _write_literals(path, [_literal_row(**changes)]) + with pytest.raises(ValueError, match="CURRENT_SOCIAL_SECURITY_"): + source._read_capture(path, rows=1) + + +def test_literal_reader_refuses_duplicate_person_keys(tmp_path): + path = tmp_path / "invented.csv" + _write_literals(path, [_literal_row(), _literal_row(A_LINENO="2")]) + with pytest.raises(ValueError, match="COORDINATE_DUPLICATE"): + source._read_capture(path, rows=2) + + +def test_a_projection_receipt_cannot_replace_live_source_authority(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + source.qualify_current_social_security({"source_authenticated": True}) + + +def _social_security_arguments(tmp_path, monkeypatch, *, ambiguous): + """Construct original SS observations before fresh maintained issuance. + + The raw CSV and original checkpoint are fixture inputs, not mutations of a + received Population. The maintained helper rebuilds identities and source + pins, then the real person-income attachment and preparation are rebuilt. + No source issuer, validation method, ready artifact, or model is replaced. + """ + arguments = fixture( + tmp_path, + monkeypatch, + zero=False, + acs_ssp_values={("2024HU0000001", 1): 1200}, + ) + root = arguments["source_dir"] / "asec" + parent_path, household_path = root / "parent.h5", root / "household-attachment.h5" + parent = load_frame_checkpoint(parent_path).frame.person + values = parent.SS_VAL.to_numpy(dtype=np.float64, copy=True) + for pid, value in ((105, 12000.0), (106, 0.0), (107, 3000.0), (108, 0.0)): + positions = np.flatnonzero(parent.person_id.to_numpy() == pid) + assert len(positions) == 1 + values[positions[0]] = value + _changed_parent(parent_path, household_path, monkeypatch, {"SS_VAL": values}) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + members, pins = {}, [] + for year, member, archive, *_ in source.coverage._MEMBER_PINS: + path = root / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + amounts = np.asarray( + [updated.loc[k, "SS_VAL"] for k in raw.PERIDNUM], dtype=np.float64 + ) + assert np.isfinite(amounts).all() and (amounts == np.floor(amounts)).all() + raw["SS_VAL"] = [str(int(v)) for v in amounts] + raw["SS_YN"] = [ + "1" if value > 0 else "0" if int(age) < 15 else "2" + for value, age in zip(amounts, raw.A_AGE, strict=True) + ] + raw["RESNSS1"] = ["1" if v > 0 else "0" for v in amounts] + raw["RESNSS2"] = "0" + raw["RESNSSA"] = "0" + raw["I_SSVAL"] = "0" + raw["I_SSYN"] = "0" + if year == 2024: + raw.loc[raw.PERIDNUM.eq(str(5).zfill(22)), "RESNSS1"] = "2" + k = raw.PERIDNUM.eq(str(7).zfill(22)) + raw.loc[k, "RESNSS1"] = "7" if ambiguous else "3" + raw.loc[k, "RESNSSA"] = "9" + raw.loc[k, "I_SSVAL"] = "15" + # Prove the join by original identity rather than source row order. + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + members[year] = path + for owner in (source.coverage, restoration): + monkeypatch.setattr(owner, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "ss-restored-money" + restoration.restore_asec_person_income_source( + parent_path, + household_path, + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, root / "person-income-attachment.h5" + ) + # Ensure the real source pin update is retained by the original owner. + assert money._SOURCE_PINS[0] == hashlib.sha256(parent_path.read_bytes()).hexdigest() + return arguments + + +@pytest.mark.parametrize("ambiguous", [False, True]) +def test_actual_current_source_qualifier_preserves_totals_reasons_and_unknownness( + tmp_path, monkeypatch, ambiguous +): + arguments = _social_security_arguments(tmp_path, monkeypatch, ambiguous=ambiguous) + prepared = source.source.prepare_authenticated_survey_population(**arguments) + before = prepared.checked_view().frame + originals = {e: before.table(e).copy(deep=True) for e in before.entities} + result = source.qualify_current_social_security(prepared) + asec = result.person.loc[result.person.source.eq("asec")].set_index( + "native_person_id" + ) + assert asec.loc[105, "social_security_source_total"] == 12000.0 + np.testing.assert_array_equal( + asec.loc[105, list(ss.COMPONENTS)].to_numpy(dtype=np.float64), [0, 12000, 0, 0] + ) + assert asec.loc[107, "social_security_source_total"] == 3000.0 + assert asec.loc[107, "allocation_origin"] == "publisher_allocated" + if ambiguous: + assert asec.loc[107, list(ss.COMPONENTS)].isna().all() + assert asec.loc[107, "basis_origin"] == "unresolved_component_split" + assert not asec.loc[107, "allowed_social_security_retirement"] + else: + assert asec.loc[107, "social_security_survivors"] == 3000.0 + acs = result.person.loc[result.person.source.eq("acs")] + positive = acs.social_security_source_total.gt(0) + assert positive.any() + assert acs.loc[positive, list(ss.COMPONENTS)].isna().all().all() + assert ( + acs.loc[positive, "basis_origin"] + .eq("acs_combined_positive_requires_model") + .all() + ) + assert ( + not result.evidence["source_admission_issued"] + and not result.evidence["release_eligible"] + ) + assert ( + result.evidence["asec_income_year"] + == result.evidence["price_basis_year"] + == 2024 + ) + assert result.evidence["asec_interview_year"] == 2025 + assert result.evidence["acs_survey_year"] == 2024 + assert len(result.evidence["acs_native_sha256"]) == 64 + assert not result.evidence["individual_beneficiary_assignment_claim"] + assert asec.source_reporting_unit.eq( + "person_report_may_combine_family_payments" + ).all() + fresh = source.qualify_current_social_security(prepared) + pd.testing.assert_frame_equal(fresh.person, result.person, check_exact=True) + assert fresh.evidence == result.evidence + result.person.loc[:, "social_security_source_total"] = -1 + result.evidence["dictionary"]["sha256"] = "0" * 64 + assert ( + source.qualify_current_social_security(prepared).evidence["dictionary"][ + "sha256" + ] + == source.DICTIONARY["sha256"] + != "0" * 64 + ) + assert ( + source.qualify_current_social_security( + prepared + ).person.social_security_source_total.dropna() + >= 0 + ).all() + for entity, original in originals.items(): + pd.testing.assert_frame_equal(before.table(entity), original, check_exact=True) From 4bdc97253c6e986f6a5e4a53c1c8a3487415de4b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 16:18:17 -0400 Subject: [PATCH 027/255] Preserve survey Social Security in an explicit PUF55 profile --- .../puf55-survey-social-security.added.md | 1 + docs/current-survey-puf59-progress.md | 8 +- docs/survey-social-security-source.md | 9 +- .../build/us_runtime/full_puf_enrichment.py | 33 +- .../tests/test_us_puf55_survey_ss_profile.py | 388 ++++++++++++++++++ 5 files changed, 427 insertions(+), 12 deletions(-) create mode 100644 changelog.d/puf55-survey-social-security.added.md create mode 100644 packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md new file mode 100644 index 000000000..1efdab5aa --- /dev/null +++ b/changelog.d/puf55-survey-social-security.added.md @@ -0,0 +1 @@ +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index 9c752900e..e165533cf 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -8,12 +8,18 @@ Current survey financial completion passed 26 ordinary tests, including 15 ASEC The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. -Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The intended successor excludes these four outcomes from PUF enrichment, retains the survey's source-report totals, and conditions the remaining 55 outcomes on an additional Social Security total predictor. That successor is not yet wired into the native build. +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The explicit `PUF55_SURVEY_SS` profile now excludes these four outcomes from PUF enrichment and conditions the remaining 55 outcomes on an additional Social Security total predictor. It preserves existing survey component fields, including unknownness. This profile is not yet wired into the native build. + +The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. + +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native donor/recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. +The combined current source also passes all **225 component integration controls**: 34 shared geography cases, 52 survey Social Security cases, 11 PUF55 cases and all 128 historical full65/PUF59 cases. The exact combined run completed in 205.6 seconds at 687.8 MB peak RSS with no skips or unexpected access refusals. Before/after/current source, model-source and code-resource checks all agree; the maintained checkout matches the tested source map. Receipt SHA256: `63ef0ad0602a66d40f0736490d4c1e29f57723ba938c58d317802b387f947a87`. This remains an invented-input component integration, not a native population certification. + The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. These tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. Genuine survey replay and complete calibration/export acceptance remain pending. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md index ae2b4d202..e7bd1cbfe 100644 --- a/docs/survey-social-security-source.md +++ b/docs/survey-social-security-source.md @@ -3,9 +3,10 @@ The source-report Social Security total precedes PUF enrichment. The current qualifier verifies the original observations and preserves their limits; it does not fit a completion model, assign individual beneficiaries, or certify a -population. The intended PUF successor has 55 outputs and uses the retained -total as a ninth conditioning input. Historical full65 and PUF59 behavior is -unchanged until a caller explicitly selects that successor. +population. The explicit `PUF55_SURVEY_SS` profile has 55 outputs and uses an +upstream-qualified tax-unit total as its ninth conditioning input. Historical +full65 and PUF59 behavior is unchanged. The new profile's native source and +whole-population attachment integration remain pending. ## Source measurements @@ -71,4 +72,4 @@ code resources were unchanged before, after and in the external postcheck. The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. These controls are not native-source execution, a fitted component model, -PUF55 integration, policy-engine validation or release acceptance. +native PUF55 integration, policy-engine validation or release acceptance. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py index 8f21b78e1..cae79e8f2 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py @@ -38,6 +38,12 @@ "puf_2015_capped_return_size", *PREDICTORS[2:], ) +SURVEY_SS_TOTAL_PREDICTOR = "puf_conditioning_social_security_total" +SURVEY_SS_COMPONENTS = tuple(support.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS) +PUF55_SURVEY_SS_PREDICTORS = (*PUF59_PREDICTORS, SURVEY_SS_TOTAL_PREDICTOR) +PUF55_SURVEY_SS_PERSON_OUTPUTS = tuple( + target for target in PERSON_OUTPUTS if target not in SURVEY_SS_COMPONENTS +) # Compatibility constants above continue to describe FULL65. PUF59 keeps the @@ -61,9 +67,12 @@ class PufOutputProfile(Enum): FULL65 = "full65" PUF59 = "puf59" + PUF55_SURVEY_SS = "puf55_survey_ss" @property def predictors(self): + if self is PUF55_SURVEY_SS: + return PUF55_SURVEY_SS_PREDICTORS return PREDICTORS if self is FULL65 else PUF59_PREDICTORS @property @@ -71,6 +80,8 @@ def source_predictors(self): # Independently measured source leaves, never derived here from generic # filing status or actual recipient membership. The six monetary # predictors retain their original ordered canonical arithmetic aliases. + if self is PUF55_SURVEY_SS: + return (*self.predictors[:2], SURVEY_SS_TOTAL_PREDICTOR) return () if self is FULL65 else self.predictors[:2] @property @@ -83,6 +94,8 @@ def donor_auxiliary_columns(self): @property def person_outputs(self): + if self is PUF55_SURVEY_SS: + return PUF55_SURVEY_SS_PERSON_OUTPUTS return PERSON_OUTPUTS @property @@ -98,11 +111,12 @@ def phase(self): # Legacy kernel parameter schemas are exact: phase carries the profile # identity into every train/apply key and receipt without widening them. # FULL65 retains its existing declarations and cache identities. - return PHASE if self is FULL65 else PHASE + ".puf59" + return PHASE if self is FULL65 else PHASE + "." + self.value FULL65 = PufOutputProfile.FULL65 PUF59 = PufOutputProfile.PUF59 +PUF55_SURVEY_SS = PufOutputProfile.PUF55_SURVEY_SS def require_puf_output_profile(profile): @@ -159,9 +173,10 @@ def _profile_source_values(table, *, profile): """ if not profile.source_predictors: return () - status, size = ( + measured = tuple( _numeric(table[column], label=column) for column in profile.source_predictors ) + status, size = measured[:2] _require( bool(np.isin(status, [1, 2, 3, 4]).all()), "PUF_PROFILE_FILING_STATUS_DOMAIN" ) @@ -169,7 +184,9 @@ def _profile_source_values(table, *, profile): bool(((size >= 1) & (size == np.floor(size))).all()), "PUF_PROFILE_RETURN_SIZE_DOMAIN", ) - return status, size + if profile is PUF55_SURVEY_SS: + _require(bool((measured[2] >= 0).all()), "PUF_PROFILE_SOCIAL_SECURITY_DOMAIN") + return measured def _ids(values, label): @@ -239,7 +256,7 @@ def canonical_full_puf_donor( ) return_capacity_column = ( profile.donor_auxiliary_columns[0] - if profile is PUF59 + if profile.donor_auxiliary_columns else "tax_unit_person_count" ) tcols = ( @@ -296,7 +313,7 @@ def canonical_full_puf_donor( _require( bool(((count >= 1) & (count == np.floor(count))).all()), "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN" - if profile is PUF59 + if profile.donor_auxiliary_columns else "PUF_PERSON_COUNT_DOMAIN", ) donor = pd.DataFrame(index=tax_unit.index) @@ -342,7 +359,7 @@ def canonical_full_puf_donor( # component alone represents the predictor. FULL65 keeps its historical # ordinary-component alias for compatibility with accepted prior evidence. schedule_c = donor.self_employment_income_before_lsr.to_numpy() - if profile is PUF59: + if profile is not FULL65: schedule_c = ( schedule_c + donor.sstb_self_employment_income_before_lsr.to_numpy() ) @@ -361,9 +378,11 @@ def canonical_full_puf_donor( # PUF59 first two features are already source-measured. Never label generic # status/membership arithmetic as the PUF2015 disclosure measurement. if profile.source_predictors: + measured = _profile_source_values(tax_unit, profile=profile) feature_values = ( - *_profile_source_values(tax_unit, profile=profile), + *measured[:2], *feature_values[2:], + *measured[2:], ) for name, values in zip(profile.predictors, feature_values, strict=True): donor[name] = _numeric(values, label=name) diff --git a/packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py b/packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py new file mode 100644 index 000000000..7422f5208 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf55_survey_ss_profile.py @@ -0,0 +1,388 @@ +"""PUF55 controls with invented measurements and actual QRF graph execution. + +No native source admission, Social Security model, or release claim. +""" + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import InventedSource +from test_us_full_puf_output_profiles import ( + InventedProfileMatrix, + _puf59_columns, + _puf59_frame, + _puf59_return_only_columns, +) + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import puf_support as support +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.keys import opaque_artifact_key + +PROFILE = full.PUF55_SURVEY_SS +SS_TOTAL = full.SURVEY_SS_TOTAL_PREDICTOR +SS = full.SURVEY_SS_COMPONENTS + + +def _columns(): + p, t, pk, tk = _puf59_columns() + t[SS_TOTAL] = [0.0, 100.0, 1200.0, 24000.0] + tk[SS_TOTAL] = True + return p.drop(columns=list(SS)), t, pk.drop(columns=list(SS)), tk + + +def _donor55(): + p, t, pk, tk = _columns() + return full.canonical_full_puf_donor( + p, t, person_known=pk, tax_unit_known=tk, profile=PROFILE + ) + + +def _frame(): + frame = _puf59_frame() + t = frame.table("tax_unit") + t[SS_TOTAL] = np.resize(np.array([0.0, 0.0, 1200.0, 24000.0]), len(t)) + for j, name in enumerate(SS): + frame.person[name] = np.resize( + np.array([-0.0, np.nan, 30.0 + j, 0.0]), len(frame.person) + ) + frame.revalidate() + return frame + + +def _known55(frame): + t = frame.table("tax_unit") + mask = support.puf_tax_detail_clone_mask(t, entity="tax_unit") + return pd.DataFrame( + True, + columns=PROFILE.predictors, + index=pd.Index(t.loc[mask, "tax_unit_id"].to_numpy(), name="tax_unit_id"), + ) + + +def test_profile_is_explicit_and_leaves_historical_contracts_unchanged(): + assert len(PROFILE.targets) == 55 and len(PROFILE.person_outputs) == 52 + assert PROFILE.predictors == (*full.PUF59.predictors, SS_TOTAL) + assert PROFILE.source_predictors == (*full.PUF59.source_predictors, SS_TOTAL) + assert not set(SS) & set(PROFILE.targets) + assert not set(full.SCF_MORTGAGE_OUTPUTS) & set(PROFILE.targets) + assert len(full.FULL65.targets) == 65 and len(full.PUF59.targets) == 59 + assert ( + full.FULL65.person_outputs == full.PUF59.person_outputs == full.PERSON_OUTPUTS + ) + assert len({PROFILE.phase, full.FULL65.phase, full.PUF59.phase}) == 3 + assert full.FULL65.phase == full.PHASE + assert full.PUF59.phase == full.PHASE + ".puf59" + + +def test_donor_total_is_an_explicit_measurement_and_ss_components_are_not_consumed(): + p, t, pk, tk = _columns() + for name in SS: + p[name] = ["not an observed component"] * len(p) + result = full.canonical_full_puf_donor( + p, t, person_known=pk, tax_unit_known=tk, profile=PROFILE + ) + assert tuple(result) == ( + *PROFILE.predictors, + *PROFILE.targets, + "weight", + "puf_person_incidence_capacity", + ) + np.testing.assert_array_equal(result[SS_TOTAL], t[SS_TOTAL]) + assert result.weight.tolist() == [1.0, 0.0, 2.0, 3.0] + assert not set(SS) & set(result) + + +def test_return_only_profile_has_no_synthetic_persons_or_ss_output_carriers(): + returns, known = _puf59_return_only_columns() + returns = returns.drop(columns=list(SS)) + known = known.drop(columns=list(SS)) + returns[SS_TOTAL] = [0.0, 100.0, 1200.0, 24000.0] + known[SS_TOTAL] = True + result = full.canonical_full_puf_donor( + None, + returns, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=PROFILE.person_outputs, + profile=PROFILE, + ) + assert len(result) == 4 + assert result.puf_person_incidence_capacity.eq(1.0).all() + assert not set(SS) & set(result) + + +@pytest.mark.parametrize("value", [-1.0, np.nan, np.inf, "0", True]) +def test_invalid_total_is_never_synthesized_from_components(value): + p, t, pk, tk = _columns() + t[SS_TOTAL] = t[SS_TOTAL].astype(object) + t.loc[0, SS_TOTAL] = value + with pytest.raises( + ValueError, + match="PUF_(PROFILE_SOCIAL_SECURITY_DOMAIN|UNKNOWN|NONFINITE|PHYSICAL_TYPE)", + ): + full.canonical_full_puf_donor( + p, t, person_known=pk, tax_unit_known=tk, profile=PROFILE + ) + + +def test_missing_or_unknown_donor_total_refuses(): + p, t, pk, tk = _columns() + with pytest.raises(ValueError, match="PUF_DONOR_COLUMNS:tax_unit"): + full.canonical_full_puf_donor( + p, + t.drop(columns=SS_TOTAL), + person_known=pk, + tax_unit_known=tk, + profile=PROFILE, + ) + tk.loc[0, SS_TOTAL] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:tax_unit"): + full.canonical_full_puf_donor( + p, t, person_known=pk, tax_unit_known=tk, profile=PROFILE + ) + + +def test_recipient_total_uses_declared_tax_unit_grain_and_knownness(): + f, donor = _frame(), _donor55() + k = _known55(f) + result = full.prepare_full_puf_inputs(f, donor, predictor_known=k, profile=PROFILE) + matrix = model_input.decode_recipient_matrix(result.matrix) + assert len(matrix.features.columns) == 9 + np.testing.assert_array_equal( + matrix.features[SS_TOTAL], [0.0, 0.0, 1200.0, 24000.0] + ) + k.iloc[0, -1] = False + with pytest.raises(ValueError, match="PUF_UNKNOWN:recipient_predictor"): + full.prepare_full_puf_inputs(f, donor, predictor_known=k, profile=PROFILE) + k.iloc[0, -1] = True + del f.table("tax_unit")[SS_TOTAL] + with pytest.raises(ValueError, match="PUF_PROFILE_PREDICTOR_SOURCE"): + full.prepare_full_puf_inputs(f, donor, predictor_known=k, profile=PROFILE) + + +def test_real_puf55_chain_preserves_survey_ss_and_replays(tmp_path): + frame, donor = _frame(), _donor55() + known = _known55(frame) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=PROFILE + ) + matrix = model_input.decode_recipient_matrix(prepared.matrix) + # An explicit minimal recipient model Frame for the ordinary graph control; + # finalization below consumes the actual retained US clone Frame. + table = matrix.features.copy() + table.insert(0, "tax_unit_id", matrix.entity_ids) + from microcosm.frame import EntitySchema + + recipient = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": matrix.entity_ids, + "person_tax_unit_id": matrix.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + { + "tax_unit": Weights( + frame.resolve_weights("tax_unit").values[ + support.puf_tax_detail_clone_mask( + frame.table("tax_unit"), entity="tax_unit" + ) + ], + WeightKind.DESIGN, + ) + }, + ) + store = ContentStore(tmp_path / "store", codecs={"frame-store": load_frame_store}) + sources, nodes = {}, [] + for name, value in (("donor", prepared.donor_frame), ("recipient", recipient)): + key = full.codec.sha(("invented-full-puf:" + name).encode()) + sources[name] = store.put_frame(key, value) + nodes.append( + Node( + name, + InventedSource.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("tax_unit", col, "float64") + for col in value.table("tax_unit") + if col != "tax_unit_id" + ), + ) + ) + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population="recipient", + matrix_producer="matrix", + seed=578, + n_estimators=2, + zero_atol=0, + profile=PROFILE, + ) + assert len(fits) == len(applies) == 55 + assert [ + e.name for e in applies[-1].artifact_inputs if e.name.startswith("prior_") + ] == [f"prior_{i:03d}" for i in range(54)] + nodes.append( + Node( + "matrix", + InventedProfileMatrix.ref, + population="recipient", + inputs=(Slice("tax_unit", PROFILE.predictors),), + artifact_outputs=( + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + ) + compiled = compile_graph( + Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*nodes, *fits, *applies), + ) + ) + registry = KernelRegistry() + for kernel in ( + InventedSource(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + registry.register(kernel) + actual = run_graph(compiled, sources=sources, store=store, kernels=registry) + + def payload(manifest, node_id, artifact): + record = manifest.node(node_id) + key = opaque_artifact_key(record.key, artifact) + assert key == record.opaque_artifacts[artifact] + return store.load_bytes(key) + + raw = { + target: payload(actual, node.id, "raw_draw") + for target, node in zip(PROFILE.targets, applies, strict=True) + } + kwargs = dict( + matrix=payload(actual, "matrix", "matrix"), + matrix_producer_key=actual.node("matrix").key, + raw_draws=raw, + apply_state=payload(actual, applies[-1].id, "apply_state"), + training_state=payload(actual, fits[-1].id, "training_state"), + seed=578, + profile=PROFILE, + ) + assert kwargs["matrix"] == prepared.matrix + before = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + last_model = payload(actual, fits[-1].id, "model") + result, receipt = full.finalize_full_puf( + frame, donor, predictor_known=known, last_model=last_model, **kwargs + ) + assert receipt["release_eligible"] is False and receipt["tail_bounds"] + assert receipt["target_order"] == list(PROFILE.targets) + for entity in frame.entities: + pd.testing.assert_frame_equal(frame.table(entity), before[entity]) + np.testing.assert_array_equal( + result.weights_for("household").values, frame.weights_for("household").values + ) + for entity, outputs in ( + ("person", PROFILE.person_outputs), + ("tax_unit", PROFILE.tax_unit_outputs), + ): + mask = support.puf_tax_detail_clone_mask(result.table(entity), entity=entity) + for output in outputs: + assert result.table(entity).loc[mask, output].notna().all() + if output not in before[entity]: + assert result.table(entity).loc[~mask, output].isna().all() + else: + actual_cells = result.table(entity).loc[~mask, output] + expected_cells = before[entity].loc[~mask, output] + # The full finalizer materializes requested Boolean outputs + # with nullable storage. Compare exact native values and NA + # semantics in that declared Boolean representation. The SS + # columns below have separate unnormalized bit-pattern checks. + if pd.api.types.is_bool_dtype(expected_cells.dtype): + assert pd.api.types.is_bool_dtype(actual_cells.dtype) + actual_cells = actual_cells.astype("boolean") + expected_cells = expected_cells.astype("boolean") + pd.testing.assert_series_equal( + actual_cells, + expected_cells, + check_exact=True, + ) + for component in SS: + np.testing.assert_array_equal( + result.person[component].to_numpy().view("uint64"), + before["person"][component].to_numpy().view("uint64"), + ) + np.testing.assert_array_equal( + result.table("tax_unit")[SS_TOTAL].to_numpy().view("uint64"), + before["tax_unit"][SS_TOTAL].to_numpy().view("uint64"), + ) + changed_frame = _frame() + tax = changed_frame.table("tax_unit") + tax.loc[tax.index[-1], SS_TOTAL] += 1.0 + with pytest.raises(ValueError, match="PUF_RECIPIENT_MATRIX_CHANGED"): + full.finalize_full_puf( + changed_frame, donor, predictor_known=known, last_model=last_model, **kwargs + ) + people = result.table("person") + child = support.puf_tax_detail_clone_mask(people, entity="person") & people.age.lt( + 15 + ) + assert people.loc[child, "employment_income_before_lsr"].eq(0).all() + changed_raw = dict(raw) + values = full.codec.read_raw_target( + raw[PROFILE.targets[0]], target=PROFILE.targets[0], index=matrix.features.index + ).copy() + values[0] += 1.0 + changed_raw[PROFILE.targets[0]] = full.codec.encode_raw_target( + values, target=PROFILE.targets[0], index=matrix.features.index + ) + with pytest.raises(ValueError, match="PUF_RAW_HISTORY"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": changed_raw}) + with pytest.raises(ValueError, match="PUF_FULL_MATRIX_BINDING"): + full.decode_full_puf_draws(**{**kwargs, "matrix_producer_key": "f" * 64}) + changed_donor = donor.copy(deep=True) + changed_donor.loc[0, "casualty_loss"] += 1.0 + with pytest.raises(ValueError, match="PUF_DONOR_CONSUMED_BYTES"): + full.finalize_full_puf( + frame, changed_donor, predictor_known=known, last_model=last_model, **kwargs + ) + incomplete = dict(raw) + incomplete.pop(PROFILE.targets[-1]) + with pytest.raises(ValueError, match="PUF_RAW_ROSTER"): + full.decode_full_puf_draws(**{**kwargs, "raw_draws": incomplete}) + changed_training = full.codec.decode_json(kwargs["training_state"]) + changed_training["models"][0]["sha256"] = "e" * 64 + with pytest.raises(ValueError, match="PUF_FULL_CHAIN_IDENTITY"): + full.decode_full_puf_draws( + **{**kwargs, "training_state": full.codec.encode_json(changed_training)} + ) + warm = run_graph( + compiled, sources=sources, store=store, kernels=registry, resume="require" + ) + assert warm.key == actual.key and all(record.hit for record in warm.nodes.values()) + for node in applies: + assert payload(warm, node.id, "raw_draw") == payload( + actual, node.id, "raw_draw" + ) From 74814151a89e9ffc0ace982846152ef87623e73d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 16:40:43 -0400 Subject: [PATCH 028/255] Project canonical PUF donors into the survey SS profile --- changelog.d/puf55-canonical-donor.added.md | 1 + docs/current-survey-puf59-progress.md | 6 +- docs/us-launch-review.md | 19 +-- .../us-survey-age-development-20260909.json | 54 ++++++++ .../build/us_runtime/puf55_canonical_donor.py | 120 ++++++++++++++++++ .../tests/test_us_puf59_canonical.py | 97 +++++++++++++- 6 files changed, 287 insertions(+), 10 deletions(-) create mode 100644 changelog.d/puf55-canonical-donor.added.md create mode 100644 experiments/us-survey-age-development-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py diff --git a/changelog.d/puf55-canonical-donor.added.md b/changelog.d/puf55-canonical-donor.added.md new file mode 100644 index 000000000..10e06ae48 --- /dev/null +++ b/changelog.d/puf55-canonical-donor.added.md @@ -0,0 +1 @@ +Add a canonical59 artifact adapter for the explicit PUF55 survey Social Security profile, retaining return identity and weights and validating the total-carrier convention. Record the scoped passing native seven-node age-development run. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index e165533cf..99e3df74b 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -22,4 +22,8 @@ The combined current source also passes all **225 component integration controls The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. -These tests do not certify a national or congressional-district file, native-data fit quality, a country-engine result, or a release. Genuine survey replay and complete calibration/export acceptance remain pending. +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority and has not yet processed the native donor. + +The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. + +This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 27c909006..0dd9b2772 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -143,9 +143,9 @@ The following work remains open: 1. Finish current survey predictors and demographic graph bindings, then connect them to the genuine combined survey population. -2. Integrate the genuine 59-output donor and its corrected integrity codec with - the combined population, complete source-qualified Social Security recipient - reconciliation, and supply SCF loan inputs. Separate component runs passed +2. Project the genuine canonical59 donor into the explicit 55-output profile, + bind the survey Social Security total as predictor nine, complete survey + beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed 128 profile/compatibility controls and 26 current-survey predictor controls; see [the implementation status](current-survey-puf59-progress.md) and [donor construction and growth](puf2015-canonical59-and-growth.md). @@ -157,11 +157,14 @@ The following work remains open: the build must retain one assigned Census block and derive its larger geographies. Prior wages remain excluded. A source operator's existence is not complete cell coverage. -4. Verify the optimized identity encoder on genuine survey replay. One genuine - successor run reached persisted-artifact verification but exhausted its - two-hour CPU limit. That run failed; it is not a verified build. Performance - changes now pass the small identity controls; a new genuine run must also pass - the current seven-node calibration/diagnostics checks before acceptance. +4. Extend the accepted small native replay to the complete build. The genuine + seven-node age-development run now passes at 1/1000: 1,584 source households, + 3,168 records after the support clone, final export/readback and owner/target + verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at + `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. + The prior two-hour failure and stale-derived-attachment failure remain failed + historical runs. [The scoped acceptance record](../experiments/us-survey-age-development-20260909.json) + does not certify enrichment, full national/CD calibration or a release. 5. Run real model evaluation, national/CD calibration, holdout checks and complete export/replay verification, followed by a dashboard bound to that exact file. Check every pruned analysis file separately. diff --git a/experiments/us-survey-age-development-20260909.json b/experiments/us-survey-age-development-20260909.json new file mode 100644 index 000000000..54df4b623 --- /dev/null +++ b/experiments/us-survey-age-development-20260909.json @@ -0,0 +1,54 @@ +{ + "artifact_readback_and_final_owner_target_verification": "passed inside the unchanged reviewed harness; final manifest exists only after final verification", + "code_resource_pins_current_equal": true, + "code_resources": 12, + "compiled_order": [ + "survey_population.create", + "survey_population.allocate", + "combined_survey_puf_support_clone", + "combined_survey_puf_support_clone.owned", + "survey.age_count_matrix", + "survey.sampling_budget", + "survey.age_calibration" + ], + "cpu_seconds": 4115.737018, + "cross_process_source_authority_granted": false, + "entity_rows": { + "current": { + "household": 3168, + "person": 6928 + }, + "prepared": { + "household": 1584, + "person": 3464 + }, + "previous": { + "household": 3168, + "person": 6928 + } + }, + "export_count": 5, + "manifest_sha256": "2bdb68de84635c6cc2e0c25555887693304792e25d40052cd77996774cb7f669", + "native_counts": { + "acs": { + "households": 1529, + "persons": 3324 + }, + "asec": { + "households": 55, + "persons": 140 + } + }, + "native_payloads_read_by_postcheck": 0, + "owned_before_after_current_equal": true, + "peak_rss_bytes": 8260485120, + "receipt_sha256": "2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca", + "release_eligible": false, + "result": 0, + "scope": "genuine seven-node ACS+ASEC age-development artifact acceptance; not full enrichment or release", + "source_before_after_current_equal": true, + "source_control_runtime_files": 473, + "source_head": "a9895d8a50886470f6dfe4b2b9d20c352c9871cf", + "unexpected_refusals": {}, + "wall_seconds": 4268.8438986669935 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py new file mode 100644 index 000000000..2c01c34f2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py @@ -0,0 +1,120 @@ +"""Project a bound canonical59 artifact into the survey-SS PUF55 donor. + +This in-memory converter grants no source authority. The caller authenticates +the artifact upstream and supplies its expected identity. No file is opened, +person is invented, or recipient Social Security component is produced here. +""" + +from __future__ import annotations + +import hashlib +import re + +import numpy as np +import pandas as pd + +from . import full_puf_enrichment as enrichment +from .puf59_canonical_artifact import decode_canonical_puf59 + +_SS_CARRIER_METHOD = ( + "Component-neutral carrier: total SS in retirement, other component carriers " + "zero; requires recipient-profile SS reconciliation. Zeros are not " + "observations of absent benefits." +) + + +def canonical_puf55_donor_from_artifact( + payload: bytes, + *, + expected_artifact_sha256: str, + expected_growth_scheme: str = "family_observed", +): + """Return the ordered donor and descriptive projection metadata. + + The canonical59 v2 producer puts grown E02400 into its retirement carrier + and zeroes the other three carriers. Require that exact modeled convention + before relabeling the total as a predictor. It is transported tax-return + income, not an observed current-year total or a beneficiary component. + + The returned table has the source RECID axis. Its mutable values and this + metadata do not constitute an authenticated graph edge or release receipt. + """ + if ( + type(payload) is not bytes + or type(expected_artifact_sha256) is not str + or re.fullmatch(r"[0-9a-f]{64}", expected_artifact_sha256) is None + or hashlib.sha256(payload).hexdigest() != expected_artifact_sha256 + ): + raise ValueError("PUF55_DONOR_ARTIFACT_IDENTITY") + arrays, source_receipt = decode_canonical_puf59( + payload, expected_growth_scheme=expected_growth_scheme + ) + assumptions = source_receipt.get("model_assumptions") + if ( + not isinstance(assumptions, dict) + or assumptions.get("schema") != "microcosm.us.puf59_baseline_models/1" + or assumptions.get("ss_method") != _SS_CARRIER_METHOD + or any( + np.any(arrays[name] != 0) + for name in enrichment.SURVEY_SS_COMPONENTS + if name != "social_security_retirement" + ) + ): + raise ValueError("PUF55_DONOR_SS_CARRIER") + + profile = enrichment.PUF55_SURVEY_SS + index = pd.Index(arrays["RECID"], name="tax_unit_id") + tax_unit = pd.DataFrame( + { + "tax_unit_id": arrays["RECID"], + "weight": arrays["weight"], + "filing_status_code": arrays["puf_2015_filing_status_code"], + "puf_person_incidence_capacity": arrays["puf_person_incidence_capacity"], + **{name: arrays[name] for name in profile.targets}, + **{name: arrays[name] for name in enrichment.PUF59.source_predictors}, + enrichment.SURVEY_SS_TOTAL_PREDICTOR: arrays["social_security_retirement"], + }, + index=index, + ) + # Known here means present canonical cells, including modeled values. It + # does not reclassify the inherited source/model provenance as observed. + known = pd.DataFrame( + True, + index=index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *profile.targets, + *profile.source_predictors, + ), + ) + donor = enrichment.canonical_full_puf_donor( + None, + tax_unit, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=profile.person_outputs, + profile=profile, + ) + return donor, { + "schema": "microcosm.us.puf55_canonical_donor_projection/1", + "artifact_sha256": expected_artifact_sha256, + "source_receipt_sha256": source_receipt["sha256"], + "source_statistical_year": 2015, + "money_year": 2024, + "rows": len(donor), + "profile": profile.value, + "ordered_columns": list(donor.columns), + "row_axis": "source RECID, tax-return grain", + "social_security_total": { + "predictor": enrichment.SURVEY_SS_TOTAL_PREDICTOR, + "raw_field": "E02400", + "origin": "modeled_transport", + "growth_scheme": expected_growth_scheme, + "growth_recipe_sha256": source_receipt["growth"]["recipe_sha256"], + "component_interpretation": "No observed component allocation", + }, + "source_authority_granted": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/tests/test_us_puf59_canonical.py b/packages/microcosm-build/tests/test_us_puf59_canonical.py index 3585e4ff2..652c39c4a 100644 --- a/packages/microcosm-build/tests/test_us_puf59_canonical.py +++ b/packages/microcosm-build/tests/test_us_puf59_canonical.py @@ -2,13 +2,16 @@ def test_puf59_canonical_source_and_artifact_controls(): + import copy import hashlib + import json from dataclasses import replace import numpy as np import pandas as pd from microcosm.build.us_runtime import full_puf_enrichment as enrichment + from microcosm.build.us_runtime import puf55_canonical_donor as donor55 from microcosm.build.us_runtime import puf59_canonical as canonical from microcosm.build.us_runtime import puf59_canonical_artifact as artifact from microcosm.build.us_runtime import puf_full_source as full @@ -360,8 +363,100 @@ def build(source, **kwargs): np.array_equal(donor.weight.to_numpy(), result.design_weight), ) + # Reuse actual source decoding, canonical construction and envelope checks. + # The old four SS carriers represent one total, never observed components. + for candidate in (small_result, result): + packed = artifact.encode_canonical_puf59(candidate) + digest = hashlib.sha256(packed).hexdigest() + projected, metadata = donor55.canonical_puf55_donor_from_artifact( + packed, expected_artifact_sha256=digest + ) + profile = enrichment.PUF55_SURVEY_SS + check( + "puf55_exact_order_" + str(len(candidate.recids)), + tuple(projected.columns) + == ( + *profile.predictors, + *profile.targets, + "weight", + "puf_person_incidence_capacity", + ), + ) + check( + "puf55_identity_axis_" + str(len(candidate.recids)), + projected.index.name == "tax_unit_id" + and np.array_equal(projected.index.to_numpy(), candidate.recids), + ) + check( + "puf55_total_predictor_is_grown_source_total", + np.array_equal( + projected[enrichment.SURVEY_SS_TOTAL_PREDICTOR].to_numpy(), + candidate.columns["social_security_retirement"], + ), + ) + check( + "puf55_all_retained_targets_and_weight_unchanged", + all( + np.array_equal(projected[k].to_numpy(), candidate.columns[k]) + for k in profile.targets + ) + and np.array_equal(projected.weight.to_numpy(), candidate.design_weight) + and (projected.puf_person_incidence_capacity == 1).all(), + ) + check( + "puf55_no_component_carriers_or_source_authority", + not set(enrichment.SURVEY_SS_COMPONENTS) & set(projected.columns) + and metadata["artifact_sha256"] == digest + and metadata["source_receipt_sha256"] == candidate.receipt["sha256"] + and metadata["social_security_total"]["origin"] == "modeled_transport" + and metadata["release_eligible"] is False + and metadata["source_authority_granted"] is False, + ) + check( + "puf55_source_bytes_unchanged", + artifact.encode_canonical_puf59(candidate) == packed, + ) + refuses( + "puf55_expected_artifact_identity_required", + lambda packed=packed: donor55.canonical_puf55_donor_from_artifact( + packed, expected_artifact_sha256="0" * 64 + ), + "PUF55_DONOR_ARTIFACT_IDENTITY", + ) + + # Correctly rehashed, internally consistent artifacts still cannot change + # the upstream SS carrier semantics accepted by this projection. + def repacked_ss_change(*, column=None, method=None): + changed_arrays = dict(arrays) + changed_receipt = copy.deepcopy(receipt) + if column is not None: + changed_arrays[column] = np.ones(len(arrays["RECID"])) + changed_receipt["growth"]["output_values_sha256"] = growth._digest( + changed_arrays + ) + if method is not None: + changed_receipt["model_assumptions"]["ss_method"] = method + changed_receipt.pop("sha256") + changed_receipt["sha256"] = hashlib.sha256( + json.dumps( + changed_receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return artifact.reencode_canonical_puf59(changed_arrays, changed_receipt) + + for altered in ( + repacked_ss_change(column="social_security_disability"), + repacked_ss_change(method="Individual component observations"), + ): + refuses( + "puf55_changed_carrier_contract_refused", + lambda altered=altered: donor55.canonical_puf55_donor_from_artifact( + altered, expected_artifact_sha256=hashlib.sha256(altered).hexdigest() + ), + "PUF55_DONOR_SS_CARRIER", + ) + # Integrity successor: domain-valid mutations cannot inherit old provenance. - import json import struct check( From 2ff0cff1fb649ca34afc290541a1e634472350a4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:08:20 -0400 Subject: [PATCH 029/255] Normalize US block support for shared geography assignment --- docs/current-survey-puf59-progress.md | 4 +- docs/geography-assignment.md | 12 + .../us-atomic-block-adapter-20260909.json | 25 ++ .../us-puf55-native-donor-20260909.json | 40 +++ .../build/us_runtime/atomic_block_support.py | 275 ++++++++++++++++++ .../tests/test_us_atomic_block_support.py | 131 +++++++++ 6 files changed, 485 insertions(+), 2 deletions(-) create mode 100644 experiments/us-atomic-block-adapter-20260909.json create mode 100644 experiments/us-puf55-native-donor-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py create mode 100644 packages/microcosm-build/tests/test_us_atomic_block_support.py diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index 99e3df74b..bddbdea27 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -12,7 +12,7 @@ Social Security handling remains a launch blocker. The historical PUF59 donor tr The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. -Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native donor/recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The native donor projection now passes as described below. The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. @@ -22,7 +22,7 @@ The combined current source also passes all **225 component integration controls The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. -The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority and has not yet processed the native donor. +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority. A subsequent separately guarded native projection passes for all 207,692 returns, retaining every target, source RECID, weight and incidence capacity and appending the modeled Social Security total as predictor nine. It took 11.05 seconds at 1.43 GB peak RSS. The original canonical artifact remained unchanged; source/control/model/code checks agree, with no unexpected access refusals. Receipt SHA256: `ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c`. The earlier attempt passed its value assertions but lacked collection accounting and was refused by the guard; that failure is preserved. This native projection ran in memory and produced metadata only. It did not fit a recipient model or attach results to the survey population. The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index 2d68ee34d..20d7c30cb 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -164,3 +164,15 @@ using the modal active-postcode constituency. Review the official [NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) before choosing or replacing that approximation. Source existence is not acceptance of a particular lookup or its application to the population. + +The US block adapter in `us_runtime/atomic_block_support.py` now preserves each +supplied block, leading zeroes, population weight and the source-labeled +tract-to-PUMA and block-to-district mappings. Its declaration uses the shared +operators, preserves observed state/PUMA separately, and labels CD119 as +`official_tabulation`. Its sampling proxy is 2020 persons; this does not imply +household counts or support for subsequent construction in unpopulated 2020 +blocks. All 13 invented tests passed, including missing/inconsistent mapping +refusals and subset-stable assignment. Receipt SHA256: +`404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. +Actual normalized block support and attachment before the survey clone remain +pending; these tests do not establish native geographic fit. diff --git a/experiments/us-atomic-block-adapter-20260909.json b/experiments/us-atomic-block-adapter-20260909.json new file mode 100644 index 000000000..1980d44db --- /dev/null +++ b/experiments/us-atomic-block-adapter-20260909.json @@ -0,0 +1,25 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 3.078341, + "district_relation": "official_tabulation", + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 353140736, + "receipt_sha256": "404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c", + "release_eligible": false, + "result": 0, + "sampling_basis": "2020_census_persons", + "scope": "Invented US block normalization and declaration through shared assignment, derivation and validation; native support and graph integration remain pending.", + "source_and_owned_files": 479, + "source_sha256": "8d776a60e5ace340b0de745461b89679653522bb672b95ab43bdfabda0f602ee", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 13 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.607667459000368 +} diff --git a/experiments/us-puf55-native-donor-20260909.json b/experiments/us-puf55-native-donor-20260909.json new file mode 100644 index 000000000..68dc87e6d --- /dev/null +++ b/experiments/us-puf55-native-donor-20260909.json @@ -0,0 +1,40 @@ +{ + "code_resources": 12, + "cpu_seconds": 8.784642, + "expected_test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "first_attempt": "Value checks passed, but removing the diagnostic plugin also removed collection accounting. Guard refused the run. The replacement retained exact collection accounting while suppressing private failure context; neither values nor access permissions changed.", + "input_artifact_sha256": "28cd44bd59218c1e77f14fa35c510ab550d2035527e3c139e7d19875c12ac11c", + "model_source_files": 5983, + "native_input_hash_verified_inside_guard_before_and_after": true, + "native_payload_reopened_by_postcheck": false, + "native_resources": 1, + "original_canonical_artifact_unchanged": true, + "outputs": 55, + "peak_rss_bytes": 1431109632, + "predictors": 9, + "qrf_fitted": false, + "receipt_sha256": "ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c", + "release_eligible": false, + "report_sha256": "7ade9f442fe25f02e013b39afa5bf8077e89a75b43b4c6c7d0d2dc123f052766", + "result": 0, + "rows": 207692, + "scope": "Native canonical59 artifact projected to the survey-SS PUF55 donor; no recipient fit or whole-Population attachment.", + "source_and_code_before_after_current_equal": true, + "source_authority_granted": false, + "source_commit": "74814151a89e9ffc0ace982846152ef87623e73d", + "source_owned_files": 478, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.048004040989326, + "whole_population_attachment": false +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py new file mode 100644 index 000000000..0033c318d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py @@ -0,0 +1,275 @@ +"""Normalize US block mappings for shared geography operators; no new kernel. + +Publisher parsers supply block POP100, the CD block-equivalency mapping and the +2020 tract-to-PUMA mapping. Source authentication remains upstream. This module +retains block rows and supplies a country declaration; assignment and geographic +derivation are executed by the shared country-neutral operators. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + encode_atomic_support, + validate_assignment_spec, +) + +SYSTEM = "us_census_block_2020" +SOURCE = "us_atomic_block_support" +_STATES = frozenset( + { + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 53, + 54, + 55, + 56, + } +) + + +def _require(condition, message): + if not condition: + raise ValueError("US atomic support: " + message) + + +def _integer(value, *, minimum=0, maximum=2**53): + return ( + isinstance(value, Integral) + and not isinstance(value, (bool, np.bool_)) + and minimum <= value <= maximum + ) + + +def _sources(source_ids): + _require( + isinstance(source_ids, Mapping) + and set(source_ids) == {"population", "district", "puma"} + and all( + isinstance(v, str) and v and v.strip() == v for v in source_ids.values() + ), + "three explicit nonempty source identities required", + ) + return dict(source_ids) + + +def assemble_atomic_block_support( + *, + block_population: Mapping, + cd_by_block: Mapping, + puma_by_tract: Mapping, + source_ids: Mapping, +) -> bytes: + """Keep each supplied block, with string codes and source-labeled mappings. + + Population is the declared 2020-person sampling proxy. This does not imply + household counts or support for subsequent construction in unpopulated + 2020 blocks. Existing PL parsers supply populated blocks only. Extra mapping + entries may cover unselected areas; every supplied support row must map. + """ + sources = _sources(source_ids) + _require( + all( + isinstance(m, Mapping) + for m in (block_population, cd_by_block, puma_by_tract) + ) + and bool(block_population), + "nonempty block support and mappings required", + ) + for block, population in block_population.items(): + _require( + _integer(block, minimum=10**13, maximum=10**15 - 1) + and int(block) // 10**13 in _STATES, + "invalid block GEOID", + ) + _require(_integer(population), "invalid population weight") + blocks = sorted(int(block) for block in block_population) + pumas, districts = [], [] + for block in blocks: + state, tract = block // 10**13, block // 10000 + puma, district = puma_by_tract.get(tract), cd_by_block.get(block) + _require( + _integer(puma, minimum=1, maximum=9999999) + and int(puma) // 100000 == state + and int(puma) % 100000 > 0, + "missing or inconsistent tract-to-PUMA mapping", + ) + _require( + _integer(district, minimum=100, maximum=5699) + and int(district) // 100 == state + and int(district) % 100 <= 53, + "missing or inconsistent block-to-district mapping", + ) + pumas.append(f"{int(puma):07d}") + districts.append(f"{int(district):04d}") + arrays = { + "area": np.asarray([f"{v:015d}" for v in blocks]), + "state": np.asarray([f"{v // 10**13:02d}" for v in blocks]), + "county": np.asarray([f"{v // 10**10:05d}" for v in blocks]), + "tract": np.asarray([f"{v // 10000:011d}" for v in blocks]), + "puma": np.asarray(pumas), + "district": np.asarray(districts), + "population": np.asarray( + [int(block_population[v]) for v in blocks], dtype=np.int64 + ), + } + columns = { + name: { + "kind": "code", + "source": sources["population"], + "vintage": "2020", + "relation": "exact", + } + for name in ("area", "state", "county", "tract") + } + columns.update( + { + "puma": { + "kind": "code", + "source": sources["puma"], + "vintage": "2020", + "relation": "exact", + }, + "district": { + "kind": "code", + "source": sources["district"], + "vintage": "119th_congress", + "relation": "official_tabulation", + }, + "population": { + "kind": "weight", + "source": sources["population"], + "basis": "2020_census_persons", + }, + } + ) + return encode_atomic_support( + { + "version": 1, + "system": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "columns": columns, + }, + arrays, + ) + + +def assignment_definition( + *, + identity: Sequence[str], + state_column: str, + puma_column: str | None, + source_ids: Mapping, + seed: int, +) -> dict: + """Declare one block draw constrained by normalized observed state/PUMA. + + Input codes must already be strings from the survey's source projection. + Callers choose original source identity columns before cloning; subsequent + clone/prune operators retain the assigned block and derived geographies. + """ + sources = _sources(source_ids) + _require(type(seed) is int and 0 <= seed < 2**63, "invalid seed") + _require(not isinstance(identity, (str, bytes)), "identity must be columns") + constraints = [{"input": state_column, "support": "state", "required": True}] + if puma_column is not None: + constraints.append({"input": puma_column, "support": "puma", "required": False}) + layers = [ + { + "input": name, + "output": output, + "vintage": vintage, + "relation": relation, + "source": sources[source], + } + for name, output, vintage, relation, source in ( + ("state", "assigned_state_fips", "2020", "exact", "population"), + ("county", "county_fips", "2020", "exact", "population"), + ("tract", "census_tract_geoid", "2020", "exact", "population"), + ("puma", "assigned_puma_geoid", "2020", "exact", "puma"), + ( + "district", + "congressional_district_geoid", + "119th_congress", + "official_tabulation", + "district", + ), + ) + ] + return validate_assignment_spec( + { + "version": 1, + "identity": list(identity), + "stream": ["sha256-u53-v1", "us-atomic-geography", 0, seed], + "outputs": { + "area": "census_block_geoid", + "system": "geography_system", + "basis": "geography_assignment_basis", + }, + "systems": [ + { + "id": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "source": SOURCE, + "selector": {}, + "constraints": constraints, + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": layers, + } + ], + } + ) diff --git a/packages/microcosm-build/tests/test_us_atomic_block_support.py b/packages/microcosm-build/tests/test_us_atomic_block_support.py new file mode 100644 index 000000000..182dd4003 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_atomic_block_support.py @@ -0,0 +1,131 @@ +"""Invented Census-shaped mappings through the shared atomic-area contract.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.us_runtime import atomic_block_support as us + + +def inputs(): + blocks = [int(v) for v in ("010010201001000", "010010201001001", "020130001001000")] + return { + "block_population": dict(zip(blocks, (2, 6, 1), strict=True)), + "cd_by_block": dict(zip(blocks, (101, 102, 200), strict=True)), + "puma_by_tract": {blocks[0] // 10000: 100001, blocks[2] // 10000: 200002}, + "source_ids": {k: "invented-" + k for k in ("population", "district", "puma")}, + } + + +def test_source_rows_and_leading_zeroes_are_preserved(): + data = inputs() + before = deepcopy(data) + payload = us.assemble_atomic_block_support(**data) + support = geo.decode_atomic_support(payload) + assert data == before + assert support.arrays["area"].tolist() == [ + "010010201001000", + "010010201001001", + "020130001001000", + ] + assert support.arrays["state"].tolist() == ["01", "01", "02"] + assert support.arrays["county"].tolist() == ["01001", "01001", "02013"] + assert support.arrays["tract"].tolist() == [ + "01001020100", + "01001020100", + "02013000100", + ] + assert support.arrays["puma"].tolist() == ["0100001", "0100001", "0200002"] + assert support.arrays["district"].tolist() == ["0101", "0102", "0200"] + assert np.array_equal(support.arrays["population"], [2, 6, 1]) + assert support.metadata["columns"]["district"]["relation"] == "official_tabulation" + assert support.metadata["columns"]["population"]["basis"] == "2020_census_persons" + for key in ("block_population", "cd_by_block", "puma_by_tract"): + data[key] = dict(reversed(list(data[key].items()))) + assert us.assemble_atomic_block_support(**data) == payload + + +def test_country_declaration_uses_shared_assignment_and_derivation(): + payload = us.assemble_atomic_block_support(**inputs()) + support = geo.decode_atomic_support(payload) + spec = us.assignment_definition( + identity=("source", "source_id"), + state_column="observed_state", + puma_column="observed_puma", + source_ids=inputs()["source_ids"], + seed=17, + ) + households = pd.DataFrame( + { + "source": ["acs", "asec", "acs"], + "source_id": [1, 1, 2], + "observed_state": ["01", "01", "02"], + "observed_puma": ["0100001", None, "0200002"], + } + ) + supports = {us.SYSTEM: support} + assigned = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + result = pd.concat( + [assigned, geo.derive_geography(assigned, spec, supports)], axis=1 + ) + geo.validate_geography(result, spec, supports) + assert result.assigned_state_fips.tolist() == households.observed_state.tolist() + assert result.congressional_district_geoid.iloc[2] == "0200" + assert len(result) == len(households) + pd.testing.assert_frame_equal(result[households.columns], households) + subset = result.iloc[[2, 0]] + geo.validate_geography(subset, spec, supports) + pd.testing.assert_frame_equal( + geo.assign_atomic(households.iloc[[2, 0]], spec, supports), + assigned.drop(columns=households.columns).iloc[[2, 0]], + ) + + +@pytest.mark.parametrize( + "change", + [ + "missing_puma", + "missing_district", + "puma_wrong_state", + "district_wrong_state", + "negative_population", + "boolean_population", + "fractional_population", + "invalid_block", + "boolean_block", + "missing_source", + "empty_source", + ], +) +def test_malformed_or_incomplete_source_mappings_refuse(change): + data = inputs() + block = next(iter(data["block_population"])) + if change == "missing_puma": + data["puma_by_tract"].pop(block // 10000) + elif change == "missing_district": + data["cd_by_block"].pop(block) + elif change == "puma_wrong_state": + data["puma_by_tract"][block // 10000] = 200001 + elif change == "district_wrong_state": + data["cd_by_block"][block] = 201 + elif change == "negative_population": + data["block_population"][block] = -1 + elif change == "boolean_population": + data["block_population"][block] = True + elif change == "fractional_population": + data["block_population"][block] = 1.5 + elif change == "invalid_block": + data["block_population"][10**15] = 1 + elif change == "boolean_block": + data["block_population"][True] = 1 + elif change == "missing_source": + data["source_ids"].pop("district") + elif change == "empty_source": + data["source_ids"]["population"] = " " + with pytest.raises(ValueError, match="US atomic support:"): + us.assemble_atomic_block_support(**data) From c1cee78e326c38987a742817d7dbb05925076a34 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:18:53 -0400 Subject: [PATCH 030/255] Place atomic geography before the survey support clone --- docs/geography-assignment.md | 14 ++ .../us-atomic-clone-graph-20260909.json | 33 ++++ .../us_runtime/graph_atomic_survey_clone.py | 44 +++++ .../test_us_atomic_survey_clone_graph.py | 171 ++++++++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 experiments/us-atomic-clone-graph-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py create mode 100644 packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index 20d7c30cb..7992f3dcf 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -176,3 +176,17 @@ refusals and subset-stable assignment. Receipt SHA256: `404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. Actual normalized block support and attachment before the survey clone remain pending; these tests do not establish native geographic fit. + +`us_runtime/graph_atomic_survey_clone.py` now declares the combined sequence: +shared import, assignment, derivation and integrity gate, then the existing +combined-survey support clone and an inherited-mapping gate. The graph compiler +makes the clone depend on every member of its base version, including the +pre-clone gate. The post-clone gate never draws another location. An actual +executor control passes cold execution and required replay: four invented +households become eight, every location column is inherited, all entity rows +are copied, and each household weight is split equally across the pair. +Subset mapping verification also passes. Receipt SHA256: +`1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. +A separate source-only Codex review found no actionable defects. Native observed +column qualification and geography-aware reconstruction in the survey runner +and origin-budget verifier remain pending. diff --git a/experiments/us-atomic-clone-graph-20260909.json b/experiments/us-atomic-clone-graph-20260909.json new file mode 100644 index 000000000..a30a95cfe --- /dev/null +++ b/experiments/us-atomic-clone-graph-20260909.json @@ -0,0 +1,33 @@ +{ + "all_location_columns_inherited": "passed", + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 4.555835, + "household_weight_halves": "passed", + "households_after": 8, + "households_before": 4, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 364150784, + "persons_after": 12, + "persons_before": 6, + "preclone_gate_order": "passed", + "production_source_sha256": "ef9b562b355abc450f3d2934ab9be04e49387d03de4b39e37fa09b1c0c1905c8", + "receipt_sha256": "1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented combined-survey actual graph: atomic assignment and derivation before support clone, inherited mapping gate and required replay. Native source and runner integration remain pending.", + "source_and_owned_files": 481, + "source_only_peer_review": "Codex peer found no actionable findings in adapter, composition and tests; no independent runtime or native verification.", + "subset_mapping_check": "passed", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.214872374985134 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py new file mode 100644 index 000000000..84caad516 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py @@ -0,0 +1,44 @@ +"""Compose shared geography before the combined-survey support clone. + +This declaration has no country-specific geography kernel or source admission. +Callers provide normalized, qualified observed columns and declared support. +""" + +from dataclasses import replace + +from microcosm.build.graph_atomic_geography import atomic_geography_nodes + +from .graph_combined_clone import us_combined_survey_clone_nodes + + +def atomic_survey_clone_nodes( + definition, + columns, + *, + base, + geography_prefix="geography", + clone_prefix="combined_survey_puf_support_clone", +): + """Assign once on the base, inherit every location, then recheck the clone. + + Structural graph nodes depend on every member of their base version. The + pre-clone integrity gate therefore precedes EXPAND even though it owns no + cells. The post-clone gate derives mappings without drawing again. + """ + columns = tuple(columns) + geography = atomic_geography_nodes( + definition, columns, base=base, prefix=geography_prefix + ) + clones = us_combined_survey_clone_nodes( + (*columns, *(output for node in geography for output in node.outputs)), + base=base, + source_channels=("acs", "asec"), + prefix=clone_prefix, + ) + final_gate = replace( + geography[-1], + id=f"{geography_prefix}.clone_gate", + population=clones[0].id, + description="Recheck inherited atomic geography on every support clone.", + ) + return (*geography, *clones, final_gate) diff --git a/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py b/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py new file mode 100644 index 000000000..43e9dda28 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py @@ -0,0 +1,171 @@ +"""Invented two-arm population through the actual geography/clone executor.""" + +import numpy as np +import pandas as pd + +from microcosm.build import atomic_geography as geo +from microcosm.build.graph_atomic_geography import register_atomic_geography_kernels +from microcosm.build.us_runtime import atomic_block_support as us +from microcosm.build.us_runtime.graph_atomic_survey_clone import ( + atomic_survey_clone_nodes, +) +from microcosm.build.us_runtime.graph_combined_clone import ( + register_us_combined_survey_clone_kernels, +) +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.population import dtype_for_token + + +def invented_population(): + arms = {} + string = dtype_for_token("string") + for channel, state, puma in (("asec", "01", None), ("acs", "02", "0200002")): + person = pd.DataFrame( + { + "person_id": [1, 2, 3], + "age": [40, 10, 50], + **{ + f"person_{entity}_id": [11, 11, 42] + for entity in US_SCHEMA.group_entities + }, + } + ) + tables = { + "person": person, + **{ + entity: pd.DataFrame({f"{entity}_id": [11, 42]}) + for entity in US_SCHEMA.group_entities + }, + } + tables["household"]["observed_state"] = pd.Series([state] * 2, dtype=string) + tables["household"]["observed_puma"] = pd.Series([puma] * 2, dtype=string) + arms[channel] = Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([3.0, 7.0]), WeightKind.DESIGN)}, + pd.Series([channel] * 3, dtype=string), + ) + frame = assemble_spines(arms, household_mass_shares={"asec": 0.6, "acs": 0.4}) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for table in tables.values(): + for name in table: + if not pd.api.types.is_numeric_dtype(table[name]): + table[name] = table[name].astype(string) + return Frame( + tables, + US_SCHEMA, + {"household": frame.weights_for("household")}, + frame.strata.astype(string), + metadata=frame.metadata, + ) + + +class InventedHost(KernelBase): + ref = "invented.us_atomic_host@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + assert context.sources["host"].read_bytes() == b"invented literal host v1" + return KernelResult(frame=invented_population()) + + +def test_block_assignment_precedes_clone_and_survives_required_replay(tmp_path): + source_ids = {k: "invented-" + k for k in ("population", "district", "puma")} + blocks = [int(x) for x in ("010010201001000", "010010201001001", "020130001001000")] + payload = us.assemble_atomic_block_support( + block_population=dict(zip(blocks, (2, 6, 1), strict=True)), + cd_by_block=dict(zip(blocks, (101, 102, 200), strict=True)), + puma_by_tract={blocks[0] // 10000: 100001, blocks[2] // 10000: 200002}, + source_ids=source_ids, + ) + spec = us.assignment_definition( + identity=("household_support_channel", "household_spine_source_id"), + state_column="observed_state", + puma_column="observed_puma", + source_ids=source_ids, + seed=17, + ) + original = invented_population() + columns = frame_column_declarations(original) + stages = atomic_survey_clone_nodes(spec, columns, base="host") + graph = compile_graph( + Graph( + "us", + ( + SourceRef("host", "raw-bytes-v1"), + SourceRef(us.SOURCE, "raw-bytes-v1"), + ), + ( + Node( + "host", + InventedHost.ref, + structural=StructuralDelta.CREATE, + sources=("host",), + outputs=columns, + ), + *stages, + ), + ) + ) + clone_id = "combined_survey_puf_support_clone" + assert graph.order.index("geography.gate") < graph.order.index(clone_id) + assert graph.order.index(clone_id) < graph.order.index("geography.clone_gate") + registry = KernelRegistry() + registry.register(InventedHost()) + register_atomic_geography_kernels(registry) + register_us_combined_survey_clone_kernels(registry) + host_path, support_path = tmp_path / "host.bin", tmp_path / "support.npz" + host_path.write_bytes(b"invented literal host v1") + support_path.write_bytes(payload) + sources = {"host": host_path, us.SOURCE: support_path} + store = ContentStore(tmp_path / "store") + cold = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph( + graph, sources=sources, store=store, kernels=registry, resume="require" + ) + for manifest in (cold, warm): + before, after = manifest.population("host"), manifest.population(clone_id) + hh = after.table("household") + assert len(hh) == 8 and len(after.person) == 12 + native = hh[hh.household_support_clone_index.eq(0)] + copies = hh[hh.household_support_clone_index.eq(1)] + geo_columns = [ + *spec["outputs"].values(), + *(x["output"] for x in spec["systems"][0]["layers"]), + ] + for column in geo_columns: + assert ( + native[column].tolist() + == copies[column].tolist() + == before.table("household")[column].tolist() + ) + for entity in original.entities: + assert len(after.table(entity)) == 2 * len(original.table(entity)) + np.testing.assert_array_equal( + after.weights_for("household").values, + np.tile(before.weights_for("household").values / 2, 2), + ) + assert manifest.nodes["geography.clone_gate"].receipt["outcome"] == "pass" + geo.validate_geography( + hh.iloc[[6, 1]], spec, {us.SYSTEM: geo.decode_atomic_support(payload)} + ) + for node in stages: + assert cold.nodes[node.id].key == warm.nodes[node.id].key From 2f18fe857e8db2917710363637092ecbedd2efd8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:45:04 -0400 Subject: [PATCH 031/255] Verify PUF55 whole-population attachment and replay --- .../puf55-survey-social-security.added.md | 2 +- changelog.d/shared-atomic-geography.added.md | 2 +- docs/current-survey-puf59-progress.md | 13 + ...us-puf55-population-controls-20260909.json | 27 ++ .../tests/test_us_graph_puf55_survey_ss.py | 349 ++++++++++++++++++ 5 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 experiments/us-puf55-population-controls-20260909.json create mode 100644 packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md index 1efdab5aa..ba625b57e 100644 --- a/changelog.d/puf55-survey-social-security.added.md +++ b/changelog.d/puf55-survey-social-security.added.md @@ -1 +1 @@ -Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. Validate canonical donor projection and whole-Population attachment/replay while retaining source identities, weights and Social Security unknownness. diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md index a269c08ca..25a10460d 100644 --- a/changelog.d/shared-atomic-geography.added.md +++ b/changelog.d/shared-atomic-geography.added.md @@ -1 +1 @@ -Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index bddbdea27..6ee1a2fcb 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -27,3 +27,16 @@ The new canonical59-to-PUF55 donor adapter also passes the maintained invented-s The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. + +The PUF55 profile now also passes three invented whole-Population attachment +controls. They execute all 55 fitted outcomes through the actual graph, typed +artifact loader and materialized population verifier, followed by Frame storage +and required replay. Every non-owned field, Social Security bit pattern and +unknown cell, membership, design anchor and household weight remains intact. +Two negative cases reject changed Social Security values and unknown-to-zero +conversions even when predictor matrices remain identical. The run took 21.03 +seconds at 631.3 MB peak RSS; receipt SHA256: +`ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. +All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected access refusals. Native recipient qualification, +measurement comparability and actual survey attachment remain pending. diff --git a/experiments/us-puf55-population-controls-20260909.json b/experiments/us-puf55-population-controls-20260909.json new file mode 100644 index 000000000..b2b9f093e --- /dev/null +++ b/experiments/us-puf55-population-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 18.73299, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 631308288, + "predictor_count": 9, + "receipt_sha256": "ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented PUF55 whole-Population graph attachment and required replay; native recipient qualification remains pending.", + "source_and_owned_files": 482, + "ss_incumbent_bits_and_unknownness_preserved": true, + "tampered_ss_population_refusals": 2, + "target_count": 55, + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "unexpected_refusals": {}, + "wall_seconds": 21.034625833010068 +} diff --git a/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py b/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py new file mode 100644 index 000000000..015c3fa07 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py @@ -0,0 +1,349 @@ +"""Invented PUF55 whole-Population attachment and required replay controls. + +These tests use the maintained graph, QRF, finalizer and materialized verifier. +They neither admit native sources nor establish production or release acceptance. +""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_enrichment import InventedSource +from test_us_full_puf_output_profiles import InventedProfileMatrix +from test_us_graph_full_puf_enrichment import ( + InventedCompleteBoundary, + _copy_population, + _load_artifact, + _verify, +) +from test_us_puf55_survey_ss_profile import SS, SS_TOTAL, _donor55, _frame, _known55 + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.build.us_runtime import puf_support as support +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ContentStore, + Graph, + KernelRegistry, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph import population as population_ops +from microcosm.graph.codecs import load_frame_store + +PROFILE = full.PUF55_SURVEY_SS + + +@pytest.fixture(scope="module") +def attached_puf55(tmp_path_factory): + root = tmp_path_factory.mktemp("puf55_population_attachment") + frame, donor = _frame(), _donor55() + known = _known55(frame) + prepared = full.prepare_full_puf_inputs( + frame, donor, predictor_known=known, profile=PROFILE + ) + decoded = model_input.decode_recipient_matrix(prepared.matrix) + table = decoded.features.copy() + table.insert(0, "tax_unit_id", decoded.entity_ids) + matrix_frame = Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + { + "person_id": decoded.entity_ids, + "person_tax_unit_id": decoded.entity_ids, + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.ones(len(table)), WeightKind.DESIGN)}, + ) + retained_store = ContentStore( + root / "retained", codecs={"frame-store": load_frame_store} + ) + sources, source_nodes = {}, [] + for name, source_frame in ( + ("survey", frame), + ("donor", prepared.donor_frame), + ("matrix_input", matrix_frame), + ): + source_name = "invented." + name + sources[source_name] = retained_store.put_frame( + full.codec.sha(("ordinary-puf55-attachment:" + name).encode()), + source_frame, + ) + outputs = ( + frame_column_declarations(source_frame) + if name == "survey" + else tuple( + Owned("tax_unit", column, "float64") + for column in source_frame.table("tax_unit") + if column != "tax_unit_id" + ) + ) + source_nodes.append( + Node( + name, + InventedSource.ref, + sources=(source_name,), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + ) + boundary = Node( + "complete_upstream", + InventedCompleteBoundary.ref, + base="survey", + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + ) + matrix_node = Node( + "matrix", + InventedProfileMatrix.ref, + population="matrix_input", + inputs=(Slice("tax_unit", PROFILE.predictors),), + artifact_outputs=(ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE),), + ) + base_graph = Graph( + "us", + tuple(SourceRef(name, "frame-store") for name in sources), + (*source_nodes, boundary, matrix_node), + ) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedCompleteBoundary(), + InventedProfileMatrix(), + LegacyQRFTrainKernel(), + LegacyQRFApplyMatrixKernel(), + ): + kernels.register(kernel) + observed = {} + retained_manifest = run_graph( + compile_graph(base_graph), + sources=sources, + store=retained_store, + kernels=kernels, + _population_observer=lambda name, value: observed.__setitem__(name, value), + ) + upstream = observed[boundary.id] + matrix = _load_artifact( + retained_manifest, + retained_store, + ArtifactInput( + "matrix", matrix_node.id, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + assert matrix.payload == prepared.matrix + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population=boundary.id, + matrix_producer=matrix_node.id, + seed=578, + n_estimators=2, + zero_atol=0, + profile=PROFILE, + ) + binding = placement.retain_full_puf_attachment( + population=upstream, + expected_population=_copy_population(upstream), + population_node=boundary, + input_owners=dict(upstream.owners), + donor=donor, + predictor_known=known, + matrix=matrix, + fit_nodes=fits, + apply_nodes=applies, + profile=PROFILE, + ) + nodes = placement.full_puf_attachment_nodes(binding) + kernels.register(placement.FullPufMaskKernel(binding)) + kernels.register(placement.FullPufAttachKernel(binding)) + compiled = compile_graph( + replace(base_graph, nodes=(*base_graph.nodes, *fits, *applies, *nodes)) + ) + store = ContentStore(root / "application", codecs={"frame-store": load_frame_store}) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=lambda name, value, target=observed: ( + target.__setitem__(name, value) + ), + ) + assert all( + record.hit is (resume == "require") for record in manifest.nodes.values() + ) + artifacts, producer_keys = placement.load_full_puf_attachment_artifacts( + binding, compiled=compiled, manifest=manifest, store=store + ) + actual_upstream, actual = observed[boundary.id], observed[nodes[-1].id] + evidence = placement.verify_materialized_full_puf_attachment( + binding, + upstream_population=actual_upstream, + population=actual, + artifacts=artifacts, + producer_keys=producer_keys, + ) + # Exercise full Frame storage as well as required column-artifact replay. + frame_key = full.codec.sha( + ("puf55-materialized:" + manifest.node(nodes[-1].id).key).encode() + ) + store.put_frame(frame_key, actual.frame) + replay.same_replayed_frame(actual.frame, store.load_frame(frame_key)) + results.append( + SimpleNamespace( + manifest=manifest, + upstream=_copy_population(actual_upstream), + population=_copy_population(actual), + artifacts=artifacts, + producer_keys=producer_keys, + evidence=evidence, + ) + ) + replay.same_replayed_population(results[0].population, results[1].population) + assert results[0].artifacts == results[1].artifacts + assert results[0].manifest.key == results[1].manifest.key + return SimpleNamespace(binding=binding, nodes=nodes, fits=fits, results=results) + + +def test_puf55_cold_and_required_replay_preserve_survey_incumbents(attached_puf55): + case = attached_puf55 + attach = case.nodes[-1] + assert len(case.fits) == len(PROFILE.targets) == 55 + assert len(PROFILE.predictors) == 9 + assert PROFILE.predictors == (*full.PUF59.predictors, SS_TOTAL) + assert tuple(owned.column for owned in attach.outputs) == PROFILE.targets + assert not set(SS) & {owned.column for owned in attach.outputs} + assert not set(SS) & set(case.binding.donor) + owned_cells = {(owned.entity, owned.column) for owned in attach.outputs} + added_cells = owned_cells | set(placement.MASKS.items()) + + for result in case.results: + evidence = result.evidence + assert evidence["complete_population_compared"] is True + assert evidence["output_profile"] == PROFILE.value + assert evidence["target_order"] == list(PROFILE.targets) + assert evidence["predictor_order"] == list(PROFILE.predictors) + assert evidence["target_count"] == 55 and evidence["tail_bounds"] + assert evidence["source_admission_issued"] is False + assert evidence["release_eligible"] is False + decoded = model_input.decode_recipient_matrix( + result.artifacts["matrix"].payload + ) + assert tuple(decoded.features) == PROFILE.predictors + np.testing.assert_array_equal( + decoded.entity_ids, case.binding.predictor_known.index + ) + + frame, upstream = result.population.frame, result.upstream.frame + assert frame.schema == upstream.schema and frame.entities == upstream.entities + assert frame.links == upstream.links and frame.metadata == upstream.metadata + assert frame.mass_log == upstream.mass_log + assert result.population.mass_ledger == result.upstream.mass_ledger + assert result.population.weight_kind == result.upstream.weight_kind + assert ( + result.population.design_weights.keys() + == result.upstream.design_weights.keys() + ) + pd.testing.assert_series_equal(frame.strata, upstream.strata, check_exact=True) + for entity in upstream.weighted_entities: + before, after = upstream.weights_for(entity), frame.weights_for(entity) + assert before.kind is after.kind + np.testing.assert_array_equal(before.values, after.values) + for entity, values in result.upstream.design_weights.items(): + np.testing.assert_array_equal( + values, result.population.design_weights[entity] + ) + for cell, owner in result.upstream.owners.items(): + if cell not in added_cells: + assert result.population.owners[cell] == owner + for entity in upstream.entities: + before, after = upstream.table(entity), frame.table(entity) + pd.testing.assert_index_equal(before.index, after.index, exact=True) + for column in before: + if (entity, column) not in owned_cells: + assert population_ops.storage_equal(before[column], after[column]) + + for owned in attach.outputs: + before, after = upstream.table(owned.entity), frame.table(owned.entity) + mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) + assert owned.rows == placement.MASKS[owned.entity] + assert after.loc[mask, owned.column].notna().all() + assert result.population.owners[(owned.entity, owned.column)] == attach.id + if owned.column in before: + assert population_ops.storage_equal( + before[owned.column], after[owned.column], ~mask + ) + else: + assert after.loc[~mask, owned.column].isna().all() + + people = upstream.table("person") + mask = support.puf_tax_detail_clone_mask(people, entity="person") + for component in SS: + # Nonzero incumbents and unknownness exist on both support arms. + for selected in (mask, ~mask): + assert people.loc[selected, component].isna().any() + assert people.loc[selected, component].gt(0).any() + np.testing.assert_array_equal( + people[component].to_numpy().view("uint64"), + frame.table("person")[component].to_numpy().view("uint64"), + ) + np.testing.assert_array_equal( + upstream.table("tax_unit")[SS_TOTAL].to_numpy().view("uint64"), + frame.table("tax_unit")[SS_TOTAL].to_numpy().view("uint64"), + ) + + +@pytest.mark.parametrize("mutation", ("observed_value", "unknown_to_zero")) +def test_puf55_required_population_verifier_rejects_ss_changes( + attached_puf55, mutation +): + case, result = attached_puf55, attached_puf55.results[-1] + changed = _copy_population(result.population) + people = changed.frame.table("person") + mask = support.puf_tax_detail_clone_mask(people, entity="person") + component = SS[0] + selected = mask & ( + people[component].gt(0) + if mutation == "observed_value" + else people[component].isna() + ) + row = people.index[selected][0] + people.loc[row, component] = ( + people.loc[row, component] + 1.0 if mutation == "observed_value" else 0.0 + ) + # SS incumbents are outside the model surface. Equal predictors cannot excuse + # changing their values or converting an unknown recipient benefit to zero. + matrices = tuple( + full.prepare_full_puf_inputs( + population.frame, + case.binding.donor, + predictor_known=case.binding.predictor_known, + profile=PROFILE, + ).matrix + for population in (result.population, changed) + ) + assert matrices[0] == matrices[1] + with pytest.raises(ValueError, match="SURVEY_POPULATION_REPLAY_"): + _verify(case, result, population=changed) From 0760c09d908ccee2dd36ba638000b4f75960f7d2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 18:03:13 -0400 Subject: [PATCH 032/255] Qualify observed survey geography and verify its graph projection --- changelog.d/shared-atomic-geography.added.md | 2 +- docs/current-survey-puf59-progress.md | 34 +- docs/survey-social-security-source.md | 49 +++ ...us-puf55-population-controls-20260909.json | 16 +- ...vey-geography-graph-controls-20260909.json | 21 ++ ...ey-geography-source-controls-20260909.json | 21 ++ .../us_runtime/current_survey_geography.py | 313 ++++++++++++++++ .../graph_current_survey_geography.py | 277 ++++++++++++++ .../test_us_current_asec_demographics.py | 4 +- .../tests/test_us_current_survey_geography.py | 192 ++++++++++ .../test_us_graph_current_survey_geography.py | 353 ++++++++++++++++++ .../tests/test_us_graph_puf55_survey_ss.py | 41 ++ 12 files changed, 1311 insertions(+), 12 deletions(-) create mode 100644 experiments/us-survey-geography-graph-controls-20260909.json create mode 100644 experiments/us-survey-geography-source-controls-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_geography.py create mode 100644 packages/microcosm-build/tests/test_us_graph_current_survey_geography.py diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md index 25a10460d..4eed5d1a9 100644 --- a/changelog.d/shared-atomic-geography.added.md +++ b/changelog.d/shared-atomic-geography.added.md @@ -1 +1 @@ -Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings, qualify observed ACS/ASEC geography against source receipts, and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md index 6ee1a2fcb..a4351354a 100644 --- a/docs/current-survey-puf59-progress.md +++ b/docs/current-survey-puf59-progress.md @@ -38,5 +38,35 @@ conversions even when predictor matrices remain identical. The run took 21.03 seconds at 631.3 MB peak RSS; receipt SHA256: `ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, -with no skips or unexpected access refusals. Native recipient qualification, -measurement comparability and actual survey attachment remain pending. +with no skips or unexpected access refusals. A subsequent independent review +required a direct finalizer comparison, because the attachment and its verifier +share a column-construction helper. The strengthened three-case run also passes: +all 55 attached outputs agree with the direct finalizer after independent entity +ID alignment, on cold and required replay. It took 20.73 seconds at 630.5 MB; +receipt SHA256 is +`e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Native recipient qualification, measurement comparability and actual survey +attachment remain pending. + +The observed-geography source qualifier passes 11 invented original-source +controls. It binds current ASEC state and ACS state/PUMA to literal source +household keys, preserves unknown locations, and keeps draw keys stable across +sampling fractions. The initial ten-case run correctly refused a smaller sample +with no positive ASEC support; its corrected fixture supplies positive support +before issuance. Review also added a receipt-digest check against mutation of +the detached ASEC projection. The passing run took 182.50 seconds at 430.3 MB; +receipt SHA256 is +`1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08`. +The 484 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected refusals. + +The observed-geography graph node separately passes 14 controls over the same +kind of invented original sources. Actual three-node execution and required +replay preserve inputs, weights and unknown geography. Declaration, source +receipt, typed-artifact and final-mutation cases refuse invalid output. The run +took 319.44 seconds at 439.1 MB; receipt SHA256 is +`28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Full geography-before-clone composition, native block-source integration and +complete calibration remain separate work. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md index e7bd1cbfe..ea9cfa1ce 100644 --- a/docs/survey-social-security-source.md +++ b/docs/survey-social-security-source.md @@ -60,6 +60,55 @@ convention, allocation to individual beneficiaries where necessary, explicit income periods, design-weighted training, native-to-clone inheritance and held-out checks remain separate required work. +## Conditioning measurement judgment, 2026-09-09 + +The recipient measurement for predictor nine remains undecided. Passing the +numerical `PUF55_SURVEY_SS` mechanism does not establish donor/recipient +comparability or authenticate a beneficiary total. + +[IRS Publication 915 (2015), pages 3 and 5–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=3) +combines spouses' benefits on a joint return and assigns benefits to the person +legally entitled to receive them. A child's payment remains the child's even +when the check names the parent. Form 1040 line 20a reports net benefits after +repayments, including equivalent tier-1 railroad benefits; Form 1040EZ does not +report benefits. Thus a sum over all modeled tax-unit members is a different +measurement from the filer/joint-spouse return amount. + +Restricting survey records to the filer and joint-return spouse would still +produce a **report-sum proxy**, not an observed beneficiary total. The +[ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and includes reasons for payments on behalf +of children. The [ACS 2024 questionnaire, question 43, page 18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +also permits jointly received income to appear entirely on one person's +record, with no receipt recorded for the other. Its income category includes +Social Security or Railroad Retirement. A parent may therefore report child +payments, or another household member may report an included person's income. +Multiple tax units within one household make reporter membership insufficient +to establish benefit ownership; household totals must not be duplicated across +returns or redistributed without a separate supported model. + +If adopted, a measurement such as +`social_security_filer_joint_spouse_report_sum_proxy` would need an explicit +upstream contract before supplying `puf_conditioning_social_security_total`. +That contract would bind actual return roles, source reporting records, +knownness and income periods; the coarsened PUF filing-class predictor alone +cannot establish joint-return membership. ASEC calendar-year income, ACS's +rolling window, transported donor money, repayment treatment and railroad +coverage remain comparability questions. Component completion is separate. + +Excluding a dependent from a parent's return feature does not assign that +dependent zero benefits. Both surveys' under-15 amounts remain unknown. If an +included filer has no in-universe report, an empty or missing report set cannot +silently become a known zero. Family-payment ambiguity also survives an +age-universe restriction. + +The exact 2015 `E02400` definition and publisher editing/omission conventions +remain unauthenticated by this lookup. The [live IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files), +checked on 2026-09-09, says the 2012–2015 files are unavailable. The verified +Form 1040 rules do not independently establish how every PUF record encodes +them. This record adopts no proxy, allocates no beneficiaries and grants no +dataset or release acceptance. + ## Verification scope The 52-case control run executes 50 numerical/literal tests and two complete diff --git a/experiments/us-puf55-population-controls-20260909.json b/experiments/us-puf55-population-controls-20260909.json index b2b9f093e..dc8cb5d14 100644 --- a/experiments/us-puf55-population-controls-20260909.json +++ b/experiments/us-puf55-population-controls-20260909.json @@ -2,26 +2,28 @@ "before_after_current_frozen_and_maintained_source_equal": true, "before_after_current_model_and_code_resources_equal": true, "code_resources": 12, - "cpu_seconds": 18.73299, + "cpu_seconds": 18.064019, + "independent_selected_value_oracle": "all 55 direct finalizer outputs compared by entity id", "model_source_files": 5983, "native_payloads_read": 0, - "peak_rss_bytes": 631308288, + "peak_rss_bytes": 630538240, "predictor_count": 9, - "receipt_sha256": "ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30", + "previous_receipt_sha256": "ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30", + "receipt_sha256": "e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291", "release_eligible": false, "required_replay": "passed", "result": 0, - "scope": "Invented PUF55 whole-Population graph attachment and required replay; native recipient qualification remains pending.", - "source_and_owned_files": 482, + "scope": "Invented PUF55 whole-Population attachment/replay and independent direct-finalizer oracle; no native recipient qualification or release acceptance.", + "source_and_owned_files": 486, "ss_incumbent_bits_and_unknownness_preserved": true, "tampered_ss_population_refusals": 2, "target_count": 55, - "tests": { + "test_counts": { "errors": 0, "failures": 0, "skipped": 0, "tests": 3 }, "unexpected_refusals": {}, - "wall_seconds": 21.034625833010068 + "wall_seconds": 20.73042149998946 } diff --git a/experiments/us-survey-geography-graph-controls-20260909.json b/experiments/us-survey-geography-graph-controls-20260909.json new file mode 100644 index 000000000..6952299e9 --- /dev/null +++ b/experiments/us-survey-geography-graph-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 313.767356, + "model_source_files": 5983, + "peak_rss_bytes": 439091200, + "receipt_sha256": "28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997", + "release_eligible": false, + "result": 0, + "scope": "Observed-geography graph node over actual invented original-source issuers: cold/replay/unknownness/binding/mutation controls. No native atomic assignment or release acceptance.", + "source_and_owned_files": 486, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 14 + }, + "unexpected_refusals": {}, + "wall_seconds": 319.43621275003534 +} diff --git a/experiments/us-survey-geography-source-controls-20260909.json b/experiments/us-survey-geography-source-controls-20260909.json new file mode 100644 index 000000000..f34e40267 --- /dev/null +++ b/experiments/us-survey-geography-source-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_and_frozen_source_control_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 178.432832, + "model_source_files": 5983, + "peak_rss_bytes": 430325760, + "receipt_sha256": "1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08", + "release_eligible": false, + "result": 0, + "scope": "Invented source qualification only; no native atomic assignment, engine, data or release acceptance.", + "source_and_control_files": 484, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 11 + }, + "unexpected_refusals": {}, + "wall_seconds": 182.49943441699725 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py new file mode 100644 index 000000000..f463b6412 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py @@ -0,0 +1,313 @@ +"""Qualified observed geography and stable household draw keys for both surveys. + +This additive, values-only projection borrows current authenticated sources. +It does not assign a location, mutate a Frame, or issue source or Population +authority. Hosts must requalify the projection before execution and replay. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass + +import pandas as pd + +from microcosm.graph.population import dtype_for_token + +from . import current_asec_demographics as demographics +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-survey-geography.v1" +COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) +MAX_HOUSEHOLDS = 64 * 1024**2 // 128 +MAX_RECEIPT_BYTES = 64 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_GEOGRAPHY_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _origin_key(row): + channel = row["source"] + _require(type(channel) is str and channel in {"acs", "asec"}, "SOURCE") + _require( + type(row["source_year"]) is int + and row["source_year"] == 2024 + and type(row["survey_year"]) is int + and row["survey_year"] == (2024 if channel == "acs" else 2025), + "SOURCE_PERIOD", + ) + raw = row["raw_native_id"] + _require(type(raw) is str and 0 < len(raw) <= 128, "NATIVE_KEY") + # Preserve the source literal, including ASEC's leading zeroes. Receiving + # integer ids are coordinates only and never enter the keyed draw stream. + return json.dumps( + [channel, row["source_year"], row["survey_year"], raw], + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def _project(origins, households, acs, asec): + """Pure alignment from just-qualified values, with no authority of its own.""" + _require( + type(origins) is list + and 0 < len(origins) <= MAX_HOUSEHOLDS + and len(origins) == len(households), + "HOUSEHOLD_COUNT", + ) + _require( + str(households.household_id.dtype) == "int64" + and households.household_id.is_unique + and households.household_id.ge(0).all(), + "HOUSEHOLD_AXIS", + ) + channel_column = support_channel_column("household") + native_column = spine_source_id_column("household") + _require(set(households[channel_column]) == {"acs", "asec"}, "CHANNEL_ROSTER") + _require( + {"household_id", "SERIALNO", "ST", "PUMA", "puma_geoid", "puma"} <= set(acs) + and acs.household_id.is_unique, + "ACS_SOURCE_COLUMNS", + ) + _require( + { + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + "state_known", + } + <= set(asec) + and asec.household_id.is_unique, + "ASEC_SOURCE_COLUMNS", + ) + acs_by_id = acs.set_index("household_id", drop=False) + asec_by_id = asec.set_index("household_id", drop=False) + origin_by_id, keys = {}, set() + for row in origins: + _require(type(row) is dict, "ORIGIN_RECORD") + household_id = row["household_id"] + _require( + type(household_id) is int + and household_id >= 0 + and household_id not in origin_by_id + and type(row["selected_receiving_household_id"]) is int + and row["selected_receiving_household_id"] >= 0, + "ORIGIN_COORDINATE", + ) + key = _origin_key(row) + _require(key not in keys, "ORIGIN_DUPLICATE") + keys.add(key) + origin_by_id[household_id] = (row, key) + _require(set(origin_by_id) == set(households.household_id), "ORIGIN_ROSTER") + rows, used_acs, used_asec = [], set(), set() + for household_id, channel, native_id in households[ + ["household_id", channel_column, native_column] + ].itertuples(index=False, name=None): + row, key = origin_by_id[household_id] + _require( + channel == row["source"] + and native_id == row["selected_receiving_household_id"], + "RECEIVING_ORIGIN_JOIN", + ) + puma = None + if channel == "acs": + _require(native_id in acs_by_id.index, "ACS_NATIVE_JOIN") + original = acs_by_id.loc[native_id] + _require(original.SERIALNO == row["raw_native_id"], "ACS_LITERAL_KEY") + state, local_puma = original.ST, original.PUMA + _require( + type(state) is str + and re.fullmatch(r"[0-9]{2}", state, re.ASCII) is not None + and int(state) > 0 + and type(local_puma) is str + and re.fullmatch(r"[0-9]{5}", local_puma, re.ASCII) is not None + and int(local_puma) > 0, + "ACS_LITERAL_GEOGRAPHY", + ) + puma = state + local_puma + _require( + original.puma_geoid == puma and original.puma == puma, + "ACS_PUMA_IDENTITY", + ) + used_acs.add(native_id) + else: + _require(household_id in asec_by_id.index, "ASEC_RECEIVING_JOIN") + original = asec_by_id.loc[household_id] + raw = row["raw_native_id"] + _require( + re.fullmatch(r"[0-9]+", raw, re.ASCII) is not None + and int(raw) == original.H_SEQ_integer + and native_id == original.native_household_id, + "ASEC_LITERAL_KEY", + ) + code, status = demographics._state_token(original.GESTFIPS) + _require( + original.state_status == status + and bool(original.state_known) == (code is not None) + and ( + pd.isna(original.state_fips) + if code is None + else original.state_fips == code + ), + "ASEC_STATE_IDENTITY", + ) + state = None if code is None else str(code).zfill(2) + used_asec.add(household_id) + rows.append((int(household_id), key, state, puma)) + _require( + used_acs == set(acs.household_id) and used_asec == set(asec.household_id), + "COMPLETE_NATIVE_ROSTER", + ) + return tuple(rows) + + +def _projection_digest(household): + """Stream bounded row encodings; the receipt never contains source keys.""" + _require( + tuple(household.columns) == COLUMNS + and household.index.name == "household_id" + and str(household.index.dtype) == "int64" + and household.index.is_unique + and 0 < len(household) <= MAX_HOUSEHOLDS + and all(household[c].dtype == dtype_for_token("string") for c in COLUMNS), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for row in household.itertuples(index=True, name=None): + values = [int(row[0]), *(None if pd.isna(v) else v for v in row[1:])] + encoded = source._encode(values, maximum=4096) + digest.update(len(encoded).to_bytes(4, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _check_asec_projection(values, receipt): + """Bind the detached household values to their producer's exact receipt.""" + _require(type(receipt) is bytes and values.receipt == receipt, "ASEC_RECEIPT") + document = json.loads(receipt) + _require( + _sha(values.household.reset_index(drop=True).to_json(orient="table").encode()) + == document.get("household_projection_sha256"), + "ASEC_PROJECTION_DIGEST", + ) + + +@dataclass(frozen=True) +class CurrentSurveyGeographyValues: + """Detached values; retained output bytes grant no downstream admission.""" + + household: pd.DataFrame + receipt: bytes + + +def qualify_current_survey_geography(preparation): + """Read current source geography, aligned to selected receiving households.""" + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + origins = json.loads(entry[1])["origins"]["households"] + acs_native, acs_frame = state.native[0], state.source_frames[0] + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + acs_owned = source.acs_native._owned(acs_native) + acs_payload = acs_owned.payload + _require(json.loads(acs_payload)["vintage"] == 2024, "ACS_NATIVE_PERIOD") + asec = demographics.qualify_current_asec_demographics(preparation) + asec_receipt = asec.receipt + _check_asec_projection(asec, asec_receipt) + projected = _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + table = pd.DataFrame( + { + column: pd.array( + [row[i + 1] for row in projected], dtype=dtype_for_token("string") + ) + for i, column in enumerate(COLUMNS) + }, + index=pd.Index( + [row[0] for row in projected], name="household_id", dtype="int64" + ), + ) + projection_sha256 = _projection_digest(table) + receipt = source._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": _sha(entry[1]), + "acs_native_sha256": _sha(acs_payload), + "asec_demographic_sha256": _sha(asec_receipt), + "projection_sha256": projection_sha256, + "columns": list(COLUMNS), + "households": len(table), + "acs_households": len(acs_frame.table("household")), + "asec_households": len(asec.household), + "state_unknown_households": int(table[COLUMNS[1]].isna().sum()), + "puma_observed_households": int(table[COLUMNS[2]].notna().sum()), + "draw_identity": ["source", "source_year", "survey_year", "raw_native_id"], + "draw_key_encoding": "compact ASCII JSON array; source literal unchanged", + "acs_survey_year": 2024, + "asec_survey_year": 2025, + "asec_income_year": 2024, + "state_is_income_year_residence_claim": False, + "observed_state": "ACS ST; current ASEC GESTFIPS; two-digit strings", + "observed_puma": "ACS ST+PUMA; ASEC remains unknown", + "unknown_state": "preserved; no carried-state fallback", + "unassigned_jurisdiction_validation": "separate geography gate", + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = CurrentSurveyGeographyValues(table, receipt) + # Complete owner I/O before pure comparisons. A callback during the last + # source check must not mutate a retained input or detached output unnoticed. + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and source.acs_native._ISSUED.get(acs_native) is acs_owned + and acs_native.payload == acs_payload + and acs_owned.frame is acs_frame + and asec.receipt == asec_receipt, + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _check_asec_projection(asec, asec_receipt) + _require( + _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + == projected + and result.receipt == receipt + and _projection_digest(result.household) == projection_sha256, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py new file mode 100644 index 000000000..416c79650 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py @@ -0,0 +1,277 @@ +"""Project authenticated survey geography into declared household cells. + +This nonstructural node qualifies observed inputs only. Shared atomic operators +own location assignment; the country runner owns receiving-Population and warm +replay admission. Preparation bytes and descriptive receipts grant no authority. +""" + +from __future__ import annotations + +import json + +import pandas as pd + +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactInput, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_population as source_graph + +NODE = "survey_population.observed_geography" +REF = "us.survey_population.observed_geography@1" +PHASE = "us.survey_population.observed_geography.v1" +OUTPUT_COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_" + reason) + + +def _qualifier(): + # Declaration-only imports need not load retained source owners. + from . import current_survey_geography + + return current_survey_geography + + +def current_survey_geography_node( + *, + preparation_sha256, + projection_receipt_sha256, + population, + node_id=NODE, +): + """Declare three new observed columns; digest arguments authenticate nothing.""" + _require( + type(population) is str + and 0 < len(population) <= 256 + and type(node_id) is str + and 0 < len(node_id) <= 256 + and population != node_id, + "NODE_NAMES", + ) + return Node( + id=node_id, + kernel=REF, + population=population, + inputs=(Slice("household", source_graph._provenance_columns("household")),), + outputs=tuple(Owned("household", name, "string") for name in OUTPUT_COLUMNS), + params={ + "phase": PHASE, + "preparation_sha256": source_graph._digest(preparation_sha256), + "projection_receipt_sha256": source_graph._digest( + projection_receipt_sha256 + ), + }, + artifact_inputs=( + ArtifactInput( + "preparation", + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + ), + ), + description="Project qualified observed state, PUMA and stable source draw keys.", + ) + + +def _check_context(context, *, payload, source_frame, receipt_sha256): + expected = current_survey_geography_node( + preparation_sha256=source_graph._sha(payload), + projection_receipt_sha256=receipt_sha256, + population=context.node.population, + node_id=context.node.id, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "DECLARATION", + ) + _require( + not context.sources + and set(context.artifacts) == {"preparation"} + and set(context.tables) == {"household"}, + "CONTEXT_ROSTER", + ) + source_graph._artifact( + context, "preparation", source_graph.PREPARATION_TYPE, payload + ) + table = context.tables["household"] + columns = ("household_id", *source_graph._provenance_columns("household")) + _require(tuple(table.columns) == columns, "CONTEXT_COLUMNS") + try: + pd.testing.assert_frame_equal( + table, + source_frame.table("household").loc[:, list(columns)], + check_exact=True, + ) + except (AssertionError, ValueError, TypeError): + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_HOUSEHOLD_ORIGIN") from None + _require( + set(context.weights) == {"household"} + and context.weights["household"].kind is WeightKind.IMPORTANCE + and len(context.weights["household"].values) == len(table), + "ALLOCATED_WEIGHTS_REQUIRED", + ) + + +def _check_projection(qualifier, projection, *, payload, receipt_sha256): + _require( + type(projection) is qualifier.CurrentSurveyGeographyValues + and qualifier.COLUMNS == OUTPUT_COLUMNS + and type(projection.receipt) is bytes + and 0 < len(projection.receipt) <= qualifier.MAX_RECEIPT_BYTES + and source_graph._sha(projection.receipt) == receipt_sha256, + "PROJECTION_RECEIPT", + ) + document = json.loads(projection.receipt) + _require( + document["protocol"] == qualifier.PROTOCOL + and document["preparation_sha256"] == source_graph._sha(payload) + and document["columns"] == list(OUTPUT_COLUMNS) + and document["households"] == len(projection.household) + and document["projection_sha256"] + == qualifier._projection_digest(projection.household) + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "QUALIFIED_PROJECTION", + ) + return document + + +class CurrentSurveyGeographyKernel(KernelBase): + """Retain the live preparation; independently qualify every executed output.""" + + ref = REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=source_graph.STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation): + self._preparation = preparation + + def implementation_hash(self): + qualifier = _qualifier() + demographics = qualifier.demographics + # Preserve the source stage's maintained dependency fence, and bind the + # additional actual projection functions and their demographic owners. + return source_graph._sha( + canonical_json( + { + "source_stage": source_graph._Kernel.implementation_hash(self), + "projection": source_hash( + type(self), + qualifier, + qualifier.qualify_current_survey_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, + qualifier.dtype_for_token, + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def run(self, context): + preparation = self._preparation + owner = source_graph._source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + receipt_sha256 = source_graph._digest( + context.params.get("projection_receipt_sha256") + ) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + qualifier = _qualifier() + projection = qualifier.qualify_current_survey_geography(preparation) + document = _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + projection.household.index.tolist() + == context.tables["household"].household_id.tolist(), + "PROJECTION_HOUSEHOLD_ORDER", + ) + receipt = { + "phase": PHASE, + "preparation_sha256": source_graph._sha(payload), + "projection_receipt_sha256": receipt_sha256, + "source_projection": document, + "population_admission_issued": False, + "release_eligible": False, + } + receipt_bytes = canonical_json(receipt) + result = KernelResult( + columns={ + ("household", name): projection.household[name].copy(deep=True) + for name in OUTPUT_COLUMNS + }, + receipt=receipt, + ) + # Finish source-owner I/O before rechecking mutable source views, + # projected context and every detached returned column/receipt. + final_entry = preparation._checked() + _require( + self._preparation is preparation + and final_entry is entry + and owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload, + "FINAL_ISSUANCE", + ) + owner._pure_final(state) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + set(result.columns) == {("household", name) for name in OUTPUT_COLUMNS} + and result.frame is result.keep is result.expand is result.weights is None + and result.strata is None + and not result.artifacts + and canonical_json(result.receipt) == receipt_bytes, + "FINAL_RESULT", + ) + returned = pd.DataFrame( + {name: result.columns[("household", name)] for name in OUTPUT_COLUMNS} + ) + _require( + qualifier._projection_digest(returned) == document["projection_sha256"], + "FINAL_OUTPUT", + ) + return result diff --git a/packages/microcosm-build/tests/test_us_current_asec_demographics.py b/packages/microcosm-build/tests/test_us_current_asec_demographics.py index 128ee0c9d..c61c62b15 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_demographics.py +++ b/packages/microcosm-build/tests/test_us_current_asec_demographics.py @@ -77,13 +77,13 @@ def test_state_capture_structure_and_exact_bytes_refuse(tmp_path, defect): owner._read_state_capture(path, pin) -def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False): +def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False, zero=True): """Extend invented source members before actual preparation/issuance. Rebuild the real person-income attachment after member bytes change. Only fixture registry pins are changed; no source issuer or ready() is replaced. """ - arguments = fixture(tmp_path, monkeypatch) + arguments = fixture(tmp_path, monkeypatch, zero=zero) source = arguments["source_dir"] / "asec" members, pins = {}, [] for year, member, archive, *_ in coverage._MEMBER_PINS: diff --git a/packages/microcosm-build/tests/test_us_current_survey_geography.py b/packages/microcosm-build/tests/test_us_current_survey_geography.py new file mode 100644 index 000000000..e8ea63dab --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_geography.py @@ -0,0 +1,192 @@ +"""Current geography from actual issuers over pinned invented source members.""" + +import json +import sys +from fractions import Fraction + +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_geography as geography +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.graph.population import dtype_for_token + + +def _prepared(tmp_path, monkeypatch, *, unknown=False, zero=True): + arguments = _demographic_arguments( + tmp_path, monkeypatch, unknown=unknown, zero=zero + ) + return arguments, source.prepare_authenticated_survey_population(**arguments) + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_source_keys_geography_counts_and_unknownness(tmp_path, monkeypatch, unknown): + _, prepared = _prepared(tmp_path, monkeypatch, unknown=unknown) + frame = prepared.checked_view().frame + before = source._frame_identity(frame) + result = geography.qualify_current_survey_geography(prepared) + table, receipt = result.household, json.loads(result.receipt) + assert tuple(table.columns) == geography.COLUMNS + assert table.index.tolist() == frame.table("household").household_id.tolist() + assert all(table[c].dtype == dtype_for_token("string") for c in table) + assert table.shape == (6, 3) + assert receipt["households"] == 6 + assert receipt["acs_households"] == receipt["puma_observed_households"] == 4 + assert receipt["asec_households"] == 2 + assert receipt["state_unknown_households"] == int(unknown) + assert receipt["asec_survey_year"] == 2025 + assert receipt["acs_survey_year"] == receipt["asec_income_year"] == 2024 + assert not receipt["state_is_income_year_residence_claim"] + assert not receipt["source_admission_issued"] + assert not receipt["population_admission_issued"] + assert not receipt["release_eligible"] + assert len(result.receipt) <= geography.MAX_RECEIPT_BYTES + assert receipt["projection_sha256"] == geography._projection_digest(table) + by_origin = {} + for row in table.itertuples(index=False, name=None): + key = tuple(json.loads(row[0])) + assert row[0] == json.dumps(key, separators=(",", ":")) + by_origin[key] = row[1:] + assert set(by_origin) == { + ("acs", 2024, 2024, "2024HU0000001"), + ("acs", 2024, 2024, "2024HU0000002"), + ("acs", 2024, 2024, "2024GQ0000001"), + ("acs", 2024, 2024, "2024GQ0000002"), + ("asec", 2024, 2025, "00007"), + ("asec", 2024, 2025, "00008"), + } + for key, (state, puma) in by_origin.items(): + if key[0] == "acs": + assert (state, puma) == ("06", "0612345") + else: + assert pd.isna(puma) + if key[-1] == "00008" and unknown: + assert pd.isna(state) + else: + assert state == ("06" if key[-1] == "00007" else "36") + # The fixture reverses the original ASEC member rows before actual issuance; + # the distinct 06/36 observations above therefore exercise the keyed join. + assert source._frame_identity(frame) == before + + +def test_sampling_rungs_keep_the_source_draw_key(tmp_path, monkeypatch): + # Both ASEC records need positive support so the smaller source cell remains + # admissible regardless of which household the deterministic sample retains. + arguments, prepared = _prepared(tmp_path, monkeypatch, zero=False) + full = geography.qualify_current_survey_geography(prepared).household + original = full.set_index(geography.COLUMNS[0]).sort_index() + request_path = arguments["source_dir"] / "selection-request.json" + request = json.loads(request_path.read_bytes()) + request["fraction"] = [2, 3] + request_path.write_bytes( + json.dumps(request, sort_keys=True, separators=(",", ":")).encode() + ) + selected = source.prepare_authenticated_survey_population( + **{**arguments, "fraction": Fraction(2, 3)} + ) + smaller = geography.qualify_current_survey_geography(selected).household + assert 0 < len(smaller) < len(full) + smaller = smaller.set_index(geography.COLUMNS[0]).sort_index() + pd.testing.assert_frame_equal( + smaller, original.loc[smaller.index], check_exact=True + ) + + +def test_detached_output_mutation_does_not_change_source_or_requalification( + tmp_path, monkeypatch +): + _, prepared = _prepared(tmp_path, monkeypatch) + first = geography.qualify_current_survey_geography(prepared) + expected = first.household.copy(deep=True) + receipt = first.receipt + first.household.loc[first.household.index[0], geography.COLUMNS[1]] = "99" + fresh = geography.qualify_current_survey_geography(prepared) + pd.testing.assert_frame_equal(fresh.household, expected, check_exact=True) + assert fresh.receipt == receipt + + +@pytest.mark.parametrize( + "change", ("asec_member", "acs_native_puma", "receiving_order", "carried_state") +) +def test_changed_original_or_receiving_population_refuses( + tmp_path, monkeypatch, change +): + arguments, prepared = _prepared(tmp_path, monkeypatch) + state = source._ISSUED[id(prepared)][2] + if change == "asec_member": + path = arguments["source_dir"] / "asec" / "hhpub25.csv" + path.write_bytes(path.read_bytes() + b"\n") + elif change == "acs_native_puma": + table = state.source_frames[0].table("household") + table.loc[table.index[0], "PUMA"] = "00100" + elif change == "receiving_order": + state.frame._tables["household"] = ( + state.frame.table("household").iloc[::-1].copy(deep=True) + ) + else: + state.frame.table("household")["state_fips"] = 99 + with pytest.raises(ValueError): + geography.qualify_current_survey_geography(prepared) + + +@pytest.mark.parametrize("change", ("source_frame", "returned_projection")) +def test_mutation_on_last_owner_return_cannot_escape(tmp_path, monkeypatch, change): + _, prepared = _prepared(tmp_path, monkeypatch) + state = source._ISSUED[id(prepared)][2] + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code + is source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and frame.f_back.f_code + is geography.qualify_current_survey_geography.__code__ + and "result" in frame.f_back.f_locals + and not fired + ): + fired.append(True) + if change == "source_frame": + state.frame.table("household")["state_fips"] = 99 + else: + result = frame.f_back.f_locals["result"] + result.household.loc[ + result.household.index[0], geography.COLUMNS[1] + ] = "99" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError): + geography.qualify_current_survey_geography(prepared) + finally: + sys.setprofile(previous) + assert fired == [True] + + +def test_detached_asec_projection_mutation_on_return_refuses(tmp_path, monkeypatch): + _, prepared = _prepared(tmp_path, monkeypatch) + fired = [] + + def trace(frame, event, result): + if ( + event == "return" + and frame.f_code + is geography.demographics.qualify_current_asec_demographics.__code__ + and not fired + ): + fired.append(True) + row = result.household.index[result.household.GESTFIPS.eq("06")][0] + # Internally consistent values must still match the upstream receipt. + result.household.loc[row, "GESTFIPS"] = "36" + result.household.loc[row, "state_fips"] = 36 + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="ASEC_PROJECTION_DIGEST"): + geography.qualify_current_survey_geography(prepared) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py new file mode 100644 index 000000000..eadbedd74 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_current_survey_geography.py @@ -0,0 +1,353 @@ +"""Observed-geography graph contracts over actual issuers and invented originals.""" + +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_geography as qualifier +from microcosm.build.us_runtime import graph_current_survey_geography as graph +from microcosm.build.us_runtime import graph_survey_population as population +from microcosm.build.us_runtime import survey_population_preparation as source +from microcosm.graph import ArtifactType, StructuralDelta, compile_graph, run_graph + + +def _executed(tmp_path, monkeypatch, *, unknown=False): + """Construct privately pinned invented originals for a complete graph run.""" + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + return _execute(arguments, store_root=tmp_path / "store") + + +def _execute(arguments, *, store_root): + """Keep real preparation/artifact owners and capture a real kernel context.""" + prefix = population.run_authenticated_survey_population( + **arguments, + store_root=store_root, + clones=False, + return_values=True, + ) + qualified = qualifier.qualify_current_survey_geography(prefix.preparation) + node = graph.current_survey_geography_node( + preparation_sha256=population._sha(prefix.preparation.payload), + projection_receipt_sha256=population._sha(qualified.receipt), + population=population.ALLOCATION_NODE, + ) + compiled = compile_graph( + replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, node)) + ) + kernel = graph.CurrentSurveyGeographyKernel(prefix.preparation) + prefix.kernels.register(kernel) + contexts = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is kernel.run.__code__: + contexts.append(frame.f_locals["context"]) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=prefix.kernels, + ) + finally: + sys.setprofile(previous) + assert len(contexts) == 1 + return SimpleNamespace( + arguments=arguments, + prefix=prefix, + qualified=qualified, + node=node, + compiled=compiled, + kernel=kernel, + context=contexts[0], + manifest=manifest, + ) + + +@pytest.fixture(scope="module") +def known_run(tmp_path_factory): + """Share one real execution while its invented registry pins remain active.""" + with pytest.MonkeyPatch.context() as patch: + run = _executed(tmp_path_factory.mktemp("survey-geography-known"), patch) + yield run + # All consumers of this fixture must leave its issued source untouched. + source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture +def fresh_source_run(known_run, tmp_path): + """Reissue independent mutable Frames over the shared immutable originals.""" + run = _execute(known_run.arguments, store_root=tmp_path / "fresh-source-store") + assert run.prefix.preparation is not known_run.prefix.preparation + assert run.prefix.preparation.frame is not known_run.prefix.preparation.frame + yield run + source.verify_survey_population_preparation(known_run.prefix.preparation) + + +def _detached_context(context): + """Each negative gets independent tables, weights and container mappings.""" + return replace( + context, + tables={name: table.copy(deep=True) for name, table in context.tables.items()}, + weights={ + name: replace(value, values=value.values.copy()) + for name, value in context.weights.items() + }, + strata=context.strata.copy(deep=True), + params=dict(context.params), + sources=dict(context.sources), + artifacts=dict(context.artifacts), + tolerances=dict(context.tolerances), + numerics=dict(context.numerics), + ) + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_real_projection_graph_execution_and_replay_preserve_inputs( + tmp_path, monkeypatch, unknown +): + run = _executed(tmp_path, monkeypatch, unknown=unknown) + assert run.compiled.order == ( + population.CREATE_NODE, + population.ALLOCATION_NODE, + graph.NODE, + ) + assert run.node.structural is StructuralDelta.NONE + assert run.node.sources == run.node.artifact_outputs == () + assert run.node.weights is None + assert tuple(o.column for o in run.node.outputs) == qualifier.COLUMNS + assert all( + o.entity == "household" and o.dtype == "string" and not o.rewrite + for o in run.node.outputs + ) + assert run.node.artifact_inputs[0].producer == population.CREATE_NODE + assert run.node.artifact_inputs[0].type == population.PREPARATION_TYPE + assert not run.manifest.node(graph.NODE).hit + original = run.prefix.allocated_population.frame + version = run.compiled.versions[graph.NODE] + projected = run.manifest.population(version) + for entity in original.entities: + pd.testing.assert_frame_equal( + projected.table(entity).loc[:, original.table(entity).columns], + original.table(entity), + check_exact=True, + ) + assert ( + projected.weights_for("household").kind + is original.weights_for("household").kind + ) + assert ( + projected.weights_for("household").values.tobytes() + == original.weights_for("household").values.tobytes() + ) + assert ( + run.manifest.mass_ledger(version) == run.prefix.allocated_population.mass_ledger + ) + table = projected.table("household").set_index("household_id") + actual = table.loc[:, list(graph.OUTPUT_COLUMNS)] + pd.testing.assert_frame_equal(actual, run.qualified.household, check_exact=True) + assert len(actual) == 6 + assert actual.survey_observed_state.isna().sum() == int(unknown) + assert actual.survey_observed_puma.notna().sum() == 4 + unknown_asec = actual.survey_geography_origin_key.eq('["asec",2024,2025,"00008"]') + assert unknown_asec.sum() == 1 + if unknown: + assert actual.loc[unknown_asec, "survey_observed_state"].isna().all() + else: + assert actual.loc[unknown_asec, "survey_observed_state"].tolist() == ["36"] + receipt = run.manifest.node(graph.NODE).receipt + assert receipt["projection_receipt_sha256"] == population._sha( + run.qualified.receipt + ) + assert receipt["source_projection"]["households"] == 6 + assert ( + not receipt["population_admission_issued"] and not receipt["release_eligible"] + ) + + warm_calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is run.kernel.run.__code__: + warm_calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + warm = run_graph( + run.compiled, + sources=run.prefix.sources, + store=run.prefix.store, + kernels=run.prefix.kernels, + resume="require", + ) + finally: + sys.setprofile(previous) + assert warm_calls == [] + assert all(node.hit for node in warm.nodes.values()) + assert warm.key == run.manifest.key + population._same_frame(projected, warm.population(version)) + # Cached graph output is not source admission. Requalify the retained + # preparation independently, as the country runner must on every replay. + current = qualifier.qualify_current_survey_geography(run.prefix.preparation) + pd.testing.assert_frame_equal(current.household, actual, check_exact=True) + assert current.receipt == run.qualified.receipt + + +@pytest.mark.parametrize( + "change,reason", + ( + ("preparation_digest", "DECLARATION"), + ("projection_digest", "PROJECTION_RECEIPT"), + ("artifact_payload", "ARTIFACT_BINDING"), + ("artifact_type", "ARTIFACT_BINDING"), + ("rewrite_declaration", "DECLARATION"), + ("source_roster", "CONTEXT_ROSTER"), + ), +) +def test_mismatched_declaration_or_typed_evidence_refuses( + known_run, tmp_path, change, reason +): + run = known_run + context = _detached_context(run.context) + if change in {"preparation_digest", "projection_digest"}: + field = ( + "preparation_sha256" + if change == "preparation_digest" + else "projection_receipt_sha256" + ) + node = replace(context.node, params={**context.node.params, field: "f" * 64}) + context = replace(context, node=node, params=node.params) + elif change in {"artifact_payload", "artifact_type"}: + artifact = context.artifacts["preparation"] + artifact = ( + replace(artifact, payload=artifact.payload + b" ") + if change == "artifact_payload" + else replace( + artifact, type=ArtifactType(population.PREPARATION_TYPE.name, 99) + ) + ) + context = replace(context, artifacts={"preparation": artifact}) + elif change == "rewrite_declaration": + first, *rest = context.node.outputs + context = replace( + context, + node=replace(context.node, outputs=(replace(first, rewrite=True), *rest)), + ) + else: + context = replace(context, sources={"undeclared": tmp_path}) + with pytest.raises(ValueError, match=reason): + run.kernel.run(context) + + +def test_same_household_ids_with_wrong_native_origin_refuse(known_run): + run = known_run + context = _detached_context(run.context) + table = context.tables["household"] + column = population.spine_source_id_column("household") + table.loc[table.index[0], column] += 1 + with pytest.raises(ValueError, match="HOUSEHOLD_ORIGIN"): + run.kernel.run(context) + + +def test_returned_series_are_detached_and_do_not_authorize_reuse(known_run): + run = known_run + before = source._frame_identity(run.prefix.preparation.frame) + result = run.kernel.run(_detached_context(run.context)) + column = ("household", "survey_observed_state") + result.columns[column].iloc[0] = "99" + result.receipt["source_projection"]["households"] = 999 + fresh = run.kernel.run(_detached_context(run.context)) + assert fresh.receipt["source_projection"]["households"] == 6 + for name in graph.OUTPUT_COLUMNS: + pd.testing.assert_series_equal( + fresh.columns[("household", name)], + run.qualified.household[name], + check_exact=True, + ) + assert source._frame_identity(run.prefix.preparation.frame) == before + assert ( + fresh.frame + is fresh.keep + is fresh.expand + is fresh.weights + is fresh.strata + is None + ) + assert not fresh.artifacts + + +def _assert_final_mutation_refuses(run, change): + context = _detached_context(run.context) + state = source._ISSUED[id(run.prefix.preparation)][2] + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code + is source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and "result" in frame.f_back.f_locals + and not fired + ): + fired.append(True) + result = frame.f_back.f_locals["result"] + if change == "source_frame": + state.frame.table("household")["state_fips"] = 99 + elif change == "returned_column": + result.columns[("household", "survey_observed_state")].iloc[0] = "99" + else: + result.receipt["phase"] = "changed" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize("change", ("returned_column", "receipt")) +def test_mutation_at_final_owner_return_refuses(known_run, change): + _assert_final_mutation_refuses(known_run, change) + + +def test_source_frame_mutation_at_final_owner_return_refuses(fresh_source_run): + _assert_final_mutation_refuses(fresh_source_run, "source_frame") + + +def test_changed_qualifier_return_cannot_hide_behind_unchanged_receipt( + known_run, +): + run = known_run + context = _detached_context(run.context) + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is qualifier.qualify_current_survey_geography.__code__ + and frame.f_back.f_code is run.kernel.run.__code__ + and not fired + ): + fired.append(True) + assert isinstance(json.loads(arg.receipt)["projection_sha256"], str) + arg.household.loc[arg.household.index[0], "survey_observed_state"] = "99" + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="QUALIFIED_PROJECTION"): + run.kernel.run(context) + finally: + sys.setprofile(previous) + assert fired == [True] diff --git a/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py b/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py index 015c3fa07..1335ece07 100644 --- a/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py +++ b/packages/microcosm-build/tests/test_us_graph_puf55_survey_ss.py @@ -49,6 +49,27 @@ PROFILE = full.PUF55_SURVEY_SS +def _independent_finalized_frame(binding, artifacts): + """Use the real finalizer without the attachment's column/result builder.""" + finalized, _ = full.finalize_full_puf( + binding.expected_population.frame, + binding.donor, + predictor_known=binding.predictor_known, + matrix=artifacts["matrix"].payload, + matrix_producer_key=artifacts["matrix"].producer_key, + raw_draws={ + target: artifacts[f"raw_{index:03d}"].payload + for index, target in enumerate(PROFILE.targets) + }, + apply_state=artifacts["apply_state"].payload, + training_state=artifacts["training_state"].payload, + last_model=artifacts["last_model"].payload, + seed=binding.fit_nodes[0].params["seed"], + profile=PROFILE, + ) + return finalized + + @pytest.fixture(scope="module") def attached_puf55(tmp_path_factory): root = tmp_path_factory.mktemp("puf55_population_attachment") @@ -219,6 +240,7 @@ def attached_puf55(tmp_path_factory): artifacts=artifacts, producer_keys=producer_keys, evidence=evidence, + finalized=_independent_finalized_frame(binding, artifacts), ) ) replay.same_replayed_population(results[0].population, results[1].population) @@ -285,18 +307,37 @@ def test_puf55_cold_and_required_replay_preserve_survey_incumbents(attached_puf5 if (entity, column) not in owned_cells: assert population_ops.storage_equal(before[column], after[column]) + has_nonzero_oracle_value = False for owned in attach.outputs: before, after = upstream.table(owned.entity), frame.table(owned.entity) mask = support.puf_tax_detail_clone_mask(before, entity=owned.entity) assert owned.rows == placement.MASKS[owned.entity] assert after.loc[mask, owned.column].notna().all() assert result.population.owners[(owned.entity, owned.column)] == attach.id + # Align by entity IDs, independently of the attachment's positional + # selection and column construction. Replay alone shares that code. + id_column = upstream.schema.entity_id_column(owned.entity) + entity_ids = pd.Index(before.loc[mask, id_column], name=id_column) + expected = ( + result.finalized.table(owned.entity) + .set_index(id_column, verify_integrity=True) + .loc[entity_ids, owned.column] + .astype(owned.dtype) + ) + actual = after.set_index(id_column, verify_integrity=True).loc[ + entity_ids, owned.column + ] + pd.testing.assert_series_equal(actual, expected, check_exact=True) + has_nonzero_oracle_value |= bool(expected.ne(0).any()) if owned.column in before: assert population_ops.storage_equal( before[owned.column], after[owned.column], ~mask ) else: assert after.loc[~mask, owned.column].isna().all() + # Keep a nontrivial oracle: replacing all selected outputs with zeros + # must be detected even when the kernel and replay verifier agree. + assert has_nonzero_oracle_value people = upstream.table("person") mask = support.puf_tax_detail_clone_mask(people, entity="person") From 212095475df075d59cddefad7de7d69710f4b59c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 18:49:37 -0400 Subject: [PATCH 033/255] Carry atomic survey geography through cloning and age calibration --- changelog.d/893-survey-atomic-prefix.added.md | 1 + docs/geography-assignment.md | 50 +- .../us-atomic-age-v2-1-controls-20260909.json | 34 ++ ...ic-block-sources-36-controls-20260909.json | 26 + ...c-budget-semantic-8-controls-20260909.json | 29 + ...c-survey-population-controls-20260909.json | 31 ++ ...tor-compatibility-1-controls-20260909.json | 29 + .../build/us_runtime/atomic_block_sources.py | 411 ++++++++++++++ .../graph_atomic_survey_population.py | 417 ++++++++++++++ .../us_runtime/survey_age_calibration.py | 205 ++++++- .../us_runtime/survey_atomic_geography.py | 519 ++++++++++++++++++ .../build/us_runtime/survey_origin_budget.py | 294 +++++++++- .../tests/test_us_atomic_block_sources.py | 395 +++++++++++++ .../test_us_graph_atomic_survey_population.py | 356 ++++++++++++ ...survey_age_calibration_atomic_geography.py | 117 ++++ ...s_survey_origin_budget_atomic_geography.py | 190 +++++++ ...y_origin_budget_predictor_compatibility.py | 44 ++ 17 files changed, 3100 insertions(+), 48 deletions(-) create mode 100644 changelog.d/893-survey-atomic-prefix.added.md create mode 100644 experiments/us-atomic-age-v2-1-controls-20260909.json create mode 100644 experiments/us-atomic-block-sources-36-controls-20260909.json create mode 100644 experiments/us-atomic-budget-semantic-8-controls-20260909.json create mode 100644 experiments/us-atomic-survey-population-controls-20260909.json create mode 100644 experiments/us-budget-predictor-compatibility-1-controls-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py create mode 100644 packages/microcosm-build/tests/test_us_atomic_block_sources.py create mode 100644 packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py create mode 100644 packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py create mode 100644 packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py create mode 100644 packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py diff --git a/changelog.d/893-survey-atomic-prefix.added.md b/changelog.d/893-survey-atomic-prefix.added.md new file mode 100644 index 000000000..30658e355 --- /dev/null +++ b/changelog.d/893-survey-atomic-prefix.added.md @@ -0,0 +1 @@ +Connect qualified survey geography to shared Census-block assignment before cloning and age calibration. Preserve raw sampling allocation separately, reconstruct geography in calibration budgets, and verify complete geography inheritance through cold execution and required replay. Add bounded source-byte normalization controls and record the remaining native-source and release checks. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index 7992f3dcf..dbd72ce24 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -187,6 +187,50 @@ households become eight, every location column is inherited, all entity rows are copied, and each household weight is split equally across the pair. Subset mapping verification also passes. Receipt SHA256: `1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. -A separate source-only Codex review found no actionable defects. Native observed -column qualification and geography-aware reconstruction in the survey runner -and origin-budget verifier remain pending. +A separate source-only Codex review found no actionable defects. + +The current survey path now qualifies observed state and ACS PUMA from the +retained original-source preparation. Eleven source controls and fourteen graph +controls passed, covering missing geography, source changes, replay and stable +household identity across sampling rungs. These controls use invented originals +through the actual source issuers. Their receipts are recorded in +`experiments/us-survey-geography-source-controls-20260909.json` and +`experiments/us-survey-geography-graph-controls-20260909.json`. + +`us_runtime/graph_atomic_survey_population.py` connects that observed projection +to block assignment, geographic derivation, the integrity gate and the combined +survey clone. It retains the raw allocation separately from the enriched +pre-clone population, even though both share the allocation version identifier. +The complete ten-node prefix passed five controls: fresh execution, required +replay, full geography inheritance, support-byte mismatch and late mutation +refusals. The test population contains six survey households before cloning and +twelve afterward. The exact tested revisions and remaining checks are recorded +in `experiments/us-atomic-survey-population-controls-20260909.json`. + +The calibration budget and age-calibration runner now consume this optional +prefix and independently reconstruct it from the raw allocation. Eight budget +controls pass, as does the complete thirteen-node age graph through fresh +execution and required replay. A separate compatibility control verifies that +the existing predictor and PUF-host qualifiers still accept the default survey +prefix. See the corresponding `us-atomic-budget-semantic-8`, +`us-atomic-age-v2-1`, and `us-budget-predictor-compatibility-1` records in +`experiments/`. + +Replay can canonicalize the hidden backing values of missing numeric cells. +Persistent budget identities therefore bind logical Frame values, population +version, complete ownership, weight kinds, mass ledger and exact design-weight +bytes. They exclude only the helper receipt's physical population stamp. +Same-object mutation checks retain physical identity. The initial physical-stamp +replay failure remains recorded; the corrected budget controls pass. + +The byte-source adapter passed thirty-six invented-source controls through the +maintained PL, CD and PUMA parsers. This does not authorize acquisition of native +PL archives, which also contain housing segments. Native support preparation +will instead request only Census `P1_001N` block populations and independent +state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first +source control; national survey assignment requires complete admitted support +for the fifty states and DC. No native acquisition is accepted by these tests. + +Publisher-qualified native block support, national and district calibration, +and pruned export verification remain necessary for a release. Source integrity +checks and successful invented-data runs do not establish geographic fit. diff --git a/experiments/us-atomic-age-v2-1-controls-20260909.json b/experiments/us-atomic-age-v2-1-controls-20260909.json new file mode 100644 index 000000000..c7fb9cfa7 --- /dev/null +++ b/experiments/us-atomic-age-v2-1-controls-20260909.json @@ -0,0 +1,34 @@ +{ + "code_resources": 12, + "configured_resource_limits": { + "cpu_seconds": 600, + "wall_seconds": 900 + }, + "cpu_seconds": 601.979524, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 506183680, + "publisher_provenance_established": false, + "receipt_sha256": "fc77f8ead3a5b8f2f611aa8d1d20d4c19ab6b315322be17241c0b9ec5c12bbdf", + "release_eligible": false, + "resource_observation": "The configured limits were unchanged. Reported cumulative CPU was 601.979524 seconds; the guarded process returned zero after all checks.", + "result": 0, + "scope": "Actual thirteen-node cold and required replay age calibration over invented original sources and block support. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 613.9018378330511 +} diff --git a/experiments/us-atomic-block-sources-36-controls-20260909.json b/experiments/us-atomic-block-sources-36-controls-20260909.json new file mode 100644 index 000000000..52c25620e --- /dev/null +++ b/experiments/us-atomic-block-sources-36-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.88845, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 350126080, + "publisher_provenance_established": false, + "receipt_sha256": "a64003994a3f3823d06a22cbf7376ae8bcb3714d1a25110782cc72e83bfcf1cd", + "release_eligible": false, + "result": 0, + "scope": "36 invented source-byte adapter controls through actual PL/CD/PUMA parsers and atomic normalization: exact source/ZIP/CRC/bounds/roster/population/mapping checks, retained byte snapshots, output codec cap. Whole native PL acquisition remains excluded; no native/publisher/release acceptance.", + "source_and_owned_files": 494, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 36 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py": "b5be0b06085f0037206836a16625bb4be5dfe6dc6a12659f207e871aea53340b", + "packages/microcosm-build/tests/test_us_atomic_block_sources.py": "5b9a9fc74b14e354ec63b237b33736434a7b66f89135e798d222c32cc7ad788c" + }, + "unexpected_refusals": {}, + "wall_seconds": 3.594141000008676 +} diff --git a/experiments/us-atomic-budget-semantic-8-controls-20260909.json b/experiments/us-atomic-budget-semantic-8-controls-20260909.json new file mode 100644 index 000000000..0d13a0f9e --- /dev/null +++ b/experiments/us-atomic-budget-semantic-8-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 313.261743, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 447545344, + "publisher_provenance_established": false, + "receipt_sha256": "76a132f01c33bd71af65eb0c17bb2187d9b15dfefc6e7e61ad5f935ea14f59bb", + "release_eligible": false, + "result": 0, + "scope": "Corrected semantic budget replay identity and FIFO/owner/mutation refusals. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 8 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 321.5954028330161 +} diff --git a/experiments/us-atomic-survey-population-controls-20260909.json b/experiments/us-atomic-survey-population-controls-20260909.json new file mode 100644 index 000000000..9a162052a --- /dev/null +++ b/experiments/us-atomic-survey-population-controls-20260909.json @@ -0,0 +1,31 @@ +{ + "before_after_current_frozen_source_equal": true, + "code_resources": 12, + "cpu_seconds": 306.843367, + "model_source_files": 5983, + "native_geography_acceptance": false, + "native_payloads_read": 0, + "peak_rss_bytes": 440320000, + "pending_followups": [ + "Nonblocking support-file open correction and its FIFO control are tested separately.", + "Geography-aware calibration budget and age-runner integration are tested separately.", + "Publisher-qualified native block support and a native calibrated build remain pending." + ], + "publisher_provenance_established": false, + "receipt_sha256": "025bc482224c63c06020acd537c603235b0c612dc805d3b86deccc683624993e", + "release_eligible": false, + "result": 0, + "scope": "Complete survey atomic-geography wrapper on invented original source files: cold and required replay, raw/enriched allocation separation, geography-before-clone ordering, full Frame/owner/weight/design inheritance and late mutation refusals.", + "source_and_owned_files": 489, + "test_source_sha256": "762c386d5333514ef1cd12c1573c2e97970573ae519f6faedff8f9e736f094b9", + "tested_reconstruction_source_sha256": "6c707c55e9c6a1b67241f86807700856a6438e98428b8f5704f22ae175e40f89", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 5 + }, + "unexpected_refusals": {}, + "wall_seconds": 315.1505964579992, + "wrapper_source_sha256": "794789a80298b9d239d707ba4cae605b61265846ad6dbc39330d0e1a28c2d1fc" +} diff --git a/experiments/us-budget-predictor-compatibility-1-controls-20260909.json b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json new file mode 100644 index 000000000..2c4af43e8 --- /dev/null +++ b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 35.759673, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 519110656, + "publisher_provenance_established": false, + "receipt_sha256": "de102101abef2cb6238092df3cf2d8c87e885efec91349d130031919a5eabd4b", + "release_eligible": false, + "result": 0, + "scope": "Historical pair-return compatibility through both actual existing current-survey financial qualifiers; no model fitting. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 38.938349917007145 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py new file mode 100644 index 000000000..aea5b8366 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py @@ -0,0 +1,411 @@ +"""Pinned Census source bytes to normalized atomic-block support, without I/O. + +The caller establishes publisher identity and obtains the bytes. This adapter +checks those bytes against explicit pins, checks full ZIP member rosters, and +decompresses only the selected geography members. Unselected PL segment contents +are neither opened nor CRC-checked. No source or release admission is issued. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from io import BytesIO +from pathlib import PurePosixPath +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile +from zlib import error as zlib_error + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import atomic_block_support as normalized +from .block_ladder_sources import ( + US_STATES, + parse_national_cd_bef, + parse_pl_geo_blocks, +) +from .puma_ladder_sources import parse_tract_to_puma_relationship + +PROTOCOL = "microcosm.us.atomic-block-sources.v1" +CD_MEMBER = "NationalCD119.txt" +_GEO_MEMBERS = {state: f"{usps.lower()}geo2020.pl" for state, usps, _ in US_STATES} +_CHUNK_BYTES = 64 * 1024 + + +@dataclass(frozen=True, slots=True) +class AtomicBlockSourceBytes: + """Literal bytes and expected digest/roster; construction conveys no authority.""" + + payload: bytes + sha256: str + zip_members: tuple[str, ...] = () + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_BLOCK_SOURCES_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_source(record, *, archive, max_source_bytes): + _require(type(record) is AtomicBlockSourceBytes, "SOURCE_TYPE") + payload, digest, members = record.payload, record.sha256, record.zip_members + _require( + type(payload) is bytes and 0 < len(payload) <= max_source_bytes, + "SOURCE_BYTES", + ) + _require( + type(digest) is str + and re.fullmatch(r"[0-9a-f]{64}", digest) is not None + and _sha(payload) == digest, + "SOURCE_DIGEST", + ) + _require(type(members) is tuple, "MEMBER_ROSTER") + if archive: + _require( + bool(members) + and all( + type(name) is str + and 0 < len(name) <= 255 + and "\0" not in name + and "\\" not in name + and not name.endswith("/") + and not PurePosixPath(name).is_absolute() + and all(part not in {"", ".", ".."} for part in name.split("/")) + for name in members + ) + and len(set(members)) == len(members), + "MEMBER_ROSTER", + ) + else: + _require(members == (), "RAW_MEMBER_ROSTER") + return payload, digest, members + + +def _selected_zip_member(record, member, *, bounds, expanded): + source_payload, source_digest, expected_members = _checked_source( + record, archive=True, max_source_bytes=bounds["max_source_bytes"] + ) + provenance = {"sha256": source_digest, "size_bytes": len(source_payload)} + _require( + member in expected_members + and len(expected_members) <= bounds["max_zip_members"], + "SELECTED_MEMBER", + ) + try: + with ZipFile(BytesIO(source_payload)) as archive: + members = archive.infolist() + names = tuple(info.filename for info in members) + _require( + len(names) == len(expected_members) + and len(set(names)) == len(names) + and all(info.orig_filename == info.filename for info in members) + and set(names) == set(expected_members), + "ZIP_MEMBER_ROSTER", + ) + info = archive.getinfo(member) + _require( + not info.is_dir() + and not info.flag_bits & 1 + and info.compress_type in {ZIP_STORED, ZIP_DEFLATED}, + "ZIP_MEMBER_FORMAT", + ) + _require( + 0 < info.file_size <= bounds["max_member_bytes"], "ZIP_MEMBER_SIZE" + ) + _require( + expanded[0] + info.file_size <= bounds["max_total_geography_bytes"], + "TOTAL_GEOGRAPHY_BYTES", + ) + # ZipExtFile validates the selected member's CRC when exhausted. + # No other member is opened, even to check its CRC. + output = BytesIO() + with archive.open(info) as stream: + while chunk := stream.read(_CHUNK_BYTES): + _require( + output.tell() + len(chunk) <= info.file_size, + "ZIP_MEMBER_SIZE", + ) + output.write(chunk) + payload = output.getvalue() + _require(len(payload) == info.file_size, "ZIP_MEMBER_SIZE") + expanded[0] += len(payload) + provenance.update( + { + "zip_members": sorted(names), + "selected_member": member, + "selected_member_sha256": _sha(payload), + "selected_member_size_bytes": len(payload), + "crc_checked_members": [member], + "unselected_member_contents_read": False, + } + ) + except (BadZipFile, EOFError, RuntimeError, NotImplementedError, zlib_error): + raise ValueError("ATOMIC_BLOCK_SOURCES_ZIP_INVALID") from None + return payload, provenance + + +def _lines(payload, *, encoding, max_line_bytes): + stream = BytesIO(payload) + while line := stream.readline(max_line_bytes + 1): + _require(len(line) <= max_line_bytes and b"\0" not in line, "TEXT_LINE") + yield line.decode(encoding) + + +def _pl_lines(payload, *, state, max_line_bytes, statistics): + """Guard parser skips: count every block, including zero-population rows.""" + seen = set() + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + fields = line.rstrip("\r\n").split("|") + _require(len(fields) == 97, "PL_ROW_WIDTH") + statistics["geography_rows"] += 1 + level = fields[2] + if level in {"040", "750"}: + raw = fields[90].strip() + _require(re.fullmatch(r"[0-9]+", raw) is not None, "PL_POPULATION") + population = int(raw) + if level == "040": + statistics["state_rows"] += 1 + _require(statistics["state_rows"] == 1, "PL_STATE_ROWS") + statistics["state_population"] = population + else: + geoid = fields[9].strip() + _require( + re.fullmatch(r"[0-9]{15}", geoid) is not None + and geoid.startswith(state), + "PL_BLOCK_STATE", + ) + _require(geoid not in seen, "PL_DUPLICATE_BLOCK") + seen.add(geoid) + statistics["block_rows"] += 1 + statistics["zero_population_blocks"] += int(population == 0) + yield line + + +def _cd_lines(payload, *, max_line_bytes, statistics, unassigned): + first = True + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + if first: + first = False + else: + parts = line.strip().split(",") + _require(len(parts) == 2, "CD_ROW_WIDTH") + statistics["source_records"] += 1 + geoid, district = (part.strip() for part in parts) + if district == "ZZ": + _require(re.fullmatch(r"[0-9]{15}", geoid) is not None, "CD_BLOCK") + block = int(geoid) + _require(block not in unassigned, "CD_DUPLICATE_BLOCK") + unassigned.add(block) + statistics["delegate_records"] += int(district == "98") + yield line + + +def assemble_atomic_block_sources( + *, + pl_archives: Iterable[tuple[str, AtomicBlockSourceBytes]], + cd_archive: AtomicBlockSourceBytes, + tract_to_puma: AtomicBlockSourceBytes, + state_fips: tuple[str, ...], + source_ids: Mapping[str, str], + max_source_bytes: int = 512 * 1024**2, + max_member_bytes: int = 2 * 1024**3, + max_total_geography_bytes: int = 32 * 1024**3, + max_zip_members: int = 16, + max_line_bytes: int = 64 * 1024, +) -> tuple[bytes, bytes]: + """Return deterministic normalized support and a canonical source receipt. + + ``state_fips`` is a nonempty sorted tuple drawn from the 50 states plus DC. + ``pl_archives`` yields exactly that ordered roster, one archive at a time; + only the current PL archive/member is retained while accumulating block maps. + Required members are ``{usps_lower}geo2020.pl`` and ``NationalCD119.txt``. + The tract relationship is raw UTF-8 text. ZIP target text uses the maintained + source readers' Latin-1 convention. All positive-POP100 blocks are retained; + zero-population exclusions are counted in each state's receipt. + + The source byte cap and hash check precede ZIP metadata parsing. Member + roster/count and expanded-byte limits precede selected-member inflation and + text parsing, with a cumulative bound on selected geography plus the raw + relationship. The member-count limit does not bound ZipFile's initial + metadata allocation. No unselected segment is decompressed or parsed. + Normalized output must fit the receiving raw-bytes-v1 codec's 64 MiB cap. + + Matching caller-supplied hashes proves byte integrity, not that the caller's + pins identify an official publisher, a current source, or an admitted dataset. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all(type(state) is str and state in _GEO_MEMBERS for state in state_fips) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + puma_payload, puma_digest, _members = _checked_source( + tract_to_puma, archive=False, max_source_bytes=max_source_bytes + ) + puma_provenance = {"sha256": puma_digest, "size_bytes": len(puma_payload)} + _require( + len(puma_payload) <= min(max_member_bytes, max_total_geography_bytes), + "TOTAL_GEOGRAPHY_BYTES", + ) + expanded = [len(puma_payload)] + cd_payload, cd_provenance = _selected_zip_member( + cd_archive, CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics = {"source_records": 0, "delegate_records": 0} + unassigned = set() + districts = parse_national_cd_bef( + _cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + _lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + + iterator, exhausted = iter(pl_archives), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 2 and item[0] == state, + "PL_SOURCE_ROSTER", + ) + record = item[1] + payload, provenance = _selected_zip_member( + record, _GEO_MEMBERS[state], bounds=bounds, expanded=expanded + ) + statistics = { + "geography_rows": 0, + "state_rows": 0, + "state_population": 0, + "block_rows": 0, + "zero_population_blocks": 0, + } + state_population = parse_pl_geo_blocks( + _pl_lines( + payload, + state=state, + max_line_bytes=max_line_bytes, + statistics=statistics, + ), + state_fips=state, + ) + _require( + statistics["state_rows"] == 1 + and statistics["state_population"] == sum(state_population.values()) + and statistics["block_rows"] + == len(state_population) + statistics["zero_population_blocks"], + "PL_RECONCILIATION", + ) + _require( + not population.keys() & state_population.keys(), "PL_OVERLAPPING_STATES" + ) + population.update(state_population) + state_totals[state] = statistics["state_population"] + provenance.update( + source_id=identities["population"], + state_fips=state, + populated_blocks=len(state_population), + **statistics, + ) + population_provenance.append(provenance) + del item, record, payload, state_population + _require(next(iterator, exhausted) is exhausted, "PL_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": _sha(payload), + "exact_input_hashes_verified": True, + "unselected_zip_member_contents_read": False, + "publisher_provenance_established": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py new file mode 100644 index 000000000..4d5d19f91 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py @@ -0,0 +1,417 @@ +"""Execute qualified survey geography before the first support clone. + +The existing runner independently admits the raw allocation. This extension +reconstructs every added column from the live preparation and pinned normalized +support, then checks the complete graph populations on cold and warm execution. +Normalized support integrity does not establish its publisher provenance. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.build import atomic_geography +from microcosm.build.graph_atomic_geography import ( + ATOMIC_SUPPORT_TYPE, + AtomicSupportImportKernel, + register_atomic_geography_kernels, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + Graph, + KernelResult, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.codecs import load_raw_bytes +from microcosm.graph.executor import ( + _all_node_keys, + _expand_declared_payload, + _expand_rewrite_coordinates, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, +) +from microcosm.graph.keys import seed as node_seed +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + _mass_record, + expand_lineage_receipt, + expand_writes_receipt, + mass_record_receipt, + weight_cap_receipt, +) + +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as projection +from . import graph_survey_population as survey +from . import puf_support +from . import survey_atomic_geography as reconstruction +from .graph_sources import frame_column_declarations +from .survey_population_replay import same_replayed_population + + +@dataclass(frozen=True) +class AtomicSurveyPopulationRunValues: + """Actual run values, without a new source or release certificate.""" + + manifest: object + preparation: object + allocated_population: Population + geography_population: Population + clone_population: Population + geography_config: reconstruction.AtomicSurveyReconstruction + compiled: object + store: object + kernels: object + sources: dict + + +def _clone_expectations(geography, nodes, compiled): + """Independently reconstruct the complete clone, including inherited cells.""" + before = geography.population + expanded = puf_support.clone_us_frame_for_puf_support( + before.frame, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + design = survey._verify_cloned_frame( + before.frame, expanded, before.design_weights["household"] + ) + expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) + claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) + gate = next(n for n in nodes if n.id == "geography.clone_gate") + ledger = ( + *before.mass_ledger, + _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), + ) + owners = { + (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) + } + expected, receipts = {}, {} + lineage, facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], facts[entity] = clone._entity_lineage( + before.frame, expanded, entity + ) + authority = puf_support.validate_puf_clone_attachment( + expanded, boundary=survey.PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + before.frame, expanded, ("acs", "asec"), authority, facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(expand) + receipt["expand_writes"] = expand_writes_receipt( + before.frame, + expanded, + expand, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), + ) + receipts[expand.id] = receipt + receipts[claim.id] = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), + } + definition = json.loads(geography.definition) + system = definition["systems"][0]["id"] + receipts[gate.id] = atomic_geography.validate_geography( + expanded.table("household"), + definition, + {system: atomic_geography.decode_atomic_support(geography.support_payload)}, + ) + # The gate and the ownership claim can have either topological order. + for node_id in compiled.order: + if node_id not in receipts: + continue + if node_id == claim.id: + owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) + expected[node_id] = Population.from_frame( + expanded, + expand.id, + owners=owners, + mass_ledger=ledger, + design_weights={"household": np.array(design, copy=True)}, + ) + return expected, receipts + + +def _states(compiled, kernels, source_keys, expected_populations, raw_receipts): + """Bind independent domain receipts to current implementation and graph keys.""" + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + states, writer_receipts = {}, {} + for node_id in compiled.order: + node = compiled.graph.node(node_id) + population = expected_populations[node_id] + key = keys[node_id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + tuple( + (e, str(c)) + for e in population.frame.entities + for c in population.frame.table(e) + ) + if structural + else tuple((o.entity, o.column) for o in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + capabilities = _capabilities_projection(kernels.get(node.kernel).capabilities) + receipt = dict(raw_receipts[node_id]) + receipt["capabilities"] = dict(capabilities) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if structural and node.structural is not StructuralDelta.CREATE: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(population.mass_ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + states[node_id] = { + "key": key, + "ref": node.kernel, + "capabilities": capabilities, + "implementation": implementations[node_id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + "receipt": _freeze_json(receipt), + } + writer_receipts[node_id] = SimpleNamespace(receipt=states[node_id]["receipt"]) + return states + + +def run_atomic_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + geography_config, + resume="auto", + return_values=False, +): + """Build one block location before cloning and independently verify replay.""" + survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") + prefix = survey.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + resume=resume, + clones=False, + return_values=True, + ) + source_owner = survey._source_owner() + preparation_entry = prefix.preparation._checked() + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + _, view = survey._checked_preparation(prefix.preparation) + instructions = survey.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = survey._allocation_output( + view.frame, view.context, instructions, survey._sha(view.payload) + ) + prefix_artifacts = { + (survey.CREATE_NODE, "preparation"): view.payload, + (survey.CREATE_NODE, "frame_context"): view.context, + (survey.ALLOCATION_NODE, "allocation"): allocation_payload, + (survey.ALLOCATION_NODE, "frame_context"): allocated_context, + } + clone_nodes = clone.us_combined_survey_clone_nodes( + frame_column_declarations(geography.population.frame), + base=survey.ALLOCATION_NODE, + source_channels=("acs", "asec"), + ) + gate = replace( + geography.nodes[-1], + id="geography.clone_gate", + population=clone_nodes[0].id, + ) + additions = (*geography.nodes, *clone_nodes, gate) + compiled = compile_graph( + Graph( + "us", + ( + *prefix.compiled.graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name, _ in geography.sources), + ), + (*prefix.compiled.graph.nodes, *additions), + ) + ) + sources = {**prefix.sources, **dict(geography.sources)} + kernels, store = prefix.kernels, prefix.store + store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + kernels.register(projection.CurrentSurveyGeographyKernel(prefix.preparation)) + register_atomic_geography_kernels(kernels) + clone.register_us_combined_survey_clone_kernels(kernels) + expected = { + survey.CREATE_NODE: Population.from_frame( + prefix.manifest.population(survey.CREATE_NODE), survey.CREATE_NODE + ), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{stage.node.id: stage.population for stage in geography.stages}, + } + receipts = { + **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, + **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, + } + cloned, clone_receipts = _clone_expectations(geography, additions, compiled) + expected.update(cloned) + receipts.update(clone_receipts) + expected_stamps = { + name: reconstruction._population_stamp(value) + for name, value in expected.items() + } + raw_stamp = reconstruction._population_stamp(prefix.allocated_population) + source_items = tuple(sorted(sources.items())) + config_bytes = geography_config.to_bytes() + _, source_keys = _source_paths_and_keys(compiled, sources, store) + states = _states(compiled, kernels, source_keys, expected, receipts) + observed, observed_stamps = {}, {} + + def observe(node_id, population): + survey._require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + same_replayed_population(expected[node_id], population) + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) + + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + survey._require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + survey._check_node_states(manifest, states) + for node in geography.nodes: + if node.kernel == AtomicSupportImportKernel.ref: + survey._final_artifact( + manifest, + store, + node_id=node.id, + name="support", + type_=ATOMIC_SUPPORT_TYPE, + payload=geography.support_payload, + capabilities=AtomicSupportImportKernel.capabilities, + ) + # Validate prefix artifacts separately from column materialization. + for (node_id, name), payload in prefix_artifacts.items(): + key = prefix.manifest.node(node_id).opaque_artifacts[name] + survey._require( + manifest.node(node_id).opaque_artifacts.get(name) == key, + "ATOMIC_PREFIX_ARTIFACT_KEY", + ) + survey._require( + store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" + ) + clone_terminal = next( + name + for name in reversed(compiled.order) + if compiled.versions[name] == clone_nodes[0].id + ) + result = AtomicSurveyPopulationRunValues( + manifest, + prefix.preparation, + prefix.allocated_population, + observed[geography.nodes[-1].id], + observed[clone_terminal], + geography_config, + compiled, + store, + kernels, + sources, + ) + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + survey._require( + current_keys == {name: value["key"] for name, value in states.items()} + and current_implementations + == {name: value["implementation"] for name, value in states.items()}, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + # Finish source/support I/O before final comparisons of returned objects. + fresh = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + survey._require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and prefix.preparation.payload == preparation_entry[1], + "ATOMIC_FINAL_PREPARATION", + ) + source_owner._pure_final(preparation_entry[2]) + survey._require( + fresh.config_sha256 == geography.config_sha256 + and fresh.projection_receipt == geography.projection_receipt + and fresh.support_payload == geography.support_payload + and fresh.definition == geography.definition + and geography_config.to_bytes() == config_bytes + and result.geography_config is geography_config + and result.preparation is prefix.preparation + and result.allocated_population is prefix.allocated_population + and result.geography_population is observed[geography.nodes[-1].id] + and result.clone_population is observed[clone_terminal] + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and tuple(sorted(result.sources.items())) == source_items + and reconstruction._population_stamp(prefix.allocated_population) == raw_stamp, + "ATOMIC_FINAL_RECONSTRUCTION", + ) + same_replayed_population(fresh.population, result.geography_population) + for node_id, population in expected.items(): + survey._require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + same_replayed_population(population, observed[node_id]) + latest = {} + for node_id in compiled.order: + latest[compiled.versions[node_id]] = node_id + for version, node_id in latest.items(): + survey._same_frame(expected[node_id].frame, manifest.population(version)) + survey._require( + manifest.mass_ledger(version) == expected[node_id].mass_ledger, + "ATOMIC_FINAL_MASS_LEDGER", + ) + survey._check_node_states(manifest, states) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py index 49a6510b2..42ee6c992 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -1,10 +1,12 @@ -"""Country admission for two explicit seven-node survey age profiles. +"""Country admission for survey age profiles with an explicit source prefix. Every call reconstructs the accepted source preparation. The numerical kernels are values-only; source, full-population, group/row-cap and cache checks live here. The original entry point remains invented-only; the named development entry point derives its targets from an explicitly pinned S0101-only capture. Both permit only the first IMPORTANCE-to-CALIBRATED transition and no release. +An optional atomic-geography recipe enriches the accepted allocation before +cloning; the sampling budget continues to retain the original raw allocation. """ from __future__ import annotations @@ -183,7 +185,7 @@ def _expected_nodes( state = json.loads(prefix_identities[node.id]) state["artifacts"] = {(e, c): k for e, c, k in state["artifacts"]} # The accepted prefix runner already independently sealed this - # receipt. No receipt from the new seven-node run is trusted here. + # receipt. No receipt from the receiving run is trusted here. states[node.id] = state continue key = keys[node.id] @@ -300,6 +302,7 @@ def run_survey_age_calibration( epochs, learning_rate, resume="auto", + geography_config=None, ): """Reconstruct sources and admit a complete invented calibration graph. @@ -327,6 +330,7 @@ def run_survey_age_calibration( learning_rate=learning_rate, resume=resume, calibration=calibration, + geography_config=geography_config, ) @@ -342,6 +346,7 @@ def run_survey_age_development( epochs, learning_rate, resume="auto", + geography_config=None, ): """Run development calibration after deriving targets from exact source pins. @@ -382,6 +387,7 @@ def run_survey_age_development( calibration=calibration, age_source_dir=age_source_dir, activation=activation, + geography_config=geography_config, ) @@ -398,6 +404,7 @@ def _run_survey_age_calibration( calibration, age_source_dir=None, activation=None, + geography_config=None, ): """Shared population, budget, replay, diagnostic and final-return checks.""" _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") @@ -414,22 +421,110 @@ def _run_survey_age_calibration( canonical_json(calibration_binding) == activation_identity, "ACTIVATION_BINDING", ) - prefix = source_graph.run_authenticated_survey_population( - source_dir, - snapshot_root=snapshot_root, - store_root=store_root, - fraction=fraction, - seed=seed_value, - resume=resume, - clones=True, - return_values=True, + geography = None + geography_identity = None + geography_config_payload = None + prefix_populations = {} + if geography_config is None: + prefix = source_graph.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + clones=True, + return_values=True, + ) + else: + from . import graph_atomic_survey_population as atomic_source + + geography_config_payload = ( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + ) + prefix = atomic_source.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + geography_config=geography_config, + return_values=True, + ) + geography = atomic_source.reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + replay.same_replayed_population( + geography.population, prefix.geography_population + ) + geography_identity = budgets._population_identity(prefix.geography_population) + prefix_populations = { + stage.node.id: stage.population for stage in geography.stages + } + expanded, _receipts = atomic_source._clone_expectations( + geography, prefix.compiled.graph.nodes, prefix.compiled + ) + prefix_populations.update(expanded) + source_owner, source_view = source_graph._checked_preparation(prefix.preparation) + preparation_entry = source_owner._ISSUED.get(id(prefix.preparation)) + instructions = source_graph.allocation_instructions( + source_view.selection_plan, source_view.receipt["origins"]["households"] ) + _, allocated_context, allocation_payload, _, _ = source_graph._allocation_output( + source_view.frame, source_view.context, instructions, _sha(source_view.payload) + ) + prefix_artifacts = ( + ( + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + source_view.payload, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.CREATE_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + source_view.context, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "allocation", + source_graph.ALLOCATION_TYPE, + allocation_payload, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + allocated_context, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ) + if geography is not None: + prefix_artifacts += tuple( + ( + node.id, + "support", + atomic_source.ATOMIC_SUPPORT_TYPE, + geography.support_payload, + atomic_source.AtomicSupportImportKernel.capabilities, + ) + for node in geography.nodes + if node.kernel == atomic_source.AtomicSupportImportKernel.ref + ) initial = prefix.clone_population _require(initial is not None, "COMPLETE_CLONE_REQUIRED") budget = budgets.freeze_survey_origin_budget( prefix.preparation, allocated_population=prefix.allocated_population, clone_population=initial, + geography_config=geography_config, ) budget_view = budget.checked_view() budget_payload = budget_view.payload @@ -446,25 +541,41 @@ def _run_survey_age_calibration( } initial_identity = budgets._population_identity(initial) allocation_identity = budgets._population_identity(prefix.allocated_population) + prefix_population_identities = { + name: budgets._population_identity(population) + for name, population in prefix_populations.items() + } prefix_budget = budget prefix_budget_entry = budgets._entry(prefix_budget, budgets.SamplingOriginBudget) kernels = prefix.kernels kernels.register(transport.SurveySamplingBudgetKernel(budget_payload)) kernels.register(ages.SurveyAgeCountArtifactKernel()) kernels.register(numerical.SurveyAgeCalibrationKernel()) + age_nodes = ( + transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), + ages.survey_age_count_artifact_node( + population=initial.version, node_id=COUNT_NODE + ), + calibration, + ) graph = replace( prefix.compiled.graph, - nodes=( - *prefix.compiled.graph.nodes, - transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), - ages.survey_age_count_artifact_node( - population=initial.version, node_id=COUNT_NODE - ), - calibration, - ), + nodes=(*prefix.compiled.graph.nodes, *age_nodes), ) compiled = compile_graph(graph) - _require(len(compiled.order) == 7, "EXACT_GRAPH") + prefix_roster = set(prefix.compiled.order) + age_roster = {transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE} + _require( + set(prefix_identities) == prefix_roster + and not prefix_roster & age_roster + and tuple(node.id for node in age_nodes) + == (transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE) + and set(compiled.order) == prefix_roster | age_roster + and len(compiled.order) == len(prefix.compiled.order) + 3 + and tuple(name for name in compiled.order if name in prefix_roster) + == prefix.compiled.order, + "EXACT_GRAPH", + ) _require( clone.COMBINED_CLONE_CLAIM_NODE in compiled.predecessors[transport.BUDGET_NODE], "OWNERSHIP_CLAIM_EDGE", @@ -528,6 +639,7 @@ def observe(node_id, population): allocated_population=observed[source_graph.ALLOCATION_NODE], clone_population=population, candidate=budget_payload, + geography_config=geography_config, ) receiving_budget_entry = budgets._entry( budget, budgets.SamplingOriginBudget @@ -538,13 +650,21 @@ def observe(node_id, population): prefix.manifest.population(node_id), population.frame ) elif node_id == clone.COMBINED_CLONE_NODE: - actual_allocation = observed[source_graph.ALLOCATION_NODE] + actual_allocation = observed[ + source_graph.ALLOCATION_NODE + if geography is None + else geography.stages[-1].node.id + ] source_graph._verify_cloned_frame( actual_allocation.frame, population.frame, actual_allocation.design_weights["household"], ) replay.same_replayed_frame(initial.frame, population.frame) + if geography is not None: + replay.same_replayed_population(prefix_populations[node_id], population) + elif node_id in prefix_populations: + replay.same_replayed_population(prefix_populations[node_id], population) else: _require(False, "UNEXPECTED_NODE") observed[node_id] = population @@ -580,6 +700,7 @@ def observe(node_id, population): "PREFIX_RECEIPT_REPLAY", ) for node_id, name, type_, payload, capabilities in ( + *prefix_artifacts, ( transport.BUDGET_NODE, "budget", @@ -710,11 +831,45 @@ def observe(node_id, population): budgets.verify_survey_origin_budget(prefix_budget) budgets.verify_survey_weight_only_successor(admitted) if activation is not None: - # This is the last external evidence read. The existing complete - # population, manifest and retained budget seals all follow it. + # Recheck target evidence before the optional final support read; + # complete population, configuration and owner seals follow both. age_activation.verify_survey_age_targets( age_source_dir, declaration=activation, registry=registry ) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG", + ) + final_support, _support_identity = atomic_source.reconstruction._read_support( + geography_config + ) + _require(final_support == geography.support_payload, "FINAL_GEOGRAPHY_SUPPORT") + _require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and preparation_entry is not None + and prefix.preparation.payload == preparation_entry[1], + "FINAL_SOURCE_ISSUANCE", + ) + source_owner._pure_final(preparation_entry[2]) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload + and budgets._population_identity(prefix.geography_population) + == geography_identity + and all( + budgets._population_identity(population) + == prefix_population_identities[name] + for name, population in prefix_populations.items() + ), + "FINAL_GEOGRAPHY_PREFIX", + ) _require(budget.payload == budget_payload, "FINAL_BUDGET_BYTES") _require( budgets._population_identity(initial) == initial_identity @@ -748,8 +903,8 @@ def observe(node_id, population): "FINAL_ACTIVATION_BINDING", ) # No decoding, store access, source borrow or implementation helper follows - # this seal. It covers all seven receipts/descriptors, including scope and - # release flags, using expectations detached before execution. + # this seal. It covers the complete selected prefix and age trio, including + # scope and release flags, using expectations detached before execution. _check_manifest(manifest, compiled, expected_nodes) _require( canonical_json(result.diagnostics) == diagnostic_identity, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py new file mode 100644 index 000000000..9671eceb9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -0,0 +1,519 @@ +"""Reconstruct the complete pre-clone geography Population from live sources. + +The immutable recipe names normalized support bytes; their digest proves byte +integrity, not publisher provenance. A native source adapter must establish the +latter separately. This helper executes the maintained pure geography operators, +not graph kernels, and issues no source, Population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from dataclasses import asdict, dataclass, replace +from io import BytesIO +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd + +from microcosm.build import atomic_geography as atomic +from microcosm.build import graph_atomic_geography as atomic_graph +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import Graph, KernelResult, Node, SourceRef, compile_graph +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES +from microcosm.graph.population import Population, _mass_record, _storage_parts, patch + +from . import atomic_block_support as blocks +from . import current_survey_geography as observed +from . import graph_current_survey_geography as observed_graph +from . import graph_survey_population as graph +from . import survey_population_preparation as source +from . import survey_population_replay as replay +from .graph_sources import frame_column_declarations + +PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v1" +MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES +MAX_SUPPORT_EXPANDED_BYTES = 2 * 1024**3 +MAX_SUPPORT_MEMBERS = 32 + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyReconstruction: + """Exact immutable inputs; construction itself authenticates no source.""" + + support_path: str + support_sha256: str + source_ids: tuple[tuple[str, str], ...] + seed: int + + def __post_init__(self): + _config_bytes(self) + + def to_bytes(self): + """Revalidate and detach the recipe, including the literal source path.""" + return _config_bytes(self) + + +def _config_bytes(config): + _require(type(config) is AtomicSurveyReconstruction, "CONFIG_TYPE") + _require( + type(config.support_path) is str + and 0 < len(config.support_path) <= 4096 + and "\0" not in config.support_path + and Path(config.support_path).is_absolute() + and ".." not in Path(config.support_path).parts, + "SUPPORT_PATH", + ) + _require( + type(config.support_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", config.support_sha256) is not None, + "SUPPORT_DIGEST", + ) + _require( + type(config.source_ids) is tuple + and len(config.source_ids) == 3 + and all( + type(pair) is tuple + and len(pair) == 2 + and all(type(value) is str for value in pair) + and 0 < len(pair[1]) <= 4096 + and pair[1].strip() == pair[1] + for pair in config.source_ids + ) + and tuple(pair[0] for pair in config.source_ids) + == ("district", "population", "puma"), + "SOURCE_IDENTITIES", + ) + _require(type(config.seed) is int and 0 <= config.seed < 2**63, "SEED") + return canonical_json( + { + "protocol": PROTOCOL, + "support_path": config.support_path, + "support_sha256": config.support_sha256, + "source_ids": dict(config.source_ids), + "seed": config.seed, + } + ) + + +def _file_identity(value): + return tuple( + getattr(value, name) + for name in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _read_support(config): + """Read one bounded regular file and verify its exact pinned bytes.""" + descriptor = os.open( + config.support_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), + ) + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, + "SUPPORT_SIZE", + ) + payload = stream.read(before.st_size + 1) + after = os.fstat(stream.fileno()) + current = os.stat(config.support_path, follow_symlinks=False) + _require( + stat.S_ISREG(current.st_mode) + and _file_identity(before) == _file_identity(after) == _file_identity(current) + and len(payload) == before.st_size + and _sha(payload) == config.support_sha256, + "SUPPORT_CHANGED", + ) + return payload, _file_identity(current) + + +def _decode_support(payload): + # Bound declared decompressed bytes before numpy opens any array. The shared + # decoder still owns every schema, value, mapping and sampling-weight check. + with ZipFile(BytesIO(payload)) as archive: + members = archive.infolist() + _require( + 0 < len(members) <= MAX_SUPPORT_MEMBERS + and sum(member.file_size for member in members) + <= MAX_SUPPORT_EXPANDED_BYTES, + "SUPPORT_EXPANDED_SIZE", + ) + for member in members: + with archive.open(member) as stream: + version = np.lib.format.read_magic(stream) + if version == (1, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION") + # A small ZIP member must not claim a huge allocation in its + # NPY header. Validate the physical payload length before load. + _require( + not dtype.hasobject + and all(type(size) is int and size >= 0 for size in shape) + and math.prod(shape) * dtype.itemsize + == member.file_size - stream.tell(), + "SUPPORT_ARRAY_SIZE", + ) + return atomic.decode_atomic_support(payload) + + +def _copy_population(population): + frame = population.frame + return replace( + population, + frame=Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + # Frame recursively freezes fresh metadata containers. The admitted + # mass records contain scalars and immutable tuples; replacing each + # record avoids deepcopy's reflective dataclass class-cache writes. + metadata=frame.metadata, + mass_log=tuple(replace(record) for record in frame.mass_log), + ), + owners=dict(population.owners), + weight_kind=dict(population.weight_kind), + design_weights={ + name: values.copy() for name, values in population.design_weights.items() + }, + mass_ledger=tuple(replace(record) for record in population.mass_ledger), + ) + + +def _population_stamp(population): + """Pure in-process seal, including storage beneath nullable masks.""" + _require(type(population) is Population, "POPULATION_TYPE") + digest = hashlib.sha256() + frame = population.frame + digest.update( + canonical_json( + { + "frame": source._frame_identity(frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": list(population.weight_kind.items()), + "mass_ledger": [asdict(record) for record in population.mass_ledger], + } + ) + ) + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + for entity, values in population.design_weights.items(): + digest.update(canonical_json((entity, str(values.dtype), values.shape))) + digest.update(values.tobytes()) + return digest.hexdigest() + + +def _raw_allocation(view, population): + """Reconstruct the original allocation, retaining all existing checks.""" + _require(type(population) is Population, "RAW_POPULATION_TYPE") + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _weights, _context, _allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, population.frame) + replay.same_replayed_frame(expected, population.frame) + design = view.frame.weights_for("household").values + graph._check_design_anchors(population, design) + cells = ( + (entity, str(column)) + for entity in view.frame.entities + for column in view.frame.table(entity) + ) + ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + graph._check_population_state( + population, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=ledger, + ) + return columns, nodes + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyStage: + """Expected complete Population and pure receipt, never executor evidence.""" + + node: Node + population: Population + receipt: bytes + artifacts: tuple[tuple[str, bytes], ...] = () + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyReconstruction: + """Detached reconstruction values; consumers must independently requalify.""" + + population: Population + stages: tuple[AtomicSurveyGeographyStage, ...] + nodes: tuple[Node, ...] + definition: bytes + projection_receipt: bytes + support_payload: bytes + sources: tuple[tuple[str, str], ...] + config_sha256: str + receipt: bytes + + +def reconstruct_atomic_survey_geography(preparation, allocated_population, config): + """Qualify sources and reconstruct geography on the exact raw allocation. + + Stage values follow the actual compiled order of the source/allocation plus + geography graph. A support import may precede the observed-column projection. + Callers compare these complete expected Populations to executor observations + on cold and required-warm runs; these values do not replace those checks. + """ + config_bytes = _config_bytes(config) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + view = source.CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + columns, source_nodes = _raw_allocation(view, allocated_population) + original_stamp = _population_stamp(allocated_population) + current = _copy_population(allocated_population) + support_payload, support_identity = _read_support(config) + support = _decode_support(support_payload) + supports = {blocks.SYSTEM: support} + definition = blocks.assignment_definition( + identity=(observed.COLUMNS[0],), + state_column=observed.COLUMNS[1], + puma_column=observed.COLUMNS[2], + source_ids=dict(config.source_ids), + seed=config.seed, + ) + definition_bytes = canonical_json(definition) + projection = observed.qualify_current_survey_geography(preparation) + projection_receipt = projection.receipt + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + projection.household.index.tolist() + == current.frame.table("household").household_id.tolist(), + "PROJECTION_ORDER", + ) + observation = observed_graph.current_survey_geography_node( + preparation_sha256=_sha(payload), + projection_receipt_sha256=_sha(projection_receipt), + population=graph.ALLOCATION_NODE, + ) + geography = atomic_graph.atomic_geography_nodes( + definition, (*columns, *observation.outputs), base=graph.ALLOCATION_NODE + ) + nodes = (observation, *geography) + compiled = compile_graph( + Graph( + "us", + ( + SourceRef(graph.SOURCE_NAME, graph.SOURCE_CODEC), + SourceRef(blocks.SOURCE, "raw-bytes-v1"), + ), + (*source_nodes, *nodes), + ) + ) + wanted = {node.id for node in nodes} + ordered = tuple( + compiled.graph.node(name) for name in compiled.order if name in wanted + ) + stages = [] + for node in ordered: + values, artifacts = {}, () + if node.id == observation.id: + values = { + ("household", name): projection.household[name].copy(deep=True) + for name in observed.COLUMNS + } + receipt = { + "phase": observed_graph.PHASE, + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "source_projection": json.loads(projection_receipt), + "population_admission_issued": False, + "release_eligible": False, + } + elif node.kernel == atomic_graph.AtomicSupportImportKernel.ref: + _require( + support.metadata["system"] == node.params["system"], "SUPPORT_SYSTEM" + ) + artifacts = (("support", support_payload),) + receipt = { + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + } + else: + households = current.frame.table("household") + if node.kernel == atomic_graph.AtomicAssignKernel.ref: + output = atomic.assign_atomic(households, definition, supports) + receipt = { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": { + name: value.sha256 for name, value in supports.items() + }, + } + elif node.kernel == atomic_graph.AtomicDeriveKernel.ref: + output = atomic.derive_geography(households, definition, supports) + receipt = { + "scope": "atomic_area_functional_lookup", + "layers": { + system["id"]: system["layers"] + for system in definition["systems"] + }, + } + else: + _require( + node.kernel == atomic_graph.AtomicGeographyGateKernel.ref, + "STAGE_ROSTER", + ) + output = None + receipt = atomic.validate_geography(households, definition, supports) + if output is not None: + ids = pd.Index(households.household_id.to_numpy(), name="household_id") + values = { + ("household", name): pd.Series(output[name].array.copy(), index=ids) + for name in output + } + current = _copy_population(patch(current, node, KernelResult(columns=values))) + stages.append( + AtomicSurveyGeographyStage( + node, current, canonical_json(receipt), artifacts + ) + ) + stage_tuple = tuple(stages) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "config_sha256": _sha(config_bytes), + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "support_sha256": _sha(support_payload), + "definition_sha256": _sha(definition_bytes), + "stages": [stage.node.id for stage in stage_tuple], + "frame_sha256": source._frame_identity(current.frame), + "population_sha256": _population_stamp(current), + "publisher_provenance_established": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + result = AtomicSurveyGeographyReconstruction( + population=_copy_population(current), + stages=stage_tuple, + nodes=nodes, + definition=definition_bytes, + projection_receipt=projection_receipt, + support_payload=support_payload, + sources=((blocks.SOURCE, config.support_path),), + config_sha256=_sha(config_bytes), + receipt=receipt, + ) + result_stamps = tuple( + _population_stamp(stage.population) for stage in result.stages + ) + stage_values = tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + final_stamp = _population_stamp(result.population) + # Finish support and source-owner I/O before pure rechecks of every retained + # input and every detached expected Population. No callback is an authority. + _require( + _read_support(config) == (support_payload, support_identity), "FINAL_SUPPORT" + ) + final_entry = preparation._checked() + support_after = os.stat(config.support_path, follow_symlinks=False) + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload + and _config_bytes(config) == config_bytes, + "FINAL_ISSUANCE", + ) + _require( + stat.S_ISREG(support_after.st_mode) + and _file_identity(support_after) == support_identity, + "FINAL_SUPPORT_STAT", + ) + source._pure_final(state) + _raw_allocation(view, allocated_population) + _require( + _population_stamp(allocated_population) == original_stamp + and tuple(_population_stamp(stage.population) for stage in result.stages) + == result_stamps + and _population_stamp(result.population) == final_stamp, + "FINAL_POPULATIONS", + ) + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + result.nodes == nodes + and tuple(stage.node for stage in result.stages) == ordered + and tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + == stage_values + and result.definition == definition_bytes + and result.projection_receipt == projection_receipt + and result.support_payload == support_payload + and result.sources == ((blocks.SOURCE, config.support_path),) + and result.config_sha256 == _sha(config_bytes) + and result.receipt == receipt, + "FINAL_RESULT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py index ef4783e07..ef15217ca 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -4,6 +4,8 @@ incoming a=s*b are different quantities. These development coefficients are provisional; this module grants neither graph execution nor release authority. Decoded bytes cannot issue a budget or a weight-only successor. +An optional immutable geography recipe independently reconstructs the complete +pre-clone Frame; the retained allocation always remains the raw source allocation. """ from __future__ import annotations @@ -39,6 +41,7 @@ from . import graph_combined_clone as clone from . import graph_survey_population as graph +from . import survey_atomic_geography as geography from . import survey_population_preparation as source from .graph_sources import frame_column_declarations from .support_provenance import support_clone_index_column, support_source_id_column @@ -124,7 +127,7 @@ def _reference(d, p, s, actual_incoming): def _modules(): - return ( + modules = ( sys.modules[__name__], graph, clone, @@ -134,7 +137,22 @@ def _modules(): sys.modules[frame_column_declarations.__module__], sys.modules[support_source_id_column.__module__], sys.modules[clone.clone_us_frame_for_puf_support.__module__], + geography, + geography.blocks, + geography.atomic, + geography.atomic_graph, + geography.observed, + geography.observed_graph, + geography.observed.demographics, + geography.observed.demographics.demographic, + geography.observed.demographics.household, + geography.observed.demographics.source_csv_builtin, + sys.modules[geography.observed.demographics.demographic._snapshot.__module__], + sys.modules[geography.atomic.keyed_uniform.__module__], + sys.modules[geography.canonical_json.__module__], + sys.modules[geography.compile_graph.__module__], ) + return tuple({module.__name__: module for module in modules}.values()) def _live(): @@ -167,6 +185,45 @@ def _live(): np.ndarray, np.float64, ) + result["geography_contract"] = source._runtime_marker( + ( + geography.PROTOCOL, + geography.MAX_SUPPORT_BYTES, + geography.MAX_SUPPORT_EXPANDED_BYTES, + geography.MAX_SUPPORT_MEMBERS, + geography.RAW_BYTES_MAX_BYTES, + geography.blocks.SYSTEM, + geography.blocks.SOURCE, + tuple(sorted(geography.blocks._STATES)), + tuple(sorted(geography.atomic.RELATIONS)), + geography.atomic._U53, + geography.observed.PROTOCOL, + geography.observed.COLUMNS, + geography.observed.MAX_HOUSEHOLDS, + geography.observed.MAX_RECEIPT_BYTES, + geography.observed_graph.NODE, + geography.observed_graph.REF, + geography.observed_graph.PHASE, + geography.observed_graph.OUTPUT_COLUMNS, + geography.observed.demographics.PROTOCOL, + geography.observed.demographics.STATE_COLUMNS, + geography.observed.demographics.STATE_CONTRACT, + ( + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.name, + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.schema_version, + ), + geography.atomic_graph._DEPENDENCIES, + tuple( + (kernel.ref, asdict(kernel.capabilities)) + for kernel in ( + geography.atomic_graph.AtomicSupportImportKernel, + geography.atomic_graph.AtomicAssignKernel, + geography.atomic_graph.AtomicDeriveKernel, + geography.atomic_graph.AtomicGeographyGateKernel, + ) + ), + ) + ) return result @@ -217,7 +274,86 @@ def _physical_nonweight_identity(frame): return digest.hexdigest() -def _initial(view, allocated, expanded): +def _config_payload(config): + return ( + None + if config is None + else geography.AtomicSurveyReconstruction.to_bytes(config) + ) + + +def _geography_binding(value, config_payload, preparation_payload): + """Seal fresh reconstruction values; decoded receipts never issue authority.""" + _require( + type(value) is geography.AtomicSurveyGeographyReconstruction, + "GEOGRAPHY_RECONSTRUCTION_TYPE", + ) + document = json.loads(value.receipt) + population_sha256 = geography._population_stamp(value.population) + _require( + value.config_sha256 == _sha(config_payload) + and document["protocol"] == geography.PROTOCOL + and document["preparation_sha256"] == _sha(preparation_payload) + and document["config_sha256"] == value.config_sha256 + and document["frame_sha256"] == source._frame_identity(value.population.frame) + and document["population_sha256"] == population_sha256 + and document["projection_receipt_sha256"] == _sha(value.projection_receipt) + and document["definition_sha256"] == _sha(value.definition) + and document["support_sha256"] == _sha(value.support_payload) + and document["publisher_provenance_established"] is False + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # The full receipt above seals this detached object exactly. Frame-store + # replay can canonicalize storage beneath masked nulls, so its physical + # stamp cannot identify an independently reconstructed object across runs. + semantic_receipt = { + key: item for key, item in document.items() if key != "population_sha256" + } + population = value.population + semantic_population = { + "frame_sha256": document["frame_sha256"], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": [ + (entity, kind.value) for entity, kind in population.weight_kind.items() + ], + "mass_ledger": [asdict(record) for record in population.mass_ledger], + "design_weights": [ + (entity, str(values.dtype), values.shape, values.tobytes().hex()) + for entity, values in population.design_weights.items() + ], + } + return _json( + { + "config_sha256": value.config_sha256, + "reconstruction_semantic_sha256": _sha(_json(semantic_receipt)), + "projection_receipt_sha256": _sha(value.projection_receipt), + "definition_sha256": _sha(value.definition), + "support_sha256": _sha(value.support_payload), + "preclone_population_semantic_sha256": _sha(_json(semantic_population)), + "identity_scope": { + "reconstruction_omitted_fields": ["population_sha256"], + "population_fields": list(semantic_population), + "null_backing": "retained_in_in_process_physical_seals_only", + }, + "publisher_provenance_established": False, + } + ) + + +def _initial( + view, + allocated, + expanded, + *, + preparation=None, + geography_config=None, + _with_geography_binding=False, +): + _require(type(_with_geography_binding) is bool, "GEOGRAPHY_BINDING_FLAG") _require( type(allocated) is Population and type(expanded) is Population, "LIVE_POPULATION_REQUIRED", @@ -237,13 +373,8 @@ def _initial(view, allocated, expanded): fraction=view.selection_plan.fraction, seed=view.selection_plan.seed, ) - clone_nodes = clone.us_combined_survey_clone_nodes( - columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") - ) graph._same_frame(expected, allocated.frame) - copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) graph._check_design_anchors(allocated, design) - graph._check_design_anchors(expanded, copied_design) allocation_ledger = ( _mass_record( view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" @@ -257,7 +388,27 @@ def _initial(view, allocated, expanded): kind=WeightKind.IMPORTANCE, ledger=allocation_ledger, ) - owners = dict.fromkeys(cells, clone.COMBINED_CLONE_NODE) + geography_binding = None + if geography_config is not None: + config_payload = geography.AtomicSurveyReconstruction.to_bytes(geography_config) + reconstructed = geography.reconstruct_atomic_survey_geography( + preparation, allocated, geography_config + ) + geography_binding = _geography_binding( + reconstructed, config_payload, view.payload + ) + expected = reconstructed.population.frame + columns = frame_column_declarations(expected) + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(expanded, copied_design) + owners = { + (entity, str(column)): clone.COMBINED_CLONE_NODE + for entity in expected.entities + for column in expected.table(entity) + } owners.update( { (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE @@ -276,10 +427,30 @@ def _initial(view, allocated, expanded): ), ), ) + if geography_config is not None: + _require( + geography.AtomicSurveyReconstruction.to_bytes(geography_config) + == config_payload + and _geography_binding(reconstructed, config_payload, view.payload) + == geography_binding, + "FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # Current survey financial and diagnostic hosts consume the historical pair. + # Only the budget owner explicitly requests its additional reconstruction seal. + if _with_geography_binding: + return instructions, allocation, geography_binding return instructions, allocation -def _document(view, allocated, expanded, instructions, allocation, producer): +def _document( + view, + allocated, + expanded, + instructions, + allocation, + producer, + geography_binding=None, +): before, after = ( allocated.frame.table("household"), expanded.frame.table("household"), @@ -375,6 +546,8 @@ def records(): "group_count": len(instructions), "release_eligible": False, } + if geography_binding is not None: + header["atomic_geography"] = json.loads(geography_binding) # Stream one bounded origin record at a time; never materialize an unbounded # list of origin dictionaries before the transport cap is checked. payload = bytearray() @@ -414,6 +587,9 @@ class _BudgetState: preparation_payload: bytes allocated_identity: tuple expanded_identity: tuple + geography_config: geography.AtomicSurveyReconstruction | None + geography_config_payload: bytes | None + geography_binding: bytes | None def _preparation_entry(preparation, payload, expected=None): @@ -434,10 +610,19 @@ def _preparation_entry(preparation, payload, expected=None): def _final_budget_state(state): """Seal retained owners after helper/file I/O, with no recursive borrow. - The owner's pure check still hashes retained Frames. It does not parse or - reread source files, issue replacement capsules or grant decoded authority. + Optional support bytes are checked before the source owner's pure final + check. That pure check hashes retained Frames without rereading survey + files, issuing replacement capsules or granting decoded authority. """ _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + if state.geography_config is not None: + # Finish support I/O before the same pure owner/Population final seals. + # Re-reading exact pinned bytes grants no publisher provenance. + geography._read_support(state.geography_config) entry = _preparation_entry( state.preparation, state.preparation_payload, state.preparation_entry ) @@ -447,6 +632,19 @@ def _final_budget_state(state): and _population_identity(state.expanded) == state.expanded_identity, "FINAL_POPULATION_SEAL", ) + _require( + ( + state.geography_config is None + and state.geography_config_payload is None + and state.geography_binding is None + ) + or ( + state.geography_config is not None + and geography.AtomicSurveyReconstruction.to_bytes(state.geography_config) + == state.geography_config_payload + ), + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") _preparation_entry( state.preparation, state.preparation_payload, state.preparation_entry @@ -455,18 +653,36 @@ def _final_budget_state(state): def _validate_budget(state, payload): _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "INITIAL_GEOGRAPHY_CONFIG_CHANGED", + ) _require( _population_identity(state.allocated) == state.allocated_identity and _population_identity(state.expanded) == state.expanded_identity, "INITIAL_POPULATION_CHANGED", ) - # This private retained view is only a reconstruction input. The single - # fresh source borrow below must authenticate it after all helper/file I/O. + # This private retained view is only a reconstruction input. The fresh + # source borrow below authenticates it after the first reconstruction. view = state.initial_view _require(view.payload == state.preparation_payload, "SOURCE_PREPARATION_CHANGED") - instructions, allocation = _initial(view, state.allocated, state.expanded) + instructions, allocation, geography_binding = _initial( + view, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(geography_binding == state.geography_binding, "GEOGRAPHY_RECONSTRUCTION") expected, constraint = _document( - view, state.allocated, state.expanded, instructions, allocation, _producer() + view, + state.allocated, + state.expanded, + instructions, + allocation, + _producer(), + geography_binding, ) _require(expected == payload, "BUDGET_RECONSTRUCTION") final = source.AuthenticatedSurveyPopulationPreparation.checked_view( @@ -478,7 +694,15 @@ def _validate_budget(state, payload): and final.selection_plan is view.selection_plan, "FINAL_SOURCE_SEAL", ) - _initial(final, state.allocated, state.expanded) + _instructions, _allocation, final_geography = _initial( + final, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(final_geography == state.geography_binding, "FINAL_GEOGRAPHY_SEAL") return constraint @@ -552,7 +776,12 @@ class CheckedSamplingOriginBudget: def freeze_survey_origin_budget( - preparation, *, allocated_population, clone_population, candidate=None + preparation, + *, + allocated_population, + clone_population, + candidate=None, + geography_config=None, ): _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") _require( @@ -562,10 +791,18 @@ def freeze_survey_origin_budget( type(preparation) is source.AuthenticatedSurveyPopulationPreparation, "ISSUED_PREPARATION_REQUIRED", ) + config_payload = _config_payload(geography_config) producer = _producer() view = source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) preparation_entry = _preparation_entry(preparation, view.payload) - instructions, allocation = _initial(view, allocated_population, clone_population) + instructions, allocation, geography_binding = _initial( + view, + allocated_population, + clone_population, + preparation=preparation, + geography_config=geography_config, + _with_geography_binding=True, + ) state = _BudgetState( preparation, preparation_entry, @@ -575,15 +812,32 @@ def freeze_survey_origin_budget( view.payload, _population_identity(allocated_population), _population_identity(clone_population), + geography_config, + config_payload, + geography_binding, ) payload, _constraint = _document( - view, allocated_population, clone_population, instructions, allocation, producer + view, + allocated_population, + clone_population, + instructions, + allocation, + producer, + geography_binding, ) final = source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) _require( final.payload == view.payload and final.frame is view.frame, "FINAL_SOURCE_SEAL" ) - _initial(final, allocated_population, clone_population) + _instructions, _allocation, final_geography = _initial( + final, + allocated_population, + clone_population, + preparation=preparation, + geography_config=geography_config, + _with_geography_binding=True, + ) + _require(final_geography == geography_binding, "FINAL_GEOGRAPHY_SEAL") _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") result = _issue(SamplingOriginBudget, payload, state) entry = _entry(result, SamplingOriginBudget) diff --git a/packages/microcosm-build/tests/test_us_atomic_block_sources.py b/packages/microcosm-build/tests/test_us_atomic_block_sources.py new file mode 100644 index 000000000..9631992a6 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_atomic_block_sources.py @@ -0,0 +1,395 @@ +"""Invented source bytes through maintained parsers and atomic normalization.""" + +import hashlib +import json +import struct +import sys +from dataclasses import replace +from io import BytesIO +from zipfile import ZIP_STORED, ZipFile, ZipInfo +from zlib import crc32 + +import pytest + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.build.us_runtime import atomic_block_sources as source +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +CA_A = "060010201001000" +CA_B = "060010201001001" +CA_ZERO = "060010202001000" +DC = "110010001001000" +UNREAD = b"UNSELECTED_SEGMENT_CONTENT" + + +def _geo_row(level, geoid, population): + fields = [""] * 97 + fields[2], fields[9], fields[90] = level, geoid, str(population) + return "|".join(fields) + "\n" + + +def _zip(entries): + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_STORED) as archive: + for name, payload in entries: + archive.writestr(ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0)), payload) + return buffer.getvalue() + + +def _record(payload, members=()): + return source.AtomicBlockSourceBytes( + payload, hashlib.sha256(payload).hexdigest(), tuple(members) + ) + + +def _arguments(*, ca_rows=None, cd_rows=None, puma_rows=None): + if ca_rows is None: + ca_rows = [ + _geo_row("040", "06", 8), + _geo_row("750", CA_A, 2), + _geo_row("750", CA_B, 6), + _geo_row("750", CA_ZERO, 0), + ] + ca_members = ("ca000012020.pl", "cageo2020.pl") + ca = _record( + _zip(((ca_members[0], UNREAD), (ca_members[1], "".join(ca_rows).encode()))), + ca_members, + ) + dc = _record( + _zip( + ( + ( + "dcgeo2020.pl", + (_geo_row("040", "11", 5) + _geo_row("750", DC, 5)).encode(), + ), + ) + ), + ("dcgeo2020.pl",), + ) + if cd_rows is None: + cd_rows = [f"{CA_A},01", f"{CA_B},02", f"{CA_ZERO},ZZ", f"{DC},98"] + cd = _record( + _zip( + ((source.CD_MEMBER, ("GEOID,CDFP\n" + "\n".join(cd_rows) + "\n").encode()),) + ), + (source.CD_MEMBER,), + ) + if puma_rows is None: + puma_rows = [ + "06,001,020100,12345", + "06,001,020200,99999", + "11,001,000100,00100", + "72,001,000100,00100", + ] + return { + "pl_archives": (("06", ca), ("11", dc)), + "cd_archive": cd, + "tract_to_puma": _record( + ( + "\ufeffSTATEFP,COUNTYFP,TRACTCE,PUMA5CE\n" + "\n".join(puma_rows) + "\n" + ).encode() + ), + "state_fips": ("06", "11"), + "source_ids": { + name: "invented-" + name for name in ("district", "population", "puma") + }, + } + + +def test_exact_sources_preserve_every_populated_block_and_reconcile_states(): + arguments = _arguments() + payload, raw_receipt = source.assemble_atomic_block_sources(**arguments) + support, receipt = decode_atomic_support(payload), json.loads(raw_receipt) + assert support.arrays["area"].tolist() == [CA_A, CA_B, DC] + assert support.arrays["population"].tolist() == [2, 6, 5] + assert support.arrays["state"].tolist() == ["06", "06", "11"] + assert support.arrays["puma"].tolist() == ["0612345", "0612345", "1100100"] + assert support.arrays["district"].tolist() == ["0601", "0602", "1100"] + assert support.metadata["columns"]["district"]["relation"] == "official_tabulation" + assert receipt["reconciliation"]["state_population"] == {"06": 8, "11": 5} + assert receipt["reconciliation"]["output_state_population"] == {"06": 8, "11": 5} + assert receipt["reconciliation"]["population_total"] == 13 + assert receipt["reconciliation"]["populated_blocks"] == 3 + assert receipt["support_sha256"] == hashlib.sha256(payload).hexdigest() + for (state, record), provenance in zip( + arguments["pl_archives"], receipt["sources"]["population"], strict=True + ): + assert provenance["state_fips"] == state + assert provenance["sha256"] == record.sha256 + assert provenance["size_bytes"] == len(record.payload) + assert provenance["zip_members"] == sorted(record.zip_members) + assert provenance["crc_checked_members"] == [provenance["selected_member"]] + assert provenance["unselected_member_contents_read"] is False + assert receipt["sources"]["population"][0]["zero_population_blocks"] == 1 + assert receipt["sources"]["population"][0]["block_rows"] == 3 + assert receipt["sources"]["district"]["unassigned_blocks"] == 1 + assert receipt["sources"]["district"]["delegate_records"] == 1 + assert receipt["sources"]["puma"]["selected_state_tract_mappings"] == 3 + assert receipt["exact_input_hashes_verified"] is True + for field in ( + "unselected_zip_member_contents_read", + "publisher_provenance_established", + "source_admission_issued", + "release_eligible", + ): + assert receipt[field] is False + + consumed = [] + + def archives(): + for state, record in arguments["pl_archives"]: + consumed.append(state) + yield state, record + + replay = source.assemble_atomic_block_sources( + **{ + **arguments, + "pl_archives": archives(), + "source_ids": dict(reversed(list(arguments["source_ids"].items()))), + } + ) + assert consumed == ["06", "11"] + assert replay == (payload, raw_receipt) + + +def test_unselected_segment_content_is_not_decompressed_or_crc_checked(): + arguments = _arguments() + state, record = arguments["pl_archives"][0] + # Deliberately damage only the stored, unselected member's CRC. The whole + # archive digest is repinned; selected geography is still exactly intact. + damaged = record.payload.replace(UNREAD, b"X" + UNREAD[1:]) + assert damaged != record.payload + with ZipFile(BytesIO(damaged)) as archive: + # Check only metadata against the bytes this test constructed. The + # unrelated member is never opened, even to demonstrate the bad CRC. + assert archive.getinfo("ca000012020.pl").CRC != crc32(b"X" + UNREAD[1:]) + arguments["pl_archives"] = ( + (state, _record(damaged, record.zip_members)), + arguments["pl_archives"][1], + ) + payload, receipt = source.assemble_atomic_block_sources(**arguments) + assert decode_atomic_support(payload).arrays["population"].tolist() == [2, 6, 5] + assert json.loads(receipt)["unselected_zip_member_contents_read"] is False + + +@pytest.mark.parametrize("which", ("pl_archives", "cd_archive", "tract_to_puma")) +def test_any_wrong_exact_source_hash_refuses(which): + arguments = _arguments() + if which == "pl_archives": + (state, record), other = arguments[which] + arguments[which] = ((state, replace(record, sha256="f" * 64)), other) + else: + arguments[which] = replace(arguments[which], sha256="f" * 64) + with pytest.raises(ValueError, match="SOURCE_DIGEST"): + source.assemble_atomic_block_sources(**arguments) + + +@pytest.mark.parametrize("which", ("pl_archives", "cd_archive", "tract_to_puma")) +def test_checked_source_bytes_are_retained_after_record_mutation(which): + arguments = _arguments() + expected = source.assemble_atomic_block_sources(**arguments) + if which == "pl_archives": + record = arguments[which][0][1] + changed = _arguments( + ca_rows=[ + _geo_row("040", "06", 8), + _geo_row("750", CA_A, 3), + _geo_row("750", CA_B, 5), + _geo_row("750", CA_ZERO, 0), + ] + )[which][0][1] + elif which == "cd_archive": + record = arguments[which] + changed = _arguments( + cd_rows=[f"{CA_A},03", f"{CA_B},02", f"{CA_ZERO},ZZ", f"{DC},98"] + )[which] + else: + record = arguments[which] + changed = _arguments( + puma_rows=[ + "06,001,020100,12346", + "06,001,020200,99999", + "11,001,000100,00100", + "72,001,000100,00100", + ] + )[which] + assert changed.payload != record.payload + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is source._checked_source.__code__ + and frame.f_locals["record"] is record + and not fired + ): + fired.append(True) + object.__setattr__(record, "payload", changed.payload) + object.__setattr__(record, "sha256", changed.sha256) + # A changed source container cannot retarget the captured ZIP roster. + object.__setattr__(record, "zip_members", ("changed.txt",)) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + actual = source.assemble_atomic_block_sources(**arguments) + finally: + sys.setprofile(previous) + assert fired == [True] + assert actual == expected + + +def test_support_codec_cap_refuses_before_decoding(monkeypatch): + assert source.RAW_BYTES_MAX_BYTES == RAW_BYTES_MAX_BYTES == 64 * 1024**2 + # Narrow a resource bound only; real parsers and normalization still execute. + monkeypatch.setattr(source, "RAW_BYTES_MAX_BYTES", 16) + decoded = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code is source.decode_atomic_support.__code__ + and frame.f_back is not None + and frame.f_back.f_code is source.assemble_atomic_block_sources.__code__ + ): + decoded.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="ATOMIC_BLOCK_SOURCES_SUPPORT_BYTES"): + source.assemble_atomic_block_sources(**_arguments()) + finally: + sys.setprofile(previous) + assert not decoded + + +@pytest.mark.parametrize( + "defect", ("missing", "reordered", "extra", "extra_none", "states") +) +def test_exact_ordered_pl_state_roster_refuses(defect): + arguments = _arguments() + records = arguments["pl_archives"] + if defect == "missing": + arguments["pl_archives"] = records[:1] + elif defect == "reordered": + arguments["pl_archives"] = records[::-1] + elif defect == "extra": + arguments["pl_archives"] = (*records, records[0]) + elif defect == "extra_none": + arguments["pl_archives"] = (*records, None, records[0]) + else: + arguments["state_fips"] = ("06", "72") + with pytest.raises(ValueError, match="(?:PL_SOURCE|STATE)_ROSTER"): + source.assemble_atomic_block_sources(**arguments) + + +@pytest.mark.parametrize( + "defect", + ( + "extra_member", + "missing_target", + "duplicate", + "unsafe_name", + "nul_name", + "crc", + "declared_size", + ), +) +def test_zip_member_identity_crc_and_declared_size_refuse(defect): + arguments = _arguments() + record = arguments["cd_archive"] + if defect == "extra_member": + record = replace(record, zip_members=(*record.zip_members, "unexpected.txt")) + elif defect == "missing_target": + record = replace(record, zip_members=("different.txt",)) + elif defect == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + payload = _zip(((source.CD_MEMBER, b"one"), (source.CD_MEMBER, b"two"))) + record = _record(payload, record.zip_members) + elif defect == "unsafe_name": + record = replace(record, zip_members=("../" + source.CD_MEMBER,)) + elif defect == "nul_name": + name = source.CD_MEMBER + "Xsuffix" + payload = _zip(((name, b"GEOID,CDFP\n"),)) + payload = payload.replace( + name.encode(), (source.CD_MEMBER + "\0suffix").encode() + ) + record = _record(payload, record.zip_members) + elif defect == "crc": + payload = record.payload.replace(CA_A.encode(), CA_B.encode(), 1) + record = _record(payload, record.zip_members) + else: + payload = bytearray(record.payload) + central = payload.index(b"PK\x01\x02") + struct.pack_into(" Date: Wed, 9 Sep 2026 18:53:15 -0400 Subject: [PATCH 034/255] Normalize population-only Census responses for atomic support --- .../893-population-only-block-source.added.md | 1 + docs/geography-assignment.md | 9 + ...lock-api-sources-35-controls-20260909.json | 26 ++ .../us_runtime/atomic_block_api_sources.py | 310 ++++++++++++++++ .../tests/test_us_atomic_block_api_sources.py | 335 ++++++++++++++++++ 5 files changed, 681 insertions(+) create mode 100644 changelog.d/893-population-only-block-source.added.md create mode 100644 experiments/us-atomic-block-api-sources-35-controls-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py create mode 100644 packages/microcosm-build/tests/test_us_atomic_block_api_sources.py diff --git a/changelog.d/893-population-only-block-source.added.md b/changelog.d/893-population-only-block-source.added.md new file mode 100644 index 000000000..09da0ddba --- /dev/null +++ b/changelog.d/893-population-only-block-source.added.md @@ -0,0 +1 @@ +Normalize pinned population-only Census API responses into atomic block support. Reconcile every positive block against independent state totals and CD/PUMA mappings, retain exact source bytes and request descriptions, and keep publisher qualification separate from byte integrity. diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index dbd72ce24..e7a3cec67 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -231,6 +231,15 @@ state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first source control; national survey assignment requires complete admitted support for the fifty states and DC. No native acquisition is accepted by these tests. +`us_runtime/atomic_block_api_sources.py` implements that population-only response +adapter. All thirty-five invented controls pass, including independent state +totals, exact block preservation, missing mappings, malformed response fields, +byte limits and late source-record mutation. Request descriptors preserve the +repeated Census `in` parameters and omit credentials. The adapter does not infer +the response's origin from its contents; acquisition and publisher qualification +remain separate. Evidence: +`experiments/us-atomic-block-api-sources-35-controls-20260909.json`. + Publisher-qualified native block support, national and district calibration, and pruned export verification remain necessary for a release. Source integrity checks and successful invented-data runs do not establish geographic fit. diff --git a/experiments/us-atomic-block-api-sources-35-controls-20260909.json b/experiments/us-atomic-block-api-sources-35-controls-20260909.json new file mode 100644 index 000000000..6053026d2 --- /dev/null +++ b/experiments/us-atomic-block-api-sources-35-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.923694, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 349110272, + "publisher_provenance_established": false, + "receipt_sha256": "e76a4373281f38f98d794c89265cedbaf50744e63c60f5256dbbdb7905a3a586", + "release_eligible": false, + "result": 0, + "scope": "35 invented population-only Census API source controls through actual CD/PUMA parsers and atomic normalization. Exact response header, row/code types, source hashes, state/block totals, retained byte snapshots and mapping completeness; no queries, credentials, native inputs, publisher admission or release.", + "source_and_owned_files": 496, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 35 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py": "4bb9f58364e75846d4cbb19c3e0254468089481e28caacb67ae17eb87d802235", + "packages/microcosm-build/tests/test_us_atomic_block_api_sources.py": "69821ca1b81c47647d4a6a7faaf0bf9279889ef409368ed20b84779613ab5241" + }, + "unexpected_refusals": {}, + "wall_seconds": 4.598902583005838 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py new file mode 100644 index 000000000..6a78f5dee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py @@ -0,0 +1,310 @@ +"""Pinned population-only Census API responses to atomic support, without I/O. + +The caller obtains original ``2020/dec/pl`` responses requesting only +``P1_001N``. No legacy PL archive or housing table is accepted. Byte pins and +request descriptions confer no publisher, source, or release authority. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Mapping + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import atomic_block_sources as sources +from . import atomic_block_support as normalized +from .block_ladder_sources import parse_national_cd_bef +from .puma_ladder_sources import parse_tract_to_puma_relationship + +PROTOCOL = "microcosm.us.atomic-block-api-sources.v1" +ENDPOINT = "https://api.census.gov/data/2020/dec/pl" +POPULATION_VARIABLE = "P1_001N" +_BLOCK_HEADER = (POPULATION_VARIABLE, "state", "county", "tract", "block") +_STATE_HEADER = (POPULATION_VARIABLE, "state") + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_BLOCK_API_SOURCES_" + reason) + + +def _request(state, *, blocks): + parameters = [["get", POPULATION_VARIABLE]] + if blocks: + parameters.extend( + [ + ["for", "block:*"], + ["in", "state:" + state], + ["in", "county:*"], + ["in", "tract:*"], + ] + ) + else: + parameters.append(["for", "state:" + state]) + return {"endpoint": ENDPOINT, "parameters": parameters} + + +def _rows(payload, *, header, max_rows): + try: + document = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError): + raise ValueError("ATOMIC_BLOCK_API_SOURCES_RESPONSE_JSON") from None + _require( + type(document) is list + and 1 < len(document) <= max_rows + 1 + and document[0] == list(header), + "RESPONSE_HEADER_OR_COUNT", + ) + for row in document[1:]: + _require( + type(row) is list + and len(row) == len(header) + and all(type(value) is str for value in row), + "RESPONSE_ROW", + ) + yield row + + +def _population(value): + _require( + len(value) <= 16 + and re.fullmatch(r"[0-9]+", value) is not None + and int(value) <= 2**53, + "POPULATION", + ) + return int(value) + + +def _state_population(block_payload, total_payload, *, state, max_rows): + total_rows = list(_rows(total_payload, header=_STATE_HEADER, max_rows=1)) + total_raw, total_state = total_rows[0] + _require(total_state == state, "STATE_TOTAL_IDENTITY") + total = _population(total_raw) + blocks, seen, zero_count = {}, set(), 0 + for row in _rows(block_payload, header=_BLOCK_HEADER, max_rows=max_rows): + raw_population, raw_state, county, tract, block = row + _require( + raw_state == state + and all( + re.fullmatch(r"[0-9]{" + str(width) + r"}", value) is not None + for value, width in ((county, 3), (tract, 6), (block, 4)) + ), + "BLOCK_IDENTITY", + ) + geoid = int(raw_state + county + tract + block) + _require(geoid not in seen, "DUPLICATE_BLOCK") + seen.add(geoid) + population = _population(raw_population) + if population: + blocks[geoid] = population + else: + zero_count += 1 + _require( + bool(blocks) + and sum(blocks.values()) == total + and len(seen) == len(blocks) + zero_count, + "STATE_RECONCILIATION", + ) + return blocks, { + "state_population": total, + "block_rows": len(seen), + "zero_population_blocks": zero_count, + "populated_blocks": len(blocks), + } + + +def _raw_source(record, *, bounds, expanded): + payload, digest, _members = sources._checked_source( + record, archive=False, max_source_bytes=bounds["max_source_bytes"] + ) + _require( + len(payload) <= bounds["max_member_bytes"] + and expanded[0] + len(payload) <= bounds["max_total_geography_bytes"], + "TOTAL_GEOGRAPHY_BYTES", + ) + expanded[0] += len(payload) + return payload, {"sha256": digest, "size_bytes": len(payload)} + + +def assemble_atomic_block_api_sources( + *, + population_responses: Iterable[ + tuple[str, sources.AtomicBlockSourceBytes, sources.AtomicBlockSourceBytes] + ], + cd_archive: sources.AtomicBlockSourceBytes, + tract_to_puma: sources.AtomicBlockSourceBytes, + state_fips: tuple[str, ...], + source_ids: Mapping[str, str], + max_source_bytes: int = 512 * 1024**2, + max_member_bytes: int = 2 * 1024**3, + max_total_geography_bytes: int = 32 * 1024**3, + max_zip_members: int = 16, + max_line_bytes: int = 64 * 1024, + max_response_rows: int = 2_000_000, +) -> tuple[bytes, bytes]: + """Normalize exact original API responses and mapping bytes, with a receipt. + + Each ordered item is ``(state, block_response, single_state_total_response)``. + There is exactly one item per selected state, drawn from the 50 states plus + DC. Expected API requests appear in the descriptive receipt with repeated + ``in`` parameters preserved and no credential parameter. The caller owns + acquisition and publisher qualification; the response cannot prove its URL. + + Response byte limits precede JSON parsing; the row limit is enforced after + JSON allocation. CD ZIP metadata and inflation follow the existing bounded + source reader. All rows, including zero-population blocks, are counted; + every positive block is retained exactly once and reconciled to its separate + state response. No county/tract aggregation or synthetic PL file is used. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all( + type(state) is str and state in sources._GEO_MEMBERS for state in state_fips + ) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + "max_response_rows": max_response_rows, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + expanded = [0] + puma_payload, puma_provenance = _raw_source( + tract_to_puma, bounds=bounds, expanded=expanded + ) + cd_payload, cd_provenance = sources._selected_zip_member( + cd_archive, sources.CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics, unassigned = {"source_records": 0, "delegate_records": 0}, set() + districts = parse_national_cd_bef( + sources._cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + sources._lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + del puma_payload + + iterator, exhausted = iter(population_responses), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 3 and item[0] == state, + "POPULATION_SOURCE_ROSTER", + ) + block_payload, block_provenance = _raw_source( + item[1], bounds=bounds, expanded=expanded + ) + total_payload, total_provenance = _raw_source( + item[2], bounds=bounds, expanded=expanded + ) + blocks, statistics = _state_population( + block_payload, total_payload, state=state, max_rows=max_response_rows + ) + _require(not population.keys() & blocks.keys(), "OVERLAPPING_STATES") + population.update(blocks) + state_totals[state] = statistics["state_population"] + population_provenance.append( + { + "source_id": identities["population"], + "state_fips": state, + "blocks": {**block_provenance, "request": _request(state, blocks=True)}, + "state_total": { + **total_provenance, + "request": _request(state, blocks=False), + }, + **statistics, + } + ) + del item, block_payload, total_payload, blocks + _require(next(iterator, exhausted) is exhausted, "POPULATION_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": sources._sha(payload), + "exact_input_hashes_verified": True, + "publisher_provenance_established": False, + "request_origin_verified": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/tests/test_us_atomic_block_api_sources.py b/packages/microcosm-build/tests/test_us_atomic_block_api_sources.py new file mode 100644 index 000000000..d23ac1ca9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_atomic_block_api_sources.py @@ -0,0 +1,335 @@ +"""Invented population-only API responses through real atomic normalization.""" + +import hashlib +import json +import sys +from dataclasses import replace +from io import BytesIO +from zipfile import ZIP_STORED, ZipFile, ZipInfo + +import pytest + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.build.us_runtime import atomic_block_api_sources as source +from microcosm.build.us_runtime.atomic_block_sources import AtomicBlockSourceBytes + +CA_A, CA_B, CA_ZERO = "060010201001000", "060010201001001", "060010202001000" +DC = "110010001001000" +BLOCK_HEADER = ["P1_001N", "state", "county", "tract", "block"] + + +def _record(payload, members=()): + return AtomicBlockSourceBytes(payload, hashlib.sha256(payload).hexdigest(), members) + + +def _json_record(rows): + return _record(json.dumps(rows, separators=(",", ":")).encode()) + + +def _block(population, geoid): + return [str(population), geoid[:2], geoid[2:5], geoid[5:11], geoid[11:]] + + +def _arguments(*, ca_rows=None, ca_total=None, cd_rows=None, puma_rows=None): + if ca_rows is None: + ca_rows = [BLOCK_HEADER, _block(2, CA_A), _block(6, CA_B), _block(0, CA_ZERO)] + if ca_total is None: + ca_total = [["P1_001N", "state"], ["8", "06"]] + if cd_rows is None: + cd_rows = [f"{CA_A},01", f"{CA_B},02", f"{CA_ZERO},ZZ", f"{DC},98"] + if puma_rows is None: + puma_rows = [ + "06,001,020100,12345", + "06,001,020200,99999", + "11,001,000100,00100", + ] + buffer = BytesIO() + with ZipFile(buffer, "w", compression=ZIP_STORED) as archive: + archive.writestr( + ZipInfo(source.sources.CD_MEMBER, date_time=(2020, 1, 1, 0, 0, 0)), + "GEOID,CDFP\n" + "\n".join(cd_rows) + "\n", + ) + return { + "population_responses": ( + ("06", _json_record(ca_rows), _json_record(ca_total)), + ( + "11", + _json_record([BLOCK_HEADER, _block(5, DC)]), + _json_record([["P1_001N", "state"], ["5", "11"]]), + ), + ), + "cd_archive": _record(buffer.getvalue(), (source.sources.CD_MEMBER,)), + "tract_to_puma": _record( + ( + "STATEFP,COUNTYFP,TRACTCE,PUMA5CE\n" + "\n".join(puma_rows) + "\n" + ).encode() + ), + "state_fips": ("06", "11"), + "source_ids": { + "population": "invented-original-2020-api-P1_001N", + "district": "invented-original-cd119-bef", + "puma": "invented-original-tract-puma2020", + }, + } + + +def test_exact_api_sources_preserve_blocks_totals_and_nonsecret_request_receipts(): + arguments = _arguments() + payload, receipt_bytes = source.assemble_atomic_block_api_sources(**arguments) + support, receipt = decode_atomic_support(payload), json.loads(receipt_bytes) + assert support.arrays["area"].tolist() == [CA_A, CA_B, DC] + assert support.arrays["population"].tolist() == [2, 6, 5] + assert support.arrays["puma"].tolist() == ["0612345", "0612345", "1100100"] + assert support.arrays["district"].tolist() == ["0601", "0602", "1100"] + assert support.metadata["columns"]["population"]["basis"] == "2020_census_persons" + assert support.metadata["columns"]["district"]["relation"] == "official_tabulation" + assert receipt["reconciliation"]["state_population"] == {"06": 8, "11": 5} + assert receipt["reconciliation"]["output_state_population"] == {"06": 8, "11": 5} + assert receipt["reconciliation"]["population_total"] == 13 + assert receipt["reconciliation"]["populated_blocks"] == 3 + assert receipt["support_sha256"] == hashlib.sha256(payload).hexdigest() + assert receipt["sources"]["population"][0]["block_rows"] == 3 + assert receipt["sources"]["population"][0]["zero_population_blocks"] == 1 + assert receipt["sources"]["district"]["delegate_records"] == 1 + for (state, blocks, total), provenance in zip( + arguments["population_responses"], receipt["sources"]["population"], strict=True + ): + assert provenance["state_fips"] == state + for name, record in (("blocks", blocks), ("state_total", total)): + assert provenance[name]["sha256"] == record.sha256 + assert provenance[name]["size_bytes"] == len(record.payload) + assert provenance[name]["request"]["endpoint"] == source.ENDPOINT + assert provenance["blocks"]["request"]["parameters"] == [ + ["get", "P1_001N"], + ["for", "block:*"], + ["in", "state:" + state], + ["in", "county:*"], + ["in", "tract:*"], + ] + assert provenance["state_total"]["request"]["parameters"] == [ + ["get", "P1_001N"], + ["for", "state:" + state], + ] + assert receipt["exact_input_hashes_verified"] is True + for field in ( + "publisher_provenance_established", + "request_origin_verified", + "source_admission_issued", + "release_eligible", + ): + assert receipt[field] is False + assert b'"key"' not in receipt_bytes + assert b"YOUR_KEY" not in receipt_bytes + replay_arguments = { + **arguments, + "population_responses": iter(arguments["population_responses"]), + "source_ids": dict(reversed(list(arguments["source_ids"].items()))), + } + assert source.assemble_atomic_block_api_sources(**replay_arguments) == ( + payload, + receipt_bytes, + ) + + +def test_reordered_original_rows_have_identical_support_and_distinct_source_pin(): + first = source.assemble_atomic_block_api_sources(**_arguments()) + second = source.assemble_atomic_block_api_sources( + **_arguments( + ca_rows=[BLOCK_HEADER, _block(0, CA_ZERO), _block(6, CA_B), _block(2, CA_A)] + ) + ) + assert first[0] == second[0] + assert first[1] != second[1] + + +@pytest.mark.parametrize("which", ("blocks", "total", "cd_archive", "tract_to_puma")) +def test_exact_source_hash_refuses(which): + arguments = _arguments() + if which in {"blocks", "total"}: + first, second = arguments["population_responses"] + state, blocks, total = first + if which == "blocks": + blocks = replace(blocks, sha256="f" * 64) + else: + total = replace(total, sha256="f" * 64) + arguments["population_responses"] = ((state, blocks, total), second) + else: + arguments[which] = replace(arguments[which], sha256="f" * 64) + with pytest.raises(ValueError, match="SOURCE_DIGEST"): + source.assemble_atomic_block_api_sources(**arguments) + + +@pytest.mark.parametrize( + "defect", + ( + "extra_column", + "wrong_header", + "numeric", + "negative", + "unicode", + "null", + "wrong_state", + "short_code", + "duplicate_zero", + "duplicate_positive", + "missing_positive", + ), +) +def test_original_block_response_schema_and_complete_totals_refuse(defect): + rows = [BLOCK_HEADER.copy(), _block(2, CA_A), _block(6, CA_B), _block(0, CA_ZERO)] + if defect == "extra_column": + rows[0].append("unexpected_measure") + for row in rows[1:]: + row.append("0") + elif defect == "wrong_header": + rows[0][0] = "P1_002N" + elif defect == "numeric": + rows[1][0] = 2 + elif defect == "negative": + rows[-1][0] = "-1" + elif defect == "unicode": + rows[1][0] = "٢" + elif defect == "null": + rows[1][0] = None + elif defect == "wrong_state": + rows[1][1] = "11" + elif defect == "short_code": + rows[1][2] = "01" + elif defect == "duplicate_zero": + rows.append(rows[-1].copy()) + elif defect == "duplicate_positive": + rows.append(rows[1].copy()) + else: + rows.pop(1) + with pytest.raises(ValueError, match="ATOMIC_BLOCK_API_SOURCES_"): + source.assemble_atomic_block_api_sources(**_arguments(ca_rows=rows)) + + +@pytest.mark.parametrize( + "defect", ("wrong_total", "wrong_state", "extra_state", "wrong_header") +) +def test_independent_state_total_refuses(defect): + rows = [["P1_001N", "state"], ["8", "06"]] + if defect == "wrong_total": + rows[1][0] = "9" + elif defect == "wrong_state": + rows[1][1] = "11" + elif defect == "extra_state": + rows.append(["5", "11"]) + else: + rows[0].append("unexpected_measure") + with pytest.raises(ValueError, match="ATOMIC_BLOCK_API_SOURCES_"): + source.assemble_atomic_block_api_sources(**_arguments(ca_total=rows)) + + +@pytest.mark.parametrize("defect", ("missing", "reordered", "extra_none", "territory")) +def test_exact_state_response_roster_refuses(defect): + arguments = _arguments() + records = arguments["population_responses"] + if defect == "missing": + arguments["population_responses"] = records[:1] + elif defect == "reordered": + arguments["population_responses"] = records[::-1] + elif defect == "extra_none": + arguments["population_responses"] = (*records, None) + else: + arguments["state_fips"] = ("06", "72") + with pytest.raises(ValueError, match="(?:SOURCE|STATE)_ROSTER"): + source.assemble_atomic_block_api_sources(**arguments) + + +@pytest.mark.parametrize("which", ("blocks", "total", "tract_to_puma")) +def test_original_source_record_mutation_after_check_uses_captured_bytes(which): + arguments = _arguments() + expected = source.assemble_atomic_block_api_sources(**arguments) + if which == "blocks": + record = arguments["population_responses"][0][1] + changed = _json_record( + [BLOCK_HEADER, _block(3, CA_A), _block(5, CA_B), _block(0, CA_ZERO)] + ) + elif which == "total": + record = arguments["population_responses"][0][2] + changed = _json_record([["P1_001N", "state"], ["9", "06"]]) + else: + record = arguments[which] + changed = _record(record.payload.replace(b"12345", b"12346")) + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is source.sources._checked_source.__code__ + and frame.f_locals["record"] is record + and not fired + ): + fired.append(True) + object.__setattr__(record, "payload", changed.payload) + object.__setattr__(record, "sha256", changed.sha256) + object.__setattr__(record, "zip_members", ("changed.txt",)) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + actual = source.assemble_atomic_block_api_sources(**arguments) + finally: + sys.setprofile(previous) + assert fired == [True] + assert actual == expected + + +@pytest.mark.parametrize( + "bounds,reason", + ( + ({"max_source_bytes": 16}, "SOURCE_BYTES"), + ({"max_total_geography_bytes": 100}, "GEOGRAPHY_BYTES"), + ({"max_response_rows": 2}, "HEADER_OR_COUNT"), + ({"max_response_rows": True}, "BOUNDS"), + ), +) +def test_bounded_source_bytes_and_response_rows_refuse(bounds, reason): + with pytest.raises(ValueError, match=reason): + source.assemble_atomic_block_api_sources(**_arguments(), **bounds) + + +def test_malformed_json_and_support_size_are_refused_before_downstream_decode( + monkeypatch, +): + arguments = _arguments() + first, second = arguments["population_responses"] + arguments["population_responses"] = ( + (first[0], _record(b"not JSON"), first[2]), + second, + ) + with pytest.raises(ValueError, match="RESPONSE_JSON"): + source.assemble_atomic_block_api_sources(**arguments) + monkeypatch.setattr(source, "RAW_BYTES_MAX_BYTES", 16) + decoded = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code is source.decode_atomic_support.__code__ + and frame.f_back is not None + and frame.f_back.f_code is source.assemble_atomic_block_api_sources.__code__ + ): + decoded.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises(ValueError, match="SUPPORT_BYTES"): + source.assemble_atomic_block_api_sources(**_arguments()) + finally: + sys.setprofile(previous) + assert not decoded + + +@pytest.mark.parametrize("mapping", ("cd", "puma")) +def test_positive_block_missing_a_mapping_refuses(mapping): + arguments = ( + _arguments(cd_rows=[f"{CA_A},ZZ", f"{CA_B},02", f"{DC},98"]) + if mapping == "cd" + else _arguments(puma_rows=["11,001,000100,00100"]) + ) + with pytest.raises(ValueError, match="US atomic support"): + source.assemble_atomic_block_api_sources(**arguments) From 304453fcc7c51c99b75de190659eb9c9bcbb381a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 18:58:20 -0400 Subject: [PATCH 035/255] Qualify source demographics for survey financial imputation --- ...3-source-demographic-conditioning.added.md | 1 + ...cial-demographics-3-controls-20260909.json | 27 ++ .../us_runtime/current_survey_predictors.py | 167 +++++++++- .../graph_current_survey_predictors.py | 62 +++- ...s_current_survey_predictor_demographics.py | 295 ++++++++++++++++++ 5 files changed, 538 insertions(+), 14 deletions(-) create mode 100644 changelog.d/893-source-demographic-conditioning.added.md create mode 100644 experiments/us-financial-demographics-3-controls-20260909.json create mode 100644 packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py diff --git a/changelog.d/893-source-demographic-conditioning.added.md b/changelog.d/893-source-demographic-conditioning.added.md new file mode 100644 index 000000000..88bc7fb66 --- /dev/null +++ b/changelog.d/893-source-demographic-conditioning.added.md @@ -0,0 +1 @@ +Add optional source-qualified sex and state predictors to survey financial imputation. Preserve observed-year distinctions and default behavior, refuse unresolved source demographics, and verify complete population preservation through fitting, attachment and required replay. diff --git a/experiments/us-financial-demographics-3-controls-20260909.json b/experiments/us-financial-demographics-3-controls-20260909.json new file mode 100644 index 000000000..fdfd17581 --- /dev/null +++ b/experiments/us-financial-demographics-3-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 111.759989, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 528891904, + "publisher_provenance_established": false, + "receipt_sha256": "37b9c1ceeefe38028772d8e6de64d10728c5fbcc5c6339549bb3b07c1513a6c0", + "release_eligible": false, + "result": 0, + "scope": "Three invented financial-demographic qualification, graph cold/required replay, unknown/default compatibility, and late detached-projection mutation controls. No fit quality, native admission, or release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 114.70848075003596 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py index 2e427fbe2..80353fa66 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -20,6 +20,7 @@ from . import acs_income_universe as universe from . import cps_carried_current as leaves +from . import current_survey_geography as observed_geography from . import graph_puf_diagnostic_consumer as host from . import support_provenance as provenance @@ -29,6 +30,11 @@ "survey_predictor_employment_income", "survey_predictor_self_employment_income", ) +DEMOGRAPHIC_FEATURES = ( + *FEATURES, + "survey_predictor_is_female", + "survey_predictor_state_fips", +) TARGETS = ("survey_current_INT_VAL", "survey_current_DIV_VAL", "survey_current_CAP_VAL") MONEY_FIELDS = leaves.CPS_CURRENT_PREDICTOR_MONEY_FIELDS OUTPUTS = leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES @@ -42,6 +48,126 @@ def require(condition, reason): raise ValueError("CURRENT_SURVEY_PREDICTOR_" + reason) +def feature_columns(demographic_conditioning=False): + require(type(demographic_conditioning) is bool, "DEMOGRAPHIC_OPTION") + return DEMOGRAPHIC_FEATURES if demographic_conditioning else FEATURES + + +def _check_demographic_values(values, receipt): + """Detached projections must still match their source producer's receipt.""" + require( + type(values) is observed_geography.demographics.CurrentAsecDemographicValues + and type(receipt) is bytes + and values.receipt == receipt, + "DEMOGRAPHIC_RECEIPT", + ) + document = json.loads(receipt) + for entity in ("person", "household"): + table = getattr(values, entity) + require( + codec.sha(table.reset_index(drop=True).to_json(orient="table").encode()) + == document[f"{entity}_projection_sha256"], + "DEMOGRAPHIC_PROJECTION_CHANGED", + ) + + +def _demographic_features(preparation, entry): + """Bind selected source sex and observation-time state, without fallback. + + The maintained ASEC owner retains unknown literals and allocation status. + This opt-in model refuses those unknowns rather than completing them. State + uses the shared literal geography join, not an assigned block's state. + """ + state = entry[2] + values = observed_geography.demographics.qualify_current_asec_demographics( + preparation + ) + receipt = values.receipt + _check_demographic_values(values, receipt) + document = json.loads(receipt) + require( + document["preparation_sha256"] == codec.sha(entry[1]) + and document["sex_observation_year"] == 2025 + and document["state_observation_year"] == 2025 + and document["income_year"] == 2024, + "DEMOGRAPHIC_PERIOD", + ) + acs_frame = state.source_frames[0] + source.acs_native.verify_acs_native_coverage(state.native[0], acs_frame) + acs_payload = source.acs_native._owned(state.native[0]).payload + require(json.loads(acs_payload)["vintage"] == 2024, "ACS_DEMOGRAPHIC_PERIOD") + # This pure operation verifies literal household keys and the complete + # selected source roster using the just-qualified ASEC household projection. + households = observed_geography._project( + json.loads(entry[1])["origins"]["households"], + state.frame.table("household"), + acs_frame.table("household"), + values.household, + ) + people = state.frame.person + index = pd.Index(_ids(people.person_id), name="person_id") + asec = people[provenance.support_channel_column("person")].eq("asec") + native_ids = people[provenance.spine_source_id_column("person")] + require( + values.person.index.is_unique + and set(values.person.index) == set(index[asec]) + and np.array_equal( + values.person.reindex(index[asec]).native_person_id.to_numpy(), + native_ids.loc[asec].to_numpy(), + ), + "DEMOGRAPHIC_ASEC_PERSON_JOIN", + ) + asec_person = values.person.reindex(index[asec]) + require( + asec_person.sex_known.all() and asec_person.is_female.notna().all(), + "DEMOGRAPHIC_UNKNOWN", + ) + original = acs_frame.person.set_index("person_id", drop=False) + require( + original.index.is_unique + and set(original.index) == set(native_ids.loc[~asec]) + and {"SEX", "is_female"} <= set(original), + "DEMOGRAPHIC_ACS_PERSON_JOIN", + ) + acs_person = original.reindex(native_ids.loc[~asec].to_numpy()) + sex = _numeric(acs_person.SEX) + require( + np.isin(sex, (1, 2)).all() + and pd.api.types.is_bool_dtype(acs_person.is_female.dtype) + and acs_person.is_female.notna().all() + and np.array_equal(acs_person.is_female.to_numpy(), sex == 2), + "ACS_SEX_IDENTITY", + ) + female = np.empty(len(index), dtype=np.float64) + female[asec] = asec_person.is_female.to_numpy(dtype=np.float64) + female[~asec] = (sex == 2).astype(np.float64) + states = {row[0]: row[2] for row in households} + selected_states = people.person_household_id.map(states) + require(selected_states.notna().all(), "DEMOGRAPHIC_UNKNOWN") + features = pd.DataFrame( + { + DEMOGRAPHIC_FEATURES[-2]: female, + DEMOGRAPHIC_FEATURES[-1]: selected_states.to_numpy(dtype=np.float64), + }, + index=index, + ) + evidence = { + "asec_projection_sha256": codec.sha(receipt), + "acs_native_sha256": codec.sha(acs_payload), + "asec_sex_and_state_observation_year": 2025, + "acs_sex_and_state_observation_year": 2024, + "state_is_income_year_residence_claim": False, + "asec_allocated_sex": "retained_when_source_owner_binding_is_known", + "acs_sex_allocation_provenance": "unresolved; no_unallocated_value_claim", + "unknown_policy": "refuse_opted_in_fit; no_fill_or_carried_fallback", + "state_encoding": "numeric_source_FIPS_split_predictor; not_geographic_distance", + "state_domain": "source_dictionary_codes; not_atomic_geography_admission", + "features_sha256": codec.sha(features.to_numpy(dtype=" 0, "TREE_COUNT") + values.feature_columns(qualified.demographic_conditioning) require( type(host_pins) is dict and set(host_pins) == {e.name for e in host.current_survey_host_edges()}, @@ -120,6 +121,7 @@ def _params(qualified, host_pins, n_estimators): "host_edges": codec.encode_json(host_pins).decode(), "seed": values.SEED, "n_estimators": n_estimators, + "demographic_conditioning": qualified.demographic_conditioning, } @@ -130,6 +132,7 @@ def current_survey_predictor_nodes( type(qualified) is values.QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE" ) params = _params(qualified, host_pins, n_estimators) + predictors = values.feature_columns(qualified.demographic_conditioning) projection = Node( PROJECTION_NODE, CurrentSurveyPredictorProjectionKernel.ref, @@ -164,7 +167,7 @@ def current_survey_predictor_nodes( inputs=_inputs(qualified.donor_frame), params=params, outputs=tuple( - Owned("person", c, "float64") for c in (*values.FEATURES, *values.TARGETS) + Owned("person", c, "float64") for c in (*predictors, *values.TARGETS) ), artifact_inputs=_projection_edges(), ) @@ -172,7 +175,7 @@ def current_survey_predictor_nodes( FIT_PREFIX, population=DONOR_NODE, entity="person", - predictors=values.FEATURES, + predictors=predictors, targets=values.TARGETS, seed=values.SEED, phase=values.PHASE, @@ -211,14 +214,18 @@ def __init__( *, host_pins, n_estimators=100, + demographic_conditioning=False, ): self.preparation = preparation self.allocated_population = allocated_population self.clone_population = clone_population self.host_pins = codec.decode_json(codec.encode_json(host_pins)) self.n_estimators = n_estimators + values.feature_columns(demographic_conditioning) + self.demographic_conditioning = demographic_conditioning def implementation_hash(self): + demographics = values.observed_geography.demographics return codec.sha( codec.encode_json( { @@ -228,6 +235,14 @@ def implementation_hash(self): values, values.leaves, values.universe, + values.observed_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, population_ops, replay, qrf, @@ -243,7 +258,10 @@ def implementation_hash(self): def _qualified(self, context): require(not context.sources, "UNDECLARED_SOURCE") result = values.qualify_current_survey_predictors( - self.preparation, self.allocated_population, self.clone_population + self.preparation, + self.allocated_population, + self.clone_population, + demographic_conditioning=self.demographic_conditioning, ) nodes = current_survey_predictor_nodes( result, @@ -363,13 +381,24 @@ def run(self, context): return KernelResult( columns={ ("person", c): qualified.donor_columns[c] - for c in (*values.FEATURES, *values.TARGETS) + for c in ( + *values.feature_columns(qualified.demographic_conditioning), + *values.TARGETS, + ) }, receipt=qualified.evidence, ) -def read_current_survey_draws(matrix, matrix_producer_key, raw_draws, apply_states): +def read_current_survey_draws( + matrix, + matrix_producer_key, + raw_draws, + apply_states, + *, + demographic_conditioning=False, +): + predictors = values.feature_columns(demographic_conditioning) require( codec._hash(matrix_producer_key) and type(raw_draws) is tuple @@ -379,8 +408,7 @@ def read_current_survey_draws(matrix, matrix_producer_key, raw_draws, apply_stat ) prepared = model_input.decode_recipient_matrix(matrix) require( - prepared.entity == "person" - and tuple(prepared.features.columns) == values.FEATURES, + prepared.entity == "person" and tuple(prepared.features.columns) == predictors, "MATRIX_FEATURE_ROSTER", ) result = pd.DataFrame(index=prepared.features.index) @@ -397,7 +425,7 @@ def read_current_survey_draws(matrix, matrix_producer_key, raw_draws, apply_stat ) require( chain.entity == "person" - and tuple(chain.predictors) == values.FEATURES + and tuple(chain.predictors) == predictors and tuple(chain.targets) == values.TARGETS and tuple(chain.completed_targets) == values.TARGETS[: i + 1] and chain.recipient_index == qrf._index_identity(prepared.features.index) @@ -443,7 +471,11 @@ def run(self, context): host.shared.artifact(context, s, MATRIX_APPLY_STATE_TYPE).payload ) drawn = read_current_survey_draws( - matrix.payload, matrix.producer_key, tuple(raw), tuple(states) + matrix.payload, + matrix.producer_key, + tuple(raw), + tuple(states), + demographic_conditioning=self.demographic_conditioning, ) columns = values.complete_predictor_columns( qualified, self.clone_population.frame, drawn @@ -474,6 +506,7 @@ def verify_materialized_current_survey_predictors( apply_states, host_pins, n_estimators=100, + demographic_conditioning=False, ): """Use actual executor-observed Population and authenticated typed artifacts. @@ -482,14 +515,21 @@ def verify_materialized_current_survey_predictors( owner, metadata value, weight and mass ledger of the expected full result. """ qualified = values.qualify_current_survey_predictors( - preparation, allocated_population, clone_population + preparation, + allocated_population, + clone_population, + demographic_conditioning=demographic_conditioning, ) require( projection == qualified.projection and matrix == qualified.matrix, "MATERIALIZED_SOURCE", ) drawn = read_current_survey_draws( - matrix, matrix_producer_key, raw_draws, apply_states + matrix, + matrix_producer_key, + raw_draws, + apply_states, + demographic_conditioning=demographic_conditioning, ) attach = current_survey_predictor_nodes( qualified, diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py b/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py new file mode 100644 index 000000000..8957fd85f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py @@ -0,0 +1,295 @@ +"""Opted-in financial conditioning over actual issuers and invented sources.""" + +import json +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments + +from microcosm.build.us_runtime import current_survey_predictors as values +from microcosm.build.us_runtime import graph_current_survey_predictors as graph +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.fit import qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import WeightKind +from microcosm.graph import compile_graph, run_graph + + +@pytest.fixture(scope="module") +def known_source(tmp_path_factory): + root = tmp_path_factory.mktemp("financial-demographics") + with pytest.MonkeyPatch.context() as patch: + arguments = _demographic_arguments(root, patch, zero=False) + live = survey.run_authenticated_survey_population( + **arguments, store_root=root / "store", clones=True, return_values=True + ) + yield live + values.source.verify_survey_population_preparation(live.preparation) + + +def _qualify(live, **kwargs): + return values.qualify_current_survey_predictors( + live.preparation, + live.allocated_population, + live.clone_population, + **kwargs, + ) + + +def test_demographic_features_fit_attach_and_require_replay(known_source): + live = known_source + original = live.clone_population + legacy = _qualify(live) + qualified = _qualify(live, demographic_conditioning=True) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + assert tuple(matrix.features) == values.DEMOGRAPHIC_FEATURES + assert tuple(graph.model_input.decode_recipient_matrix(legacy.matrix).features) == ( + values.FEATURES + ) + assert len(values.DEMOGRAPHIC_FEATURES) == 5 + assert not any("prior" in name for name in values.DEMOGRAPHIC_FEATURES) + pd.testing.assert_frame_equal(qualified.native_money, legacy.native_money) + details = qualified.evidence["model_judgments"]["demographic_conditioning"] + assert details["asec_sex_and_state_observation_year"] == 2025 + assert details["acs_sex_and_state_observation_year"] == 2024 + assert details["state_is_income_year_residence_claim"] is False + # The maintained fixture reverses ASEC source CSV rows before issuance. + # These independently expected native IDs therefore test the keyed join. + origins = qualified.origins + asec_ids = origins.loc[origins.source.eq("asec")].sort_values("native_person_id") + assert asec_ids.native_person_id.tolist() == [105, 106, 107, 108] + donor = qualified.donor_columns.loc[asec_ids.index] + np.testing.assert_array_equal(donor.survey_predictor_is_female, [0, 1, 0, 1]) + np.testing.assert_array_equal(donor.survey_predictor_state_fips, [6, 6, 36, 36]) + assert matrix.features.survey_predictor_is_female.eq(0).all() + assert matrix.features.survey_predictor_state_fips.eq(6).all() + assert qualified.donor_frame.resolve_weights("person").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + qualified.donor_frame.resolve_weights("person").values, + legacy.donor_frame.resolve_weights("person").values, + ) + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, original.frame, host_pins=pins, n_estimators=2 + ) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls( + live.preparation, + live.allocated_population, + original, + host_pins=pins, + n_estimators=2, + demographic_conditioning=True, + ) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(len(values.TARGETS)): + artifacts.extend( + ( + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ) + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + payload = live.store.load_bytes(receipt.opaque_artifacts[name]) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + payload = loaded[f"{graph.FIT_PREFIX}.{i:03d}", "model"] + model = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert model.target == target + assert model._target_model.columns == ( + *values.DEMOGRAPHIC_FEATURES, + *values.TARGETS[:i], + ) + final = observed[graph.ATTACH_NODE] + verification = { + "population": final, + "projection": loaded[graph.PROJECTION_NODE, "projection"], + "matrix": loaded[graph.PROJECTION_NODE, "matrix"], + "matrix_producer_key": manifest.node(graph.PROJECTION_NODE).key, + "raw_draws": tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ), + "apply_states": tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ), + "host_pins": pins, + "n_estimators": 2, + "demographic_conditioning": True, + } + result = graph.verify_materialized_current_survey_predictors( + live.preparation, live.allocated_population, original, **verification + ) + assert result["all_output_cells_available"] + assert not result["release_eligible"] + with pytest.raises(ValueError, match="MATERIALIZED_SOURCE"): + graph.verify_materialized_current_survey_predictors( + live.preparation, + live.allocated_population, + original, + **{**verification, "demographic_conditioning": False}, + ) + for entity in original.frame.entities: + before, after = original.frame.table(entity), final.frame.table(entity) + retained = [ + c for c in before if entity != "person" or c not in values.OUTPUTS + ] + pd.testing.assert_frame_equal( + before[retained], after[retained], check_exact=True + ) + assert final.version == original.version + assert final.weight_kind == original.weight_kind + assert final.mass_ledger == original.mass_ledger + for entity in original.design_weights: + np.testing.assert_array_equal( + final.design_weights[entity], original.design_weights[entity] + ) + np.testing.assert_array_equal( + final.frame.weights_for(entity).values, + original.frame.weights_for(entity).values, + ) + expected_owners = dict(original.owners) + expected_owners.update( + {("person", c): graph.ATTACH_NODE for c in values.OUTPUTS} + ) + assert dict(final.owners) == expected_owners + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + results.append(final) + graph.replay.same_replayed_population(*results) + + +def test_unresolved_source_demographics_refuse_opt_in_and_preserve_default( + tmp_path, monkeypatch +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=True) + live = survey.run_authenticated_survey_population( + **arguments, store_root=tmp_path / "store", clones=True, return_values=True + ) + original = values.source._frame_identity(live.preparation.frame) + default = _qualify(live) + assert not default.demographic_conditioning + with pytest.raises( + ValueError, match="CURRENT_SURVEY_PREDICTOR_DEMOGRAPHIC_UNKNOWN" + ): + _qualify(live, demographic_conditioning=True) + assert values.source._frame_identity(live.preparation.frame) == original + assert _qualify(live).projection == default.projection + source_values = ( + values.observed_geography.demographics.qualify_current_asec_demographics( + live.preparation + ) + ) + assert source_values.person.is_female.isna().any() + assert source_values.household.state_fips.isna().any() + assert json.loads(source_values.receipt)["sex_unknown_persons"] == 1 + + +def test_last_money_owner_borrow_cannot_change_detached_demographic_values( + known_source, +): + live = known_source + captured, changed = [], [] + parent = values.source.asec_native._ISSUED[ + id(live.preparation._checked()[2].native[1]) + ][2].parent + ready_code = parent.ready.__code__ + + def profile(frame, event, arg): + if event != "return": + return + if frame.f_code is values._demographic_features.__code__ and arg is not None: + captured.append(arg[2]) + elif frame.f_code is ready_code and captured and not changed: + table = captured[-1].person + table.loc[table.index[0], "is_female"] = not bool(table.iloc[0].is_female) + changed.append(True) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(ValueError, match="DEMOGRAPHIC_PROJECTION_CHANGED"): + _qualify(live, demographic_conditioning=True) + finally: + sys.setprofile(previous) + assert changed == [True] + # Only a detached projection changed; the live source can still qualify. + assert _qualify(live, demographic_conditioning=True).demographic_conditioning From c107bf627e4e85c47b58b3cb0451100bdec24c5d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 19:06:22 -0400 Subject: [PATCH 036/255] Update launch review scope and financial compatibility evidence --- docs/us-launch-review.md | 31 +++++++++++++------ ...financial-default-2-controls-20260909.json | 27 ++++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 experiments/us-financial-default-2-controls-20260909.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 0dd9b2772..94f6b3251 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -14,10 +14,12 @@ exports. Enrichment clones inherit their household's assigned location. The PUF operator acts on the combined survey frame. A separate ASEC–PUF base is not the intended architecture. -Block-first assignment is a required change, not completed functionality in this -branch. The existing geography operator chooses a joint tract/district cell. -The US and UK assignment contracts, existing implementation and remaining graph -work are distinguished in [Geography assignment](geography-assignment.md). +The optional ten-node survey prefix now assigns a block before cloning, and its +thirteen-node age-calibration extension passes cold execution and required +replay on invented originals. Native support and national/CD acceptance remain +pending. The earlier joint tract/district operator is a separate path. See +[Geography assignment](geography-assignment.md) for the country contracts, +implementation and source boundaries. | Area | Main source entry points | What to review | | --- | --- | --- | @@ -26,7 +28,7 @@ work are distinguished in [Geography assignment](geography-assignment.md). | Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | | Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | | Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | -| Geography | `us_runtime/{puma_ladder,puma_ladder_sources,graph_geography}.py` | Existing joint-cell implementation; pending block assignment, versioned projections and source/PUMA preservation | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Block-first assignment, observed-source constraints, versioned mappings and clone inheritance; the older joint-cell operator remains separate | | Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | | Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | @@ -141,10 +143,14 @@ from the benchmark's accepted baseline. The following work remains open: -1. Finish current survey predictors and demographic graph bindings, then connect - them to the genuine combined survey population. -2. Project the genuine canonical59 donor into the explicit 55-output profile, - bind the survey Social Security total as predictor nine, complete survey +1. Connect the accepted source-qualified demographic financial model to the + atomic survey prefix and genuine combined survey population. Three opted-in + financial controls and two existing default-path fit/replay controls pass; + native fit quality remains unassessed. The composed runner is in progress. +2. Qualify recipient return roles and adopt an explicit Social Security + conditioning measurement before native PUF55 fitting. The genuine + 207,692-return canonical donor has already passed its 55-output projection; + that does not establish recipient matching. Complete survey beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed 128 profile/compatibility controls and 26 current-survey predictor controls; see [the implementation status](current-survey-puf59-progress.md) and @@ -172,6 +178,13 @@ The following work remains open: These are active workstreams. No release, merge or deployment is implied by this draft, and the source changes do not relax the outstanding acceptance checks. +The population-only Census API adapter passes thirty-five invented controls. +The first native Delaware acquisition obtained only P1_001N block/state responses +and CD119/PUMA mapping bytes from the fixed Census endpoints. Those bytes are +awaiting maintained source normalization; successful download is not source or +population acceptance. Delaware-only support must not enter the national survey +graph. Complete support for the fifty states and DC is required first. + ## Related reviews - [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): diff --git a/experiments/us-financial-default-2-controls-20260909.json b/experiments/us-financial-default-2-controls-20260909.json new file mode 100644 index 000000000..1d3d92fef --- /dev/null +++ b/experiments/us-financial-default-2-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 93.988173, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 526712832, + "publisher_provenance_established": false, + "receipt_sha256": "d2775bd9e1c1e14dabaa9b7162070310fdd00adc8ea996084932692bce8f9dc9", + "release_eligible": false, + "result": 0, + "scope": "Two existing default financial predictor cold/required replay cases, with the accepted optional demographic implementation left disabled. Invented original sources only; no native/quality/release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 98.86093041597633 +} From c6f6b06dde8161fd46fb19e7f29785976a5fbf45 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 19:43:09 -0400 Subject: [PATCH 037/255] Compose atomic survey geography with financial imputation --- .../us-atomic-financial-composition.added.md | 1 + docs/us-launch-review.md | 34 +- ...-atomic-financial-4-controls-20260909.json | 29 + ...ncial-composition-acceptance-20260909.json | 42 ++ ...nancial-corrected-2-controls-20260909.json | 24 + ...s-atomic-native-de-1-control-20260909.json | 42 ++ ...ative-de-corrected-1-control-20260909.json | 175 ++++++ .../us_runtime/current_survey_predictors.py | 102 +++- .../graph_atomic_survey_financial.py | 556 ++++++++++++++++++ .../graph_current_survey_predictors.py | 23 + ...t_us_atomic_block_api_sources_native_de.py | 265 +++++++++ .../test_us_graph_atomic_survey_financial.py | 235 ++++++++ 12 files changed, 1505 insertions(+), 23 deletions(-) create mode 100644 changelog.d/us-atomic-financial-composition.added.md create mode 100644 experiments/us-atomic-financial-4-controls-20260909.json create mode 100644 experiments/us-atomic-financial-composition-acceptance-20260909.json create mode 100644 experiments/us-atomic-financial-corrected-2-controls-20260909.json create mode 100644 experiments/us-atomic-native-de-1-control-20260909.json create mode 100644 experiments/us-atomic-native-de-corrected-1-control-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py create mode 100644 packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py create mode 100644 packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py diff --git a/changelog.d/us-atomic-financial-composition.added.md b/changelog.d/us-atomic-financial-composition.added.md new file mode 100644 index 000000000..a28385f6c --- /dev/null +++ b/changelog.d/us-atomic-financial-composition.added.md @@ -0,0 +1 @@ +Compose survey atomic geography and current financial imputation in a checked twenty-node graph, retaining geography through cloning and financial attachment. Add native Delaware population-only source normalization and complete support readback controls. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 94f6b3251..56b8681f4 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -16,8 +16,11 @@ intended architecture. The optional ten-node survey prefix now assigns a block before cloning, and its thirteen-node age-calibration extension passes cold execution and required -replay on invented originals. Native support and national/CD acceptance remain -pending. The earlier joint tract/district operator is a separate path. See +replay on invented originals. A twenty-node extension now carries atomic +geography through current financial imputation and required replay. Native +Delaware block support passes source normalization; complete national support +and national/CD acceptance remain pending. The earlier joint tract/district +operator is a separate path. See [Geography assignment](geography-assignment.md) for the country contracts, implementation and source boundaries. @@ -143,10 +146,15 @@ from the benchmark's accepted baseline. The following work remains open: -1. Connect the accepted source-qualified demographic financial model to the - atomic survey prefix and genuine combined survey population. Three opted-in - financial controls and two existing default-path fit/replay controls pass; - native fit quality remains unassessed. The composed runner is in progress. +1. Run the accepted twenty-node atomic and financial composition on the genuine + combined survey population. Its cold/required replay, complete retained + fields and geography, default three-feature versus explicit five-feature + conditioning, and both final mutation refusals pass on invented inputs. + Two test-code mistakes in the initial four-case run were corrected and + rerun separately; the passing mutation cases were preserved. See the + [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). + Native fit quality and admission of the financial result into subsequent + calibration remain unassessed. 2. Qualify recipient return roles and adopt an explicit Social Security conditioning measurement before native PUF55 fitting. The genuine 207,692-return canonical donor has already passed its 55-output projection; @@ -179,10 +187,16 @@ These are active workstreams. No release, merge or deployment is implied by this draft, and the source changes do not relax the outstanding acceptance checks. The population-only Census API adapter passes thirty-five invented controls. -The first native Delaware acquisition obtained only P1_001N block/state responses -and CD119/PUMA mapping bytes from the fixed Census endpoints. Those bytes are -awaiting maintained source normalization; successful download is not source or -population acceptance. Delaware-only support must not enter the national survey +The native Delaware source control preserves all 15,317 populated blocks and +reconciles them to the independent Census state total of 989,948. It counts +4,881 zero-population blocks, checks CD119/PUMA joins and exactly reads back its +79,769-byte support artifact. The run took 10.94 seconds and 1.735 GB peak RSS. +The first attempt correctly refused an incomplete ZIP-member declaration; +a separate metadata probe established the six-member archive roster, and the +corrected check still decompresses only `NationalCD119.txt`. The +[accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) +preserves the original failed evidence. This is not national population or +release acceptance. Delaware-only support must not enter the national survey graph. Complete support for the fifty states and DC is required first. ## Related reviews diff --git a/experiments/us-atomic-financial-4-controls-20260909.json b/experiments/us-atomic-financial-4-controls-20260909.json new file mode 100644 index 000000000..508aeb133 --- /dev/null +++ b/experiments/us-atomic-financial-4-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "control_accepted": false, + "cpu_seconds": 467.05899700000003, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 565133312, + "publisher_provenance_established": false, + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "release_eligible": false, + "result": 1, + "scope": "Four invented atomic-plus-financial controls: twenty-node cold/required replay with five qualified features, default three-feature compatibility, detached money mutation refusal at final support return, and final geography mutation refusal. No fit quality, native admission, PUF recipient admission, budget successor, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 2, + "skipped": 0, + "tests": 4 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "8d0fc394f7f59bf64f387cb137819c1789bfdf310f1573a4b41d4bd3ea3cfcdb" + }, + "unexpected_refusals": {}, + "wall_seconds": 472.7063618339598 +} diff --git a/experiments/us-atomic-financial-composition-acceptance-20260909.json b/experiments/us-atomic-financial-composition-acceptance-20260909.json new file mode 100644 index 000000000..d6f21bf54 --- /dev/null +++ b/experiments/us-atomic-financial-composition-acceptance-20260909.json @@ -0,0 +1,42 @@ +{ + "calibration_successor_accepted": false, + "completed_controls": [ + "cold and required replay across all20 actual graph nodes", + "complete retained fields, metadata, IDs, weights, owners and clone inheritance", + "explicit five-feature conditioning and default three-feature compatibility", + "refuse detached money mutation during final support return", + "refuse materialized geography mutation during final owner return" + ], + "fit_quality_accepted": false, + "graph_nodes": 20, + "native_payloads_read": 0, + "native_survey_replay_accepted": false, + "negative_controls_repeated": false, + "population_scope": "invented original survey inputs; native survey replay pending", + "puf_attachment_accepted": false, + "release_eligible": false, + "runs": [ + { + "experiment": "us-atomic-financial-4-controls-20260909.json", + "failed_test_code_defects": [ + "nonexistent assertion helper name", + "incorrect expected refusal string" + ], + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "result": "two negative mutation controls passed; two test-code defects failed" + }, + { + "experiment": "us-atomic-financial-corrected-2-controls-20260909.json", + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "result": "two corrected tests passed with the same implementation and full cold/required fixture" + } + ], + "status": "invented_composition_controls_passed", + "test_only_corrections": true, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "1b441049e527f24f339e2c2fed289ae8ff434015d7bc0b025cebda87f32d52d0" + } +} diff --git a/experiments/us-atomic-financial-corrected-2-controls-20260909.json b/experiments/us-atomic-financial-corrected-2-controls-20260909.json new file mode 100644 index 000000000..042978bd3 --- /dev/null +++ b/experiments/us-atomic-financial-corrected-2-controls-20260909.json @@ -0,0 +1,24 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 248.723104, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 552976384, + "publisher_provenance_established": false, + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "release_eligible": false, + "result": 0, + "scope": "Identical unexecuted-v2 source closure; select only the two previously failing corrected tests, retaining their actual cold/required replay fixture. Passed original negative controls are not rerun. No ongoing successor implementation, fit quality, native admission, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": {}, + "unexpected_refusals": {}, + "wall_seconds": 252.2364875409985 +} diff --git a/experiments/us-atomic-native-de-1-control-20260909.json b/experiments/us-atomic-native-de-1-control-20260909.json new file mode 100644 index 000000000..84d9bb62e --- /dev/null +++ b/experiments/us-atomic-native-de-1-control-20260909.json @@ -0,0 +1,42 @@ +{ + "archive_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ], + "automatic_retry": false, + "code_resources_rechecked": 12, + "corrected_control_executed": false, + "cpu_seconds": 3.002104, + "diagnosis": "The acquired CD archive contains six members; the test incorrectly declared only NationalCD119.txt. The exact-roster validation correctly refused it.", + "diagnostic_receipt_sha256": "54405497257274709b458baad3d0a8b0f4fd2727406d794d42f962d12cd851ca", + "failure_class": "ValueError", + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "metadata_probe_member_contents_opened": 0, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "peak_rss_bytes": 372555776, + "receipt_sha256": "063d36af8b4a8352bb573396b39a921c21c8703d484e32542f24680702428df0", + "release_eligible": false, + "reserved_housing_targets_read": false, + "result": 1, + "scope": "One native DE original P1 population/state-total plus CD119/PUMA source normalization control. Exact four input pins, block preservation and state reconciliation, full support readback. No national coverage, survey Frame, source admission or release acceptance.", + "selected_member": "NationalCD119.txt", + "selected_member_size_bytes": 163499110, + "source_and_owned_files": 497, + "source_control_accepted": false, + "status": "failed_pending_diagnosis", + "test_counts": { + "errors": 0, + "failures": 1, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.633959750004578 +} diff --git a/experiments/us-atomic-native-de-corrected-1-control-20260909.json b/experiments/us-atomic-native-de-corrected-1-control-20260909.json new file mode 100644 index 000000000..1fea35f10 --- /dev/null +++ b/experiments/us-atomic-native-de-corrected-1-control-20260909.json @@ -0,0 +1,175 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 9.85925, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "national_coverage": false, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "normalization": { + "exact_support_readback": true, + "national_coverage": false, + "protocol": "microcosm.us.native-de-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 536870912, + "max_response_rows": 250000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 805306368, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "10": 989948 + }, + "populated_blocks": 15317, + "population_total": 989948, + "selected_geography_bytes": 165910870, + "state_population": { + "10": 989948 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + } + ], + "puma": { + "selected_state_tract_mappings": 262, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "10" + ], + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d" + }, + "source_receipt_sha256": "b0783c16860e1333b271876a48f0b82d16c543813c93904c3eb460aecfc1111b", + "state_fips": [ + "10" + ], + "status": "source_control_passed", + "support_filename": "de-atomic-support.npz", + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d", + "support_size_bytes": 79769, + "survey_frame_created": false + }, + "normalization_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "peak_rss_bytes": 1734787072, + "receipt_sha256": "ea260a22f8701839aa0ac9beab98d82e17738cf9359570a0317e26338697d26e", + "release_eligible": false, + "result": 0, + "scope": "Correct only the native DE test archive roster to the six metadata-observed names; selected decompression/CRC remains solely NationalCD119.txt. No adapter, input, other source or guard behavior change. Failed original run preserved.", + "source_and_owned_files": 497, + "source_control_accepted": true, + "status": "DE_source_control_passed", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py": { + "after": "4b918e4d28044045faad6618dd931077b3b07086929c34aa31cff049fdf096a2", + "before": "e22a550f4e95592a6bd596f32ffd772d6f43117f284ce0eebe992b0de42ed7ab" + } + }, + "unexpected_refusals": {}, + "wall_seconds": 10.939260708983056 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py index 80353fa66..1dc823e39 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -248,6 +248,57 @@ class QualifiedSurveyPredictors: origins: pd.DataFrame evidence: dict demographic_conditioning: bool = False + geography_config_payload: bytes | None = None + + +def _qualified_seal(value): + """Seal the actual returned values across the last source/support borrow.""" + require(type(value) is QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE") + geography = host.survey_budget.geography + tables = [] + for table in (value.donor_columns, value.native_money, value.origins): + require(type(table) is pd.DataFrame, "QUALIFIED_TABLE_TYPE") + metadata = codec.encode_json( + { + "columns": list(table.columns), + "column_axis": [ + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + ], + "index_axis": [ + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ], + "shape": list(table.shape), + } + ) + selected = np.ones(len(table), dtype=np.bool_) + parts = tuple( + (str(series.dtype), geography._storage_parts(series, selected)) + for series in ( + pd.Series(table.index.to_numpy(copy=False)), + *(table[c] for c in table), + ) + ) + tables.append((metadata, parts)) + return ( + value.projection, + value.matrix, + value.demographic_conditioning, + value.geography_config_payload, + codec.encode_json(value.evidence), + tuple( + geography._population_stamp( + host.survey_budget.Population.from_frame( + frame, "survey_predictors.detached_values" + ) + ) + for frame in (value.source_frame, value.donor_frame) + ), + tuple(tables), + ) def qualify_current_survey_predictors( @@ -256,6 +307,7 @@ def qualify_current_survey_predictors( clone_population, *, demographic_conditioning=False, + geography_config=None, ): """Read the actual full-parent money owner once, then seal all retained state. @@ -267,6 +319,7 @@ def qualify_current_survey_predictors( it does not claim the observation windows are equivalent. """ predictors = feature_columns(demographic_conditioning) + config_payload = host.survey_budget._config_payload(geography_config) require( type(preparation) is source.AuthenticatedSurveyPopulationPreparation, "PREPARATION_TYPE", @@ -276,8 +329,13 @@ def qualify_current_survey_predictors( view = source.CheckedSurveyPopulationView( entry[1], state.context, state.frame, state.plan, json.loads(entry[1]) ) - _, allocation = host.survey_budget._initial( - view, allocated_population, clone_population + _, allocation, geography_binding = host.survey_budget._initial( + view, + allocated_population, + clone_population, + preparation=preparation, + geography_config=geography_config, + _with_geography_binding=True, ) seals = tuple( host.survey_budget._population_identity(p) @@ -430,6 +488,8 @@ def qualify_current_survey_predictors( "source_admission_issued": False, "release_eligible": False, } + if geography_binding is not None: + evidence["atomic_geography"] = json.loads(geography_binding) matrix = model_input.encode_recipient_matrix( features.loc[pids[acs]], entity="person", entity_ids=pids[acs].astype(" Date: Wed, 9 Sep 2026 19:50:12 -0400 Subject: [PATCH 038/255] Add source-preserving Social Security conditioning for PUF55 --- .../us-puf55-survey-ss-measurement.added.md | 5 + docs/puf55-survey-ss-measurement-decision.md | 107 +++++ docs/us-launch-review.md | 12 +- ...y-ss-measurement-31-controls-20260909.json | 28 ++ .../build/us_runtime/full_puf_enrichment.py | 4 +- .../us_runtime/puf55_survey_ss_measurement.py | 320 ++++++++++++++ .../test_us_puf55_survey_ss_measurement.py | 399 ++++++++++++++++++ 7 files changed, 872 insertions(+), 3 deletions(-) create mode 100644 changelog.d/us-puf55-survey-ss-measurement.added.md create mode 100644 docs/puf55-survey-ss-measurement-decision.md create mode 100644 experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py create mode 100644 packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py diff --git a/changelog.d/us-puf55-survey-ss-measurement.added.md b/changelog.d/us-puf55-survey-ss-measurement.added.md new file mode 100644 index 000000000..ebe159455 --- /dev/null +++ b/changelog.d/us-puf55-survey-ss-measurement.added.md @@ -0,0 +1,5 @@ +Add an authenticated filer/joint-spouse Social Security report-sum measurement +for PUF55 conditioning, with an explicit eight-predictor fallback for incomplete +reports and preservation of all survey Social Security cells and masks. Record +31 passing invented controls and the remaining recipient integration and +measurement limitations. diff --git a/docs/puf55-survey-ss-measurement-decision.md b/docs/puf55-survey-ss-measurement-decision.md new file mode 100644 index 000000000..fc12e6b85 --- /dev/null +++ b/docs/puf55-survey-ss-measurement-decision.md @@ -0,0 +1,107 @@ +# PUF55 survey Social Security conditioning decision + +On 2026-09-09 the build owner adopted +`social_security_filer_joint_spouse_report_sum_proxy` for PUF matching. This +supersedes the pending measurement judgment in +[survey-social-security-source.md](survey-social-security-source.md). The +decision authorizes this conditioning measurement and its explicit fallback; +it does not assert donor/recipient equivalence, allocate beneficiaries, complete +Social Security components, approve a fit, or admit a release. + +For each modeled return, include exactly one `HEAD` and, only when the actual +`filing_status_input` is `JOINT`, exactly one `SPOUSE`. Sum their available +nonnegative source report totals. Preserve combined family and child reports +without splitting, deduplicating, or subtracting amounts. Exclude dependents +from this return feature without changing any person's observed or unknown +Social Security cells or component masks. + +The public helper requires the actual authenticated survey preparation and +qualifies current source reports. It uses the exact modeled return roles +retained by that preparation. It does **not** rerun tax-unit construction using +current money. That limitation is accepted for this conditioning stage; later +production integration must qualify the chosen role/current-money relationship +explicitly. A caller-made role table or decoded receipt grants no authority. +The PUF disclosure-capped return-size predictor is not a person count, and the +coarsened PUF filing class 2 cannot identify an actual spouse: surviving-spouse +returns also map to that class. + +## Availability and refusal + +An available report-sum means every included reporter has an in-universe, +finite source amount. It does not mean those reporters legally own all the +reported benefits. Known zero reports are valid. Unknown components do not +invalidate a known total. Family/child ambiguity remains an explicit limitation +and does not itself trigger missingness. + +An unavailable included report, including an under-15 filer outside the source +question universe, leaves the proxy unknown and selects the separately named +`PUF55_SURVEY_SS_NO_TOTAL` profile. This profile has the same 55 outputs and +eight predictors. The existing `PUF55_SURVEY_SS` retains nine predictors. +Neither produces the four Social Security components. There is no partial sum, +dependent-zero convention, component-sum fallback, silent row deletion, or +implicit change to the 59/65-output profiles. + +Missing/invalid roles, duplicate or orphan membership, conflicting source +values, nonnumeric amounts, negative/nonfinite observations, and known amounts +outside the reporting universe refuse. They cannot route to eight predictors: +the first two PUF predictors still require valid roles. A source qualifier's own +refusals remain refusals; the fallback does not override them. Graph integration +must bind both disjoint recipient routes, account for each recipient exactly +once, authenticate inherited clone membership, and execute the corresponding +complete fit/apply/attachment chain. This numerical helper does none of that +graph admission or fitting. + +## Measurement limits + +[IRS Publication 915 (2015), pages 2–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=2) +assigns benefits to the legal beneficiary, including a child whose parent +receives the check; joint filers combine spouses' benefits. The return amount +uses benefits net of repayments and includes the Social Security equivalent +part of tier-1 Railroad Retirement, excluding other pension portions. Form +1040EZ omits benefits, and some recipients need not file. These rules motivate +the selected return members but do not establish survey report ownership or +PUF editing conventions. [SSA's representative-payee guidance](https://www.ssa.gov/payee/faqrep.htm) +likewise distinguishes the person managing a payment from its beneficiary. + +The [2025 ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and identifies payments on behalf of children. +Its source total covers calendar 2024 for a 2025 interview. The +[ACS 2024 questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +uses the prior twelve months, permits jointly received income entirely on one +person's record, and combines Social Security with Railroad Retirement. +[ACS SSP metadata, PDF page 45](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf#page=45) +defines the age-15 universe, rounded/topcoded amounts and ADJINC price +adjustment. The helper preserves those distinct observation windows; it does +not calendarize ACS or estimate repayments/Railroad adjustments. + +The ninth donor predictor remains the existing transported 2015 `E02400` +carrier under its declared 2024 money-growth convention. Carrier zeros do not +prove absent benefits. Exact 2015 publisher editing/omission rules remain +unverified; the [IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files) +reports the 2012–2015 files unavailable. Matching is therefore conditional on +this documented proxy assumption, not a newly authenticated observed 2024 +return total. Using eight predictors does not remove donor coverage concerns +for nonfilers or resolve assumptions in the other donor outcomes. + +## Diagnostics required before relying on a fit + +Family payments on selected reporters may overstate their own benefits; +payments reported elsewhere may understate them. An all-member sum includes +dependent reports and cannot solve this attribution problem. Topcoding, +allocation, return editing and transported donor money can distort zeros and +tails. Growth of fixed 2015 amounts does not predict 2024 cohort composition. + +Report route counts, known zero/positive distributions and tails by survey, +actual filing status, age, child-report reasons, allocation provenance, and +multiple-return household status. Preserve original-household holdouts across +clones. Compare design-weighted held-out performance and recipient overlap for +eight versus nine predictors; donor-only validation cannot establish cross-source +measurement validity. An adult in-universe all-member report sum may be a +labelled sensitivity diagnostic, with under-15 unknownness retained. It is not +the default feature. Benefit-component modeling and release checks remain +separate work. + +This change provides source-bound numerical values, explicit route metadata, +and an isolated graph profile declaration. Actual source guards, route-aware +graph integration, donor projection for the fallback, fits and diagnostics are +subsequent verification steps, not results claimed by this decision. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 56b8681f4..0714b6e39 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -155,8 +155,16 @@ The following work remains open: [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). Native fit quality and admission of the financial result into subsequent calibration remain unassessed. -2. Qualify recipient return roles and adopt an explicit Social Security - conditioning measurement before native PUF55 fitting. The genuine +2. Integrate the adopted Social Security conditioning measurement and qualify + recipient return roles before native PUF55 fitting. Thirty-one invented + controls pass the authenticated filer/joint-spouse report-sum proxy, source + preservation, missingness and refusal checks. An incomplete selected report + uses a separately declared eight-predictor route; a known total uses nine. + The helper uses the preparation's retained modeled roles, whose relationship + to current-money tax-unit construction still needs explicit qualification. + See the [measurement decision](puf55-survey-ss-measurement-decision.md) and + [scoped control result](../experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json). + Route-aware graph fitting and attachment remain pending. The genuine 207,692-return canonical donor has already passed its 55-output projection; that does not establish recipient matching. Complete survey beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed diff --git a/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json new file mode 100644 index 000000000..581263a6f --- /dev/null +++ b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json @@ -0,0 +1,28 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 38.894051, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 517455872, + "publisher_provenance_established": false, + "receipt_sha256": "ac932822b7693129e3be97923bbc6798f8883f9fad0365b76b671117f7770f8e", + "release_eligible": false, + "result": 0, + "scope": "31 invented filer/joint-spouse Social Security report-sum measurement, missingness/refusal, authenticated retained-source callback controls, and isolated55-output/eight-predictor declarations. No native source reads, route-aware graph fitting, donor transport acceptance, beneficiary allocation, source admission or release acceptance.", + "source_and_owned_files": 498, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 31 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py": "f76bdc251811286e93f03d3aa9f5caac8548b60595739b5b17efce18d38e715e", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py": "06c4441f6397a69154f39ae905986a0a7928859cd38d7896fd1d8910ed5504cd", + "packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py": "6e7b63e2ac1f7bc2cd0cd6902a89facb91685f2c1f5c8b0d184a7e89143b3dce" + }, + "unexpected_refusals": {}, + "wall_seconds": 41.406911208992824 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py index cae79e8f2..2982805f9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py @@ -68,6 +68,7 @@ class PufOutputProfile(Enum): FULL65 = "full65" PUF59 = "puf59" PUF55_SURVEY_SS = "puf55_survey_ss" + PUF55_SURVEY_SS_NO_TOTAL = "puf55_survey_ss_no_total" @property def predictors(self): @@ -94,7 +95,7 @@ def donor_auxiliary_columns(self): @property def person_outputs(self): - if self is PUF55_SURVEY_SS: + if self in (PUF55_SURVEY_SS, PUF55_SURVEY_SS_NO_TOTAL): return PUF55_SURVEY_SS_PERSON_OUTPUTS return PERSON_OUTPUTS @@ -117,6 +118,7 @@ def phase(self): FULL65 = PufOutputProfile.FULL65 PUF59 = PufOutputProfile.PUF59 PUF55_SURVEY_SS = PufOutputProfile.PUF55_SURVEY_SS +PUF55_SURVEY_SS_NO_TOTAL = PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL def require_puf_output_profile(profile): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py new file mode 100644 index 000000000..d5abe6afb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py @@ -0,0 +1,320 @@ +"""Filer/joint-spouse report sums for conditioning, never beneficiary amounts. + +The public entry point requires a live authenticated survey preparation. Its +retained modeled return roles are inputs, not roles inferred from the coarsened +PUF filing-class predictor. The private arithmetic is not a source issuer. +Graph hosts must retain/requalify the preparation and authenticate clone +inheritance, routing and typed model edges independently. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import current_social_security_source as reports +from . import full_puf_enrichment as full +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.puf55-survey-ss-measurement.v1" +MEASUREMENT = "social_security_filer_joint_spouse_report_sum_proxy" +TOTAL = full.SURVEY_SS_TOTAL_PREDICTOR +KNOWN = "puf_conditioning_social_security_total_known" +ROUTE = "puf55_conditioning_profile" +COLUMNS = (TOTAL, KNOWN, ROUTE) +_PERSON_COLUMNS = ("person_id", "person_tax_unit_id", "tax_unit_role_input") +_UNIT_COLUMNS = ("tax_unit_id", "filing_status_input") +_REPORT_COLUMNS = ( + "social_security_source_total", + "source_reporting_universe", +) +_STATUSES = {"SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE"} + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_SS_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _axis(values): + _require( + values.dtype == np.dtype("int64") + and not values.isna().any() + and values.ge(0).all(), + "AXIS", + ) + + +def _measure(person, tax_unit, report): + """Pure numerical checks on already-qualified values; no admission issued.""" + for table, columns in ( + (person, _PERSON_COLUMNS), + (tax_unit, _UNIT_COLUMNS), + (report, _REPORT_COLUMNS), + ): + _require( + type(table) is pd.DataFrame + and table.columns.is_unique + and set(columns) <= set(table), + "COLUMNS", + ) + for column in (person.person_id, person.person_tax_unit_id, tax_unit.tax_unit_id): + _axis(column) + _require( + len(tax_unit) > 0 + and person.person_id.is_unique + and tax_unit.tax_unit_id.is_unique + and set(person.person_tax_unit_id) == set(tax_unit.tax_unit_id) + and report.index.name == "person_id" + and report.index.dtype == np.dtype("int64") + and report.index.is_unique + and set(report.index) == set(person.person_id), + "MEMBERSHIP", + ) + roles, statuses = person.tax_unit_role_input, tax_unit.filing_status_input + _require( + not roles.isna().any() + and roles.map(lambda v: type(v) is str).all() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all() + and not statuses.isna().any() + and statuses.map(lambda v: type(v) is str).all() + and statuses.isin(_STATUSES).all(), + "ROLE_VALUES", + ) + # Validate every return's roles before deciding any missing-report route. + # SURVIVING_SPOUSE maps to PUF class 2 elsewhere but has no actual spouse. + groups = person.groupby("person_tax_unit_id", sort=False) + included = [] + for tid, status in tax_unit[list(_UNIT_COLUMNS)].itertuples(index=False, name=None): + members = groups.get_group(tid) + head = members.tax_unit_role_input.eq("HEAD") + spouse = members.tax_unit_role_input.eq("SPOUSE") + _require( + head.sum() == 1 and spouse.sum() == int(status == "JOINT"), + "ROLE_STRUCTURE", + ) + included.append(members.loc[head | spouse, "person_id"].to_numpy(copy=True)) + amount, universe = report[_REPORT_COLUMNS[0]], report[_REPORT_COLUMNS[1]] + _require( + pd.api.types.is_numeric_dtype(amount.dtype) + and not pd.api.types.is_bool_dtype(amount.dtype) + and not pd.api.types.is_complex_dtype(amount.dtype), + "REPORT_PHYSICAL_TYPE", + ) + _require(universe.dtype == np.dtype("bool"), "UNIVERSE_PHYSICAL_TYPE") + if pd.api.types.is_integer_dtype(amount.dtype): + _require(amount.dropna().between(-(2**53), 2**53).all(), "REPORT_PRECISION") + numeric = amount.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + available = np.isfinite(numeric) + _require( + (np.isnan(numeric) | available).all() and (numeric[available] >= 0).all(), + "REPORT_DOMAIN", + ) + _require( + np.isnan(numeric[~universe.to_numpy()]).all(), "OUTSIDE_UNIVERSE_OBSERVATION" + ) + values, known, routes = [], [], [] + missing_people, included_people = 0, 0 + for ids in included: + selected = report.loc[ids] + observed = selected[_REPORT_COLUMNS[0]].to_numpy( + dtype=np.float64, na_value=np.nan, copy=True + ) + present = selected[_REPORT_COLUMNS[1]].to_numpy() & np.isfinite(observed) + complete = bool(present.all()) + included_people += len(ids) + missing_people += int((~present).sum()) + # No pandas skipna, empty sum, dependent zero, component sum or partial + # report sum can manufacture a known measurement. + total = float(np.sum(observed)) if complete else np.nan + _require(not complete or np.isfinite(total), "SUM_OVERFLOW") + values.append(total) + known.append(complete) + routes.append( + (full.PUF55_SURVEY_SS if complete else full.PUF55_SURVEY_SS_NO_TOTAL).value + ) + result = pd.DataFrame( + { + TOTAL: np.asarray(values, dtype=np.float64), + KNOWN: np.asarray(known, dtype=bool), + ROUTE: pd.array(routes, dtype="string"), + }, + index=pd.Index(tax_unit.tax_unit_id.to_numpy(copy=True), name="tax_unit_id"), + ) + counts = { + "returns": len(result), + "included_reporters": included_people, + "excluded_dependents": len(person) - included_people, + "unavailable_included_reports": missing_people, + "nine_predictor_returns": int(result[KNOWN].sum()), + "eight_predictor_returns": int((~result[KNOWN]).sum()), + } + return result, counts + + +def _projection_digest(table): + _require( + type(table) is pd.DataFrame + and tuple(table) == COLUMNS + and table.index.name == "tax_unit_id" + and table.index.dtype == np.dtype("int64") + and table.index.is_unique + and table[TOTAL].dtype == np.dtype("float64") + and table[KNOWN].dtype == np.dtype("bool"), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for tid, total, known, route in table.itertuples(index=True, name=None): + payload = source._encode( + [ + int(tid), + None if np.isnan(total) else float(total).hex(), + bool(known), + route, + ] + ) + digest.update(len(payload).to_bytes(4, "big")) + digest.update(payload) + return digest.hexdigest() + + +def _source_identity(people, projection): + """Check detached source output against the exact retained source cells. + + SS qualification already authenticates original current ASEC literals and + ACS SSP adjustment. This additional comparison prevents a return callback + from changing its detached amount/universe without changing the live owner. + """ + _require(type(projection) is reports.CurrentSocialSecurityProjection, "REPORT_TYPE") + report = projection.person + _require( + report.index.is_unique and set(report.index) == set(people.person_id), + "SOURCE_AXIS", + ) + report = report.reindex(people.person_id) + channels = people[support_channel_column("person")] + _require( + np.array_equal(report.source.to_numpy(), channels.to_numpy()) + and np.array_equal( + report.native_person_id.to_numpy(), + people[spine_source_id_column("person")].to_numpy(), + ), + "SOURCE_ORIGIN", + ) + for channel, age_column, amount_column in ( + ("asec", "A_AGE", "SS_VAL"), + ("acs", "AGEP", "acs_social_security_income"), + ): + mask = channels.eq(channel).to_numpy() + selected = people.loc[mask] + age = reports._numeric(selected[age_column]) + expected = reports._numeric(selected[amount_column]) + eligible = age >= 15 + expected[~eligible] = np.nan + observed = report.iloc[np.flatnonzero(mask)] + _require( + np.array_equal(observed.source_reporting_universe.to_numpy(), eligible) + and np.array_equal( + reports._numeric(observed.social_security_source_total), + expected, + equal_nan=True, + ), + "SOURCE_REPORT_IDENTITY", + ) + + +@dataclass(frozen=True) +class Puf55SurveySSMeasurement: + """Detached values and live upstream reference, not a reusable certificate.""" + + preparation: source.AuthenticatedSurveyPopulationPreparation + tax_unit: pd.DataFrame + evidence: bytes + + +def qualify_puf55_survey_ss_measurement(preparation): + """Project native stacked returns; clone inheritance remains the host's duty. + + Roles are the modeled roles retained by the exact preparation, not a claim + that this helper reran current-money tax-unit construction. Receipt bytes, + caller-made tables, and this result cannot replace the live upstream owner. + """ + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + projection = reports.qualify_current_social_security(preparation) + _source_identity(state.frame.person, projection) + report_evidence = source._encode(projection.evidence) + _require( + projection.evidence["preparation_sha256"] == _sha(entry[1]), + "REPORT_PREPARATION", + ) + values, counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + stamp = _projection_digest(values) + evidence = source._encode( + { + "protocol": PROTOCOL, + "measurement": MEASUREMENT, + "preparation_sha256": _sha(entry[1]), + "source_report_evidence_sha256": _sha(report_evidence), + "projection_sha256": stamp, + "counts": counts, + "columns": list(COLUMNS), + "membership": "HEAD plus SPOUSE only for literal JOINT; no dependents", + "role_boundary": "exact modeled return roles retained by live preparation", + "current_money_tax_units_reconstructed_here": False, + "knownness": "all included source reports available; not benefit ownership", + "fallback": full.PUF55_SURVEY_SS_NO_TOTAL.value, + "family_and_child_reports": "retained without allocation or deduplication", + "source_periods": { + "asec": "2024 calendar income; 2025 interview", + "acs": "rolling prior 12 months at 2024 interview; ADJINC price basis", + }, + "repayment_and_railroad_alignment": "unresolved; no adjustment applied", + "donor_E02400_editing": "unresolved; separate donor transport convention", + "person_social_security_changed": False, + "individual_beneficiary_assignment_claim": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=64 * 1024, + ) + result = Puf55SurveySSMeasurement(preparation, values, evidence) + # Finish live owner I/O, then only compare retained owners and computed + # values. The returned projection is never itself treated as authority. + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1], + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _source_identity(state.frame.person, projection) + fresh, fresh_counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + _require( + source._encode(projection.evidence) == report_evidence + and fresh_counts == counts + and _projection_digest(fresh) == stamp + and result.preparation is preparation + and result.evidence == evidence + and _projection_digest(result.tax_unit) == stamp, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py b/packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py new file mode 100644 index 000000000..aaf7583c0 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py @@ -0,0 +1,399 @@ +"""Invented report-sum values and real retained-source qualification controls.""" + +import json +import sys + +import numpy as np +import pandas as pd +import pytest +from test_us_full_puf_output_profiles import _puf59_columns +from test_us_survey_social_security import _social_security_arguments + +from microcosm.build.us_runtime import full_puf_enrichment as full +from microcosm.build.us_runtime import puf55_survey_ss_measurement as measurement + + +def _values(): + person = pd.DataFrame( + { + "person_id": np.arange(1, 11, dtype=np.int64), + "person_tax_unit_id": [10, 10, 20, 20, 20, 30, 30, 40, 50, 50], + "tax_unit_role_input": pd.array( + [ + "HEAD", + "DEPENDENT", + "HEAD", + "SPOUSE", + "DEPENDENT", + "HEAD", + "DEPENDENT", + "HEAD", + "HEAD", + "DEPENDENT", + ], + dtype="string", + ), + } + ) + units = pd.DataFrame( + { + "tax_unit_id": [10, 20, 30, 40, 50], + "filing_status_input": pd.array( + [ + "SINGLE", + "JOINT", + "SURVIVING_SPOUSE", + "SEPARATE", + "HEAD_OF_HOUSEHOLD", + ], + dtype="string", + ), + } + ) + reports = pd.DataFrame( + { + "social_security_source_total": [ + 100, + np.nan, + 200, + 300, + 900, + 400, + np.nan, + 0, + np.nan, + 800, + ], + "source_reporting_universe": [ + True, + False, + True, + True, + True, + True, + False, + True, + False, + True, + ], + }, + index=pd.Index(person.person_id.to_numpy(), name="person_id"), + ) + # These cells are deliberately unrelated to report availability. The proxy + # neither consumes them nor turns excluded dependents into known zeros. + for i, column in enumerate(full.SURVEY_SS_COMPONENTS): + reports[column] = np.resize([-0.0, np.nan, 10.0 + i], len(reports)) + reports["allowed_" + column] = np.resize([True, False], len(reports)) + return person, units, reports + + +def test_exact_head_and_actual_joint_spouse_sum_preserves_dependents_and_components(): + person, units, reports = _values() + before = [table.copy(deep=True) for table in (person, units, reports)] + got, counts = measurement._measure(person, units, reports) + np.testing.assert_array_equal(got[measurement.TOTAL], [100, 500, 400, 0, np.nan]) + assert got[measurement.KNOWN].tolist() == [True, True, True, True, False] + assert got[measurement.ROUTE].tolist() == [full.PUF55_SURVEY_SS.value] * 4 + [ + full.PUF55_SURVEY_SS_NO_TOTAL.value + ] + assert counts == { + "returns": 5, + "included_reporters": 6, + "excluded_dependents": 4, + "unavailable_included_reports": 1, + "nine_predictor_returns": 4, + "eight_predictor_returns": 1, + } + for original, current in zip(before, (person, units, reports), strict=True): + pd.testing.assert_frame_equal(original, current, check_exact=True) + assert reports.loc[2, list(full.SURVEY_SS_COMPONENTS)].isna().all() + assert np.signbit(reports.loc[1, full.SURVEY_SS_COMPONENTS[0]]) + + +def test_alignment_uses_person_membership_and_return_ids_not_row_order(): + person, units, reports = _values() + expected, counts = measurement._measure(person, units, reports) + got, got_counts = measurement._measure( + person.iloc[::-1], + units.iloc[::-1], + reports.iloc[[3, 0, 8, 2, 1, 5, 7, 6, 9, 4]], + ) + pd.testing.assert_frame_equal(got.loc[expected.index], expected, check_exact=True) + assert got_counts == counts + + +def test_missing_joint_spouse_report_routes_whole_return_without_partial_sum(): + person, units, reports = _values() + reports.loc[4, "social_security_source_total"] = np.nan + got, counts = measurement._measure(person, units, reports) + assert np.isnan(got.loc[20, measurement.TOTAL]) + assert not got.loc[20, measurement.KNOWN] + assert got.loc[20, measurement.ROUTE] == full.PUF55_SURVEY_SS_NO_TOTAL.value + assert counts["eight_predictor_returns"] == 2 + assert reports.loc[3, "social_security_source_total"] == 200 + + +@pytest.mark.parametrize( + "case", + [ + "no_head", + "two_heads", + "missing_joint_spouse", + "widow_spouse", + "single_spouse", + "unknown_role", + "missing_role", + "unknown_status", + ], +) +def test_invalid_roles_refuse_even_when_included_report_is_unavailable(case): + person, units, reports = _values() + reports.loc[3, "social_security_source_total"] = np.nan + if case == "no_head": + person.loc[0, "tax_unit_role_input"] = "DEPENDENT" + elif case == "two_heads": + person.loc[1, "tax_unit_role_input"] = "HEAD" + elif case == "missing_joint_spouse": + person.loc[3, "tax_unit_role_input"] = "DEPENDENT" + elif case == "widow_spouse": + person.loc[6, "tax_unit_role_input"] = "SPOUSE" + elif case == "single_spouse": + person.loc[1, "tax_unit_role_input"] = "SPOUSE" + elif case == "unknown_role": + person.loc[3, "tax_unit_role_input"] = "CO_FILER" + elif case == "missing_role": + person.loc[3, "tax_unit_role_input"] = pd.NA + else: + units.loc[1, "filing_status_input"] = "2" + with pytest.raises(ValueError, match="PUF55_SURVEY_SS_ROLE_"): + measurement._measure(person, units, reports) + + +@pytest.mark.parametrize("value", ["100", True, 100 + 0j]) +def test_report_physical_values_are_not_coerced(value): + person, units, reports = _values() + reports["social_security_source_total"] = ( + reports.social_security_source_total.astype(object) + ) + reports.loc[1, "social_security_source_total"] = value + with pytest.raises(ValueError, match="REPORT_PHYSICAL_TYPE"): + measurement._measure(person, units, reports) + + +@pytest.mark.parametrize("value", [-1.0, np.inf, -np.inf]) +def test_invalid_report_refuses_including_excluded_dependent(value): + person, units, reports = _values() + reports.loc[5, "social_security_source_total"] = value + with pytest.raises(ValueError, match="REPORT_DOMAIN"): + measurement._measure(person, units, reports) + + +def test_outside_universe_zero_cannot_be_a_known_report(): + person, units, reports = _values() + reports.loc[9, "social_security_source_total"] = 0 + with pytest.raises(ValueError, match="OUTSIDE_UNIVERSE_OBSERVATION"): + measurement._measure(person, units, reports) + + +def test_numeric_universe_is_not_a_boolean_knownness_mask(): + person, units, reports = _values() + reports["source_reporting_universe"] = reports.source_reporting_universe.astype(int) + with pytest.raises(ValueError, match="UNIVERSE_PHYSICAL_TYPE"): + measurement._measure(person, units, reports) + + +def test_integer_report_cannot_silently_lose_precision_during_conversion(): + person, units, reports = _values() + reports["social_security_source_total"] = pd.Series( + 0, index=reports.index, dtype="int64" + ) + reports.loc[1, "social_security_source_total"] = 2**53 + 1 + with pytest.raises(ValueError, match="REPORT_PRECISION"): + measurement._measure(person, units, reports) + + +def test_complete_joint_reports_refuse_sum_overflow(): + person, units, reports = _values() + reports.loc[[3, 4], "social_security_source_total"] = np.finfo(np.float64).max + with np.errstate(over="ignore"), pytest.raises(ValueError, match="SUM_OVERFLOW"): + measurement._measure(person, units, reports) + + +@pytest.mark.parametrize( + "case", ["duplicate_person", "missing_report", "orphan_member"] +) +def test_incomplete_or_ambiguous_membership_is_not_a_fallback_route(case): + person, units, reports = _values() + if case == "duplicate_person": + person.loc[1, "person_id"] = 1 + elif case == "missing_report": + reports = reports.iloc[1:] + else: + person.loc[0, "person_tax_unit_id"] = 99 + with pytest.raises(ValueError, match="MEMBERSHIP"): + measurement._measure(person, units, reports) + + +def test_explicit_eight_predictor_profile_keeps_all_55_outputs_and_distinct_graph_phase(): + profile = full.PUF55_SURVEY_SS_NO_TOTAL + assert len(profile.targets) == 55 + assert len(profile.person_outputs) == 52 + assert profile.targets == full.PUF55_SURVEY_SS.targets + assert profile.predictors == full.PUF59.predictors and len(profile.predictors) == 8 + assert profile.source_predictors == full.PUF59.source_predictors + assert not set(full.SURVEY_SS_COMPONENTS) & set(profile.targets) + assert len(full.PUF55_SURVEY_SS.predictors) == 9 + assert len(full.PUF59.targets) == 59 and len(full.FULL65.targets) == 65 + assert len({item.phase for item in full.PufOutputProfile}) == 4 + assert full.FULL65.phase == full.PHASE + assert full.PUF55_SURVEY_SS.phase == full.PHASE + ".puf55_survey_ss" + fits, applies = full.full_puf_train_apply_nodes( + donor_population="donor", + recipient_population="recipient", + matrix_producer="matrix", + seed=123, + n_estimators=2, + zero_atol=0.0, + prefix="without_report", + profile=profile, + ) + assert len(fits) == len(applies) == 55 + assert tuple(node.params["target"] for node in fits) == profile.targets + assert all(tuple(node.params["predictors"]) == profile.predictors for node in fits) + assert all(node.params["phase"] == profile.phase for node in (*fits, *applies)) + + +def test_eight_predictor_canonical_donor_does_not_read_missing_total_or_ss_components(): + person, units, pk, tk = _puf59_columns() + profile = full.PUF55_SURVEY_SS_NO_TOTAL + person = person.drop(columns=list(full.SURVEY_SS_COMPONENTS)) + pk = pk.drop(columns=list(full.SURVEY_SS_COMPONENTS)) + donor = full.canonical_full_puf_donor( + person, units, person_known=pk, tax_unit_known=tk, profile=profile + ) + assert tuple(donor) == ( + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ) + assert measurement.TOTAL not in donor + + +def test_caller_role_tables_and_descriptive_receipts_cannot_qualify_a_source(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + measurement.qualify_puf55_survey_ss_measurement( + { + "roles": _values()[0], + "source_authenticated": True, + } + ) + + +@pytest.fixture(scope="module") +def retained_source(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + arguments = _social_security_arguments( + tmp_path_factory.mktemp("ss-measurement"), patch, ambiguous=True + ) + yield measurement.source.prepare_authenticated_survey_population(**arguments) + + +def test_actual_retained_source_boundary_and_descriptive_evidence(retained_source): + state = retained_source._checked()[2] + before = { + name: state.frame.table(name).copy(deep=True) for name in state.frame.entities + } + got = measurement.qualify_puf55_survey_ss_measurement(retained_source) + assert got.preparation is retained_source + assert ( + got.tax_unit.index.tolist() + == state.frame.table("tax_unit").tax_unit_id.tolist() + ) + evidence = json.loads(got.evidence) + assert evidence["measurement"] == measurement.MEASUREMENT + assert evidence["preparation_sha256"] == measurement._sha(retained_source.payload) + assert evidence["projection_sha256"] == measurement._projection_digest(got.tax_unit) + assert evidence["counts"]["returns"] == len(got.tax_unit) + assert evidence["counts"]["nine_predictor_returns"] + evidence["counts"][ + "eight_predictor_returns" + ] == len(got.tax_unit) + assert not evidence["source_admission_issued"] + assert not evidence["population_admission_issued"] + assert not evidence["individual_beneficiary_assignment_claim"] + assert not evidence["current_money_tax_units_reconstructed_here"] + for name, original in before.items(): + pd.testing.assert_frame_equal( + state.frame.table(name), original, check_exact=True + ) + + +def test_detached_source_result_mutation_is_not_source_evidence(retained_source): + changed = [] + + def callback(frame, event, value): + if ( + event == "return" + and frame.f_code + is measurement.reports.qualify_current_social_security.__code__ + and value is not None + ): + value.person.iloc[ + 0, value.person.columns.get_loc("social_security_source_total") + ] = 999999.0 + changed.append(True) + + prior = sys.getprofile() + try: + sys.setprofile(callback) + with pytest.raises(ValueError, match="SOURCE_REPORT_IDENTITY"): + measurement.qualify_puf55_survey_ss_measurement(retained_source) + finally: + sys.setprofile(prior) + assert changed == [True] + + +def test_final_owner_return_callback_cannot_change_detached_output(retained_source): + changed = [] + + def callback(frame, event, value): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is measurement.source.AuthenticatedSurveyPopulationPreparation._checked.__code__ + and caller is not None + and caller.f_code + is measurement.qualify_puf55_survey_ss_measurement.__code__ + and "result" in caller.f_locals + ): + result = caller.f_locals["result"] + result.tax_unit.iloc[ + 0, result.tax_unit.columns.get_loc(measurement.TOTAL) + ] = 999999.0 + changed.append(True) + + prior = sys.getprofile() + try: + sys.setprofile(callback) + with pytest.raises(ValueError, match="FINAL_PROJECTION"): + measurement.qualify_puf55_survey_ss_measurement(retained_source) + finally: + sys.setprofile(prior) + assert changed == [True] + + +def test_changed_retained_role_cannot_be_routed_as_missing_report(retained_source): + state = retained_source._checked()[2] + people = state.frame.person + original = people.tax_unit_role_input.copy(deep=True) + try: + people.loc[people.index[0], "tax_unit_role_input"] = "DEPENDENT" + # Guarantee a changed role even if the first fixture person was already + # a dependent. The pure preparation seal rejects before numerical routing. + if people.tax_unit_role_input.equals(original): + people.loc[people.index[0], "tax_unit_role_input"] = "HEAD" + with pytest.raises(ValueError, match="PREPARED_FRAME_CHANGED"): + measurement.qualify_puf55_survey_ss_measurement(retained_source) + finally: + people["tax_unit_role_input"] = original From 38eefa2dc4c7058585b6d0cf8f711a4ddaa30650 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 20:03:02 -0400 Subject: [PATCH 039/255] Verify national population-only Census block support --- .../us-national-atomic-support.added.md | 4 + docs/us-launch-review.md | 15 +- ...-atomic-national-acquisition-20260909.json | 598 +++ ...ic-native-national-1-control-20260909.json | 3434 +++++++++++++++++ ...tomic_block_api_sources_native_national.py | 502 +++ 5 files changed, 4552 insertions(+), 1 deletion(-) create mode 100644 changelog.d/us-national-atomic-support.added.md create mode 100644 experiments/us-atomic-national-acquisition-20260909.json create mode 100644 experiments/us-atomic-native-national-1-control-20260909.json create mode 100644 packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py diff --git a/changelog.d/us-national-atomic-support.added.md b/changelog.d/us-national-atomic-support.added.md new file mode 100644 index 000000000..085c1627a --- /dev/null +++ b/changelog.d/us-national-atomic-support.added.md @@ -0,0 +1,4 @@ +Add an opt-in national Census population-only source control covering the fifty +states and DC, with independent block/state reconciliation, complete CD119 and +PUMA joins, and full support serialization/readback. Record the accepted native +normalization separately from household assignment and dataset release checks. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 0714b6e39..794cfc5f3 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -205,7 +205,20 @@ corrected check still decompresses only `NationalCD119.txt`. The [accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) preserves the original failed evidence. This is not national population or release acceptance. Delaware-only support must not enter the national survey -graph. Complete support for the fifty states and DC is required first. +graph. + +The subsequent population-only national acquisition and normalization now pass +for all fifty states and DC. The 104 exact sources produce 5,769,942 populated +blocks, independently reconciling every state to a total of 331,449,281 people. +Every block retains its identity and population, with complete CD119/PUMA joins. +The full support artifact is 28,862,508 bytes and passes exact serialization and +readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is +`5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; +see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). +Postchecks reauthenticated code and compared the 104 native pins from bounded +receipts without reopening source bodies. This clears the national support +normalization check; assigning households, evaluating geographic fit and +calibrating the enriched survey remain separate acceptance steps. ## Related reviews diff --git a/experiments/us-atomic-national-acquisition-20260909.json b/experiments/us-atomic-national-acquisition-20260909.json new file mode 100644 index 000000000..42485f90e --- /dev/null +++ b/experiments/us-atomic-national-acquisition-20260909.json @@ -0,0 +1,598 @@ +{ + "acquired_at_utc": "2026-09-09T23:45:56.994286+00:00", + "acquisition_receipt_sha256": "6e36758e59aa8536d35d25881caed122111da59e200a634c5a4b8d7baa6485d5", + "automatic_retry": false, + "cpu_seconds": 16.783967, + "de_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "guard_sha256": "1c9bf5810b787f19043cb744f1155d823b0f80703038c2a0f861fdcdf8276a18", + "native_inputs_reopened_for_postcheck": false, + "native_normalization": false, + "peak_rss_acceptance_bytes": 4294967296, + "peak_rss_bytes": 390889472, + "project_imports": false, + "release_eligible": false, + "result": 0, + "rss_hard_os_enforcement": false, + "scope": "104 once-only verified-HTTPS requests: P1_001N block populations and independent state totals for50states+DC, CD119 mapping archive and2020tract-to-PUMA mapping. Acquisition only; no normalization, survey construction or calibration.", + "script_and_roster_before_after_equal": true, + "script_sha256": "eccecc8f8927dd52b2f704cb1cc22c3c3fb10a59f7c1ae01f301edee240c8894", + "source_admission_issued": false, + "source_count": 104, + "sources": [ + { + "id": "pl2020_p1_state01_blocks", + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + { + "id": "pl2020_p1_state01_total", + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state02_blocks", + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + { + "id": "pl2020_p1_state02_total", + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state04_blocks", + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + { + "id": "pl2020_p1_state04_total", + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state05_blocks", + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + { + "id": "pl2020_p1_state05_total", + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state06_blocks", + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + { + "id": "pl2020_p1_state06_total", + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state08_blocks", + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + { + "id": "pl2020_p1_state08_total", + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state09_blocks", + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + { + "id": "pl2020_p1_state09_total", + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state10_blocks", + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + { + "id": "pl2020_p1_state10_total", + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state11_blocks", + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + { + "id": "pl2020_p1_state11_total", + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state12_blocks", + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + { + "id": "pl2020_p1_state12_total", + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state13_blocks", + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + { + "id": "pl2020_p1_state13_total", + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state15_blocks", + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + { + "id": "pl2020_p1_state15_total", + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state16_blocks", + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + { + "id": "pl2020_p1_state16_total", + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state17_blocks", + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + { + "id": "pl2020_p1_state17_total", + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state18_blocks", + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + { + "id": "pl2020_p1_state18_total", + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state19_blocks", + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + { + "id": "pl2020_p1_state19_total", + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state20_blocks", + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + { + "id": "pl2020_p1_state20_total", + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state21_blocks", + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + { + "id": "pl2020_p1_state21_total", + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state22_blocks", + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + { + "id": "pl2020_p1_state22_total", + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state23_blocks", + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + { + "id": "pl2020_p1_state23_total", + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state24_blocks", + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + { + "id": "pl2020_p1_state24_total", + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state25_blocks", + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + { + "id": "pl2020_p1_state25_total", + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state26_blocks", + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + { + "id": "pl2020_p1_state26_total", + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state27_blocks", + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + { + "id": "pl2020_p1_state27_total", + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state28_blocks", + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + { + "id": "pl2020_p1_state28_total", + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state29_blocks", + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + { + "id": "pl2020_p1_state29_total", + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state30_blocks", + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + { + "id": "pl2020_p1_state30_total", + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state31_blocks", + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + { + "id": "pl2020_p1_state31_total", + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state32_blocks", + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + { + "id": "pl2020_p1_state32_total", + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state33_blocks", + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + { + "id": "pl2020_p1_state33_total", + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state34_blocks", + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + { + "id": "pl2020_p1_state34_total", + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state35_blocks", + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + { + "id": "pl2020_p1_state35_total", + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state36_blocks", + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + { + "id": "pl2020_p1_state36_total", + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state37_blocks", + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + { + "id": "pl2020_p1_state37_total", + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state38_blocks", + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + { + "id": "pl2020_p1_state38_total", + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state39_blocks", + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + { + "id": "pl2020_p1_state39_total", + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state40_blocks", + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + { + "id": "pl2020_p1_state40_total", + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state41_blocks", + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + { + "id": "pl2020_p1_state41_total", + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state42_blocks", + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + { + "id": "pl2020_p1_state42_total", + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state44_blocks", + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + { + "id": "pl2020_p1_state44_total", + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state45_blocks", + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + { + "id": "pl2020_p1_state45_total", + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state46_blocks", + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + { + "id": "pl2020_p1_state46_total", + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state47_blocks", + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + { + "id": "pl2020_p1_state47_total", + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state48_blocks", + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + { + "id": "pl2020_p1_state48_total", + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state49_blocks", + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + { + "id": "pl2020_p1_state49_total", + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state50_blocks", + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + { + "id": "pl2020_p1_state50_total", + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state51_blocks", + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + { + "id": "pl2020_p1_state51_total", + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state53_blocks", + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + { + "id": "pl2020_p1_state53_total", + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state54_blocks", + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + { + "id": "pl2020_p1_state54_total", + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state55_blocks", + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + { + "id": "pl2020_p1_state55_total", + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state56_blocks", + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + { + "id": "pl2020_p1_state56_total", + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + { + "id": "census_cd119_bef", + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130 + }, + { + "id": "census_tract2020_puma2020", + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076 + } + ], + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "transferred_bytes": 306552379, + "wall_seconds": 328.79918500001077 +} diff --git a/experiments/us-atomic-native-national-1-control-20260909.json b/experiments/us-atomic-native-national-1-control-20260909.json new file mode 100644 index 000000000..822ea0e7c --- /dev/null +++ b/experiments/us-atomic-native-national-1-control-20260909.json @@ -0,0 +1,3434 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 81.174622, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 104, + "normalization": { + "exact_support_readback": true, + "mapping_oracle": "separate maintained-parser pass; complete selected joins", + "national_coverage": true, + "population_oracle": { + "01": { + "block_rows": 185976, + "ordered_positive_block_population_sha256": "db629f189d2658bec941146c499464bec1d7278d83da0b07ac8ed338d5d993ab", + "populated_blocks": 128366, + "state_population": 5024279, + "zero_population_blocks": 57610 + }, + "02": { + "block_rows": 28568, + "ordered_positive_block_population_sha256": "c82ab4e84dfee99b16fc0397e3bf57d0ef5c87e1e37e5fc766879c690d0f4caf", + "populated_blocks": 11765, + "state_population": 733391, + "zero_population_blocks": 16803 + }, + "04": { + "block_rows": 155444, + "ordered_positive_block_population_sha256": "57059741545abff19005434c387543790c8cb047bd8453c17aada74bda370c46", + "populated_blocks": 107938, + "state_population": 7151502, + "zero_population_blocks": 47506 + }, + "05": { + "block_rows": 136422, + "ordered_positive_block_population_sha256": "af7158b0ab1d4e4baf46766a0953bfe74f52036272af13c4cfe6b65ef572c5ed", + "populated_blocks": 88121, + "state_population": 3011524, + "zero_population_blocks": 48301 + }, + "06": { + "block_rows": 519723, + "ordered_positive_block_population_sha256": "fc244613d2faab0aa55e0064fff67ee2df2cda32d8e1cfef0f934a6e80e41a68", + "populated_blocks": 377591, + "state_population": 39538223, + "zero_population_blocks": 142132 + }, + "08": { + "block_rows": 140345, + "ordered_positive_block_population_sha256": "d69d69546b402b60bd4eb98fbb4003417ddc3d26c442e6051437628dff7708a1", + "populated_blocks": 99899, + "state_population": 5773714, + "zero_population_blocks": 40446 + }, + "09": { + "block_rows": 49926, + "ordered_positive_block_population_sha256": "5be4f0b5fba158152132db2671dbb607f0c8233c1dfc532d5beb99d51965ee0d", + "populated_blocks": 42008, + "state_population": 3605944, + "zero_population_blocks": 7918 + }, + "10": { + "block_rows": 20198, + "ordered_positive_block_population_sha256": "f32d84817caa2c3f3ea776da426dbee30f049c82b9335af41699ec4bee00e235", + "populated_blocks": 15317, + "state_population": 989948, + "zero_population_blocks": 4881 + }, + "11": { + "block_rows": 6012, + "ordered_positive_block_population_sha256": "c0d37648dff6f3ba16d524e2e375417669565acab25027d64ec03b05c89c603d", + "populated_blocks": 4500, + "state_population": 689545, + "zero_population_blocks": 1512 + }, + "12": { + "block_rows": 390066, + "ordered_positive_block_population_sha256": "683fb2c4854b4c7ede8815bfc5620c3cf26a5d9149eab4bbd25fd4eedb5eb00e", + "populated_blocks": 289652, + "state_population": 21538187, + "zero_population_blocks": 100414 + }, + "13": { + "block_rows": 232717, + "ordered_positive_block_population_sha256": "dbc9e610217cd9e5de12fb3825addb9508ae368c1dc4a88b98801960ff91cdb0", + "populated_blocks": 165333, + "state_population": 10711908, + "zero_population_blocks": 67384 + }, + "15": { + "block_rows": 14732, + "ordered_positive_block_population_sha256": "245ccc4831fdcb75b1d2d5202f0d38c2ac9c1fab91e4916a61191729d4fbc88f", + "populated_blocks": 10169, + "state_population": 1455271, + "zero_population_blocks": 4563 + }, + "16": { + "block_rows": 81879, + "ordered_positive_block_population_sha256": "47cc73fe9b89c0db1c66621484f12b2a426f7332c289c28a0c30915ab7135dc8", + "populated_blocks": 46134, + "state_population": 1839106, + "zero_population_blocks": 35745 + }, + "17": { + "block_rows": 369978, + "ordered_positive_block_population_sha256": "bf7e305eae384c4338298d4b2dad71cfa48a5a0dc2858dda558b92bc26cbceaa", + "populated_blocks": 278166, + "state_population": 12812508, + "zero_population_blocks": 91812 + }, + "18": { + "block_rows": 204568, + "ordered_positive_block_population_sha256": "a85fae03b48ad4703a5ed7f39bb81eca4974df835d7c9e99590486fbaf11e163", + "populated_blocks": 162739, + "state_population": 6785528, + "zero_population_blocks": 41829 + }, + "19": { + "block_rows": 175199, + "ordered_positive_block_population_sha256": "f89589b4fd957d0643af40784431bd793d8ebfa05ceed97b3921689f424ee938", + "populated_blocks": 124175, + "state_population": 3190369, + "zero_population_blocks": 51024 + }, + "20": { + "block_rows": 172529, + "ordered_positive_block_population_sha256": "75877b15ff41b689be4774c7ddd9294701b97b1762fd4a935d10fce1cfb9c2e4", + "populated_blocks": 109149, + "state_population": 2937880, + "zero_population_blocks": 63380 + }, + "21": { + "block_rows": 132662, + "ordered_positive_block_population_sha256": "c7b86e0bef4455178c4da1b99b16e615454602062db58de9b9e8bf5a389c9954", + "populated_blocks": 91572, + "state_population": 4505836, + "zero_population_blocks": 41090 + }, + "22": { + "block_rows": 142874, + "ordered_positive_block_population_sha256": "fa3122a5aee3bfb4f78dba6ad64d582c1515ee045298e8f2359870ec8d884457", + "populated_blocks": 92180, + "state_population": 4657757, + "zero_population_blocks": 50694 + }, + "23": { + "block_rows": 47138, + "ordered_positive_block_population_sha256": "c870139aa6e05735c7f1ff1e05399995dc54a722dc2df99198059e7f89324f85", + "populated_blocks": 30548, + "state_population": 1362359, + "zero_population_blocks": 16590 + }, + "24": { + "block_rows": 83827, + "ordered_positive_block_population_sha256": "519b855f8fa876aed559a5c4b57458301e9aa1323bd7ae5575b916055abe00f4", + "populated_blocks": 65274, + "state_population": 6177224, + "zero_population_blocks": 18553 + }, + "25": { + "block_rows": 107278, + "ordered_positive_block_population_sha256": "aa920cf2bee71bf21a2ef68d76b04eb417c2c72fbcf468523fd032956917ac41", + "populated_blocks": 88207, + "state_population": 7029917, + "zero_population_blocks": 19071 + }, + "26": { + "block_rows": 254730, + "ordered_positive_block_population_sha256": "8e57ac1ca0ab6cafc9385f23b658a0eafd4f4e01f55468d42da327355d95a445", + "populated_blocks": 198244, + "state_population": 10077331, + "zero_population_blocks": 56486 + }, + "27": { + "block_rows": 198705, + "ordered_positive_block_population_sha256": "e616047478baea1a265c120762ebc1ad234531a1c1459e8a97263dfa5645b225", + "populated_blocks": 137689, + "state_population": 5706494, + "zero_population_blocks": 61016 + }, + "28": { + "block_rows": 112241, + "ordered_positive_block_population_sha256": "ab34034369ef3795f7a2a8112de09a48789c5921102f64681e8f772fe5f24fe8", + "populated_blocks": 77426, + "state_population": 2961279, + "zero_population_blocks": 34815 + }, + "29": { + "block_rows": 253632, + "ordered_positive_block_population_sha256": "ec441928aa5f5ef297b96c37db8f81132a0ac50399eada71edff432e0d922b03", + "populated_blocks": 169004, + "state_population": 6154913, + "zero_population_blocks": 84628 + }, + "30": { + "block_rows": 88417, + "ordered_positive_block_population_sha256": "b09a76edbea2619f38d2ad70a31108726f24abd23b17099a0ff010f07b48b43d", + "populated_blocks": 43717, + "state_population": 1084225, + "zero_population_blocks": 44700 + }, + "31": { + "block_rows": 119103, + "ordered_positive_block_population_sha256": "ab1afedaab5dd2392593270ce5fd7536d54bfe4cf99b34d53ce7a66aad96830f", + "populated_blocks": 79779, + "state_population": 1961504, + "zero_population_blocks": 39324 + }, + "32": { + "block_rows": 57409, + "ordered_positive_block_population_sha256": "760c7b9e590052d373316c9800545a20a71495b42f0d4b7a20b55b2fbf6f12f9", + "populated_blocks": 35249, + "state_population": 3104614, + "zero_population_blocks": 22160 + }, + "33": { + "block_rows": 31948, + "ordered_positive_block_population_sha256": "d88a28f2f35cf1b8ef64fd5d42600e416aad959de76e06de9ac76dd4e86a21a8", + "populated_blocks": 25317, + "state_population": 1377529, + "zero_population_blocks": 6631 + }, + "34": { + "block_rows": 137972, + "ordered_positive_block_population_sha256": "d924c85d9ae80d624ed2902e925a9c7dabb4a996680a081fbe237fcd9a1746d3", + "populated_blocks": 113212, + "state_population": 9288994, + "zero_population_blocks": 24760 + }, + "35": { + "block_rows": 107215, + "ordered_positive_block_population_sha256": "3905ad0b0d16d2c86502cfa4797024fd6d2f895efe0280379bb7963836ccab82", + "populated_blocks": 56605, + "state_population": 2117522, + "zero_population_blocks": 50610 + }, + "36": { + "block_rows": 288819, + "ordered_positive_block_population_sha256": "65a2be271d6a207e1f3e7cabedd6778d19d909ba0e37f5232bb010ad8a30b2f7", + "populated_blocks": 230339, + "state_population": 20201249, + "zero_population_blocks": 58480 + }, + "37": { + "block_rows": 236638, + "ordered_positive_block_population_sha256": "0c135c4919f7b904a298fbb183620e452e02ba9fca6644e27c2e8cf65e40ed50", + "populated_blocks": 174988, + "state_population": 10439388, + "zero_population_blocks": 61650 + }, + "38": { + "block_rows": 84566, + "ordered_positive_block_population_sha256": "fb4b3241fa5f1d81ed332353f90b0130b2383dd17e840a11badc1c911d155597", + "populated_blocks": 40198, + "state_population": 779094, + "zero_population_blocks": 44368 + }, + "39": { + "block_rows": 276428, + "ordered_positive_block_population_sha256": "a3e40aaf54aaaa347573881c39a06eed40a1e7d02b1e57a4a70fdfdaafb6b7fb", + "populated_blocks": 219669, + "state_population": 11799448, + "zero_population_blocks": 56759 + }, + "40": { + "block_rows": 180154, + "ordered_positive_block_population_sha256": "10240ff44682b047e63e5da0e4e8a5c9f6750d80b9a7f176ad397500f06492c1", + "populated_blocks": 121826, + "state_population": 3959353, + "zero_population_blocks": 58328 + }, + "41": { + "block_rows": 130807, + "ordered_positive_block_population_sha256": "01b6c50dbc1320f96741aa159354a3e42b318ee7e3bda21dac3d5f122e6799b0", + "populated_blocks": 79081, + "state_population": 4237256, + "zero_population_blocks": 51726 + }, + "42": { + "block_rows": 336985, + "ordered_positive_block_population_sha256": "70bb990a03502d1c2359e19a5f469727933661dcabfdcb9082c7c9f5ba34ee07", + "populated_blocks": 275023, + "state_population": 13002700, + "zero_population_blocks": 61962 + }, + "44": { + "block_rows": 25649, + "ordered_positive_block_population_sha256": "2777acda8ed305d3fcad00ec03bcc9d5b16d295a82fc8250caa73a0acf4184cd", + "populated_blocks": 21382, + "state_population": 1097379, + "zero_population_blocks": 4267 + }, + "45": { + "block_rows": 146844, + "ordered_positive_block_population_sha256": "d1a102de62516378b036f30c3642a17d3f2a4040b8a76e380b2f01f8b33d8a4f", + "populated_blocks": 105469, + "state_population": 5118425, + "zero_population_blocks": 41375 + }, + "46": { + "block_rows": 71383, + "ordered_positive_block_population_sha256": "8c372d4e82aff59d8701ecd211e24224885561645bedb7de74db55ca0c60205c", + "populated_blocks": 40766, + "state_population": 886667, + "zero_population_blocks": 30617 + }, + "47": { + "block_rows": 179717, + "ordered_positive_block_population_sha256": "f3804b33c433afd24a7b433c6524b0d124754d289cc2d589798fde5987794357", + "populated_blocks": 133846, + "state_population": 6910840, + "zero_population_blocks": 45871 + }, + "48": { + "block_rows": 668757, + "ordered_positive_block_population_sha256": "83903c280fd7fb16e9cc5a5ce7266753a0445118c8a3151bfadc86e27b891b3d", + "populated_blocks": 449085, + "state_population": 29145505, + "zero_population_blocks": 219672 + }, + "49": { + "block_rows": 71207, + "ordered_positive_block_population_sha256": "d1ea8f0e3bfa6225218b0f3bd0c022453fa3038a196ebef8a17dadb1be0ca575", + "populated_blocks": 46792, + "state_population": 3271616, + "zero_population_blocks": 24415 + }, + "50": { + "block_rows": 24611, + "ordered_positive_block_population_sha256": "3df326b12b20dbf6af14229fae0dc2e5733b4eabbe7a142515bebf0455881a93", + "populated_blocks": 17976, + "state_population": 643077, + "zero_population_blocks": 6635 + }, + "51": { + "block_rows": 163491, + "ordered_positive_block_population_sha256": "756a961291abef27225b26548e1fc67a58725285de9bbf4d93b04a618d7731bb", + "populated_blocks": 117361, + "state_population": 8631393, + "zero_population_blocks": 46130 + }, + "53": { + "block_rows": 158093, + "ordered_positive_block_population_sha256": "11bbff165b4c5004c526e0420dff07601a637a2d9b145288c86ffc476dcd6f5c", + "populated_blocks": 111441, + "state_population": 7705281, + "zero_population_blocks": 46652 + }, + "54": { + "block_rows": 72558, + "ordered_positive_block_population_sha256": "3fac3d7a2c520bcfb0e3ddf04776e2e147d157d58b8b8dd04cebb91ec89a8a6e", + "populated_blocks": 51850, + "state_population": 1793716, + "zero_population_blocks": 20708 + }, + "55": { + "block_rows": 203059, + "ordered_positive_block_population_sha256": "16f7d47ca52eced522acd712209a41215fd99ddb43f5a2232974b5ef6bc510de", + "populated_blocks": 144948, + "state_population": 5893718, + "zero_population_blocks": 58111 + }, + "56": { + "block_rows": 53769, + "ordered_positive_block_population_sha256": "33eac054e9aaf0d266460d4c4df67192a5ac7c993658ece720b398f868f45648", + "populated_blocks": 22857, + "state_population": 576851, + "zero_population_blocks": 30912 + } + }, + "protocol": "microcosm.us.native-national-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_count": 104, + "source_read_limits": { + "mapping_verification_passes": 1, + "selected_cd_member_bytes_per_pass": 163499110, + "selected_geography_bytes_per_adapter_pass": 1237240934, + "unique_acquired_bytes": 1073741824 + }, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 163499110, + "max_response_rows": 2000000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 1237240934, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + }, + "populated_blocks": 5769942, + "population_total": 331449281, + "selected_geography_bytes": 447092359, + "state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 185976, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:01" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + "populated_blocks": 128366, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "01", + "state_population": 5024279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:01" + ] + ] + }, + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + "zero_population_blocks": 57610 + }, + { + "block_rows": 28568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:02" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + "populated_blocks": 11765, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "02", + "state_population": 733391, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:02" + ] + ] + }, + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + "zero_population_blocks": 16803 + }, + { + "block_rows": 155444, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:04" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + "populated_blocks": 107938, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "04", + "state_population": 7151502, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:04" + ] + ] + }, + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + "zero_population_blocks": 47506 + }, + { + "block_rows": 136422, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:05" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + "populated_blocks": 88121, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "05", + "state_population": 3011524, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:05" + ] + ] + }, + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + "zero_population_blocks": 48301 + }, + { + "block_rows": 519723, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:06" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + "populated_blocks": 377591, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "06", + "state_population": 39538223, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:06" + ] + ] + }, + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + "zero_population_blocks": 142132 + }, + { + "block_rows": 140345, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:08" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + "populated_blocks": 99899, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "08", + "state_population": 5773714, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:08" + ] + ] + }, + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + "zero_population_blocks": 40446 + }, + { + "block_rows": 49926, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:09" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + "populated_blocks": 42008, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "09", + "state_population": 3605944, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:09" + ] + ] + }, + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + "zero_population_blocks": 7918 + }, + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + }, + { + "block_rows": 6012, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:11" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + "populated_blocks": 4500, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "11", + "state_population": 689545, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:11" + ] + ] + }, + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + "zero_population_blocks": 1512 + }, + { + "block_rows": 390066, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:12" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + "populated_blocks": 289652, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "12", + "state_population": 21538187, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:12" + ] + ] + }, + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + "zero_population_blocks": 100414 + }, + { + "block_rows": 232717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:13" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + "populated_blocks": 165333, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "13", + "state_population": 10711908, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:13" + ] + ] + }, + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + "zero_population_blocks": 67384 + }, + { + "block_rows": 14732, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:15" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + "populated_blocks": 10169, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "15", + "state_population": 1455271, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:15" + ] + ] + }, + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + "zero_population_blocks": 4563 + }, + { + "block_rows": 81879, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:16" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + "populated_blocks": 46134, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "16", + "state_population": 1839106, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:16" + ] + ] + }, + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + "zero_population_blocks": 35745 + }, + { + "block_rows": 369978, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:17" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + "populated_blocks": 278166, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "17", + "state_population": 12812508, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:17" + ] + ] + }, + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + "zero_population_blocks": 91812 + }, + { + "block_rows": 204568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:18" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + "populated_blocks": 162739, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "18", + "state_population": 6785528, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:18" + ] + ] + }, + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + "zero_population_blocks": 41829 + }, + { + "block_rows": 175199, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:19" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + "populated_blocks": 124175, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "19", + "state_population": 3190369, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:19" + ] + ] + }, + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + "zero_population_blocks": 51024 + }, + { + "block_rows": 172529, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:20" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + "populated_blocks": 109149, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "20", + "state_population": 2937880, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:20" + ] + ] + }, + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + "zero_population_blocks": 63380 + }, + { + "block_rows": 132662, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:21" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + "populated_blocks": 91572, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "21", + "state_population": 4505836, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:21" + ] + ] + }, + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + "zero_population_blocks": 41090 + }, + { + "block_rows": 142874, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:22" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + "populated_blocks": 92180, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "22", + "state_population": 4657757, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:22" + ] + ] + }, + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + "zero_population_blocks": 50694 + }, + { + "block_rows": 47138, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:23" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + "populated_blocks": 30548, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "23", + "state_population": 1362359, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:23" + ] + ] + }, + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + "zero_population_blocks": 16590 + }, + { + "block_rows": 83827, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:24" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + "populated_blocks": 65274, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "24", + "state_population": 6177224, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:24" + ] + ] + }, + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + "zero_population_blocks": 18553 + }, + { + "block_rows": 107278, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:25" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + "populated_blocks": 88207, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "25", + "state_population": 7029917, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:25" + ] + ] + }, + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + "zero_population_blocks": 19071 + }, + { + "block_rows": 254730, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:26" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + "populated_blocks": 198244, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "26", + "state_population": 10077331, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:26" + ] + ] + }, + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + "zero_population_blocks": 56486 + }, + { + "block_rows": 198705, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:27" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + "populated_blocks": 137689, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "27", + "state_population": 5706494, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:27" + ] + ] + }, + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + "zero_population_blocks": 61016 + }, + { + "block_rows": 112241, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:28" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + "populated_blocks": 77426, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "28", + "state_population": 2961279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:28" + ] + ] + }, + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + "zero_population_blocks": 34815 + }, + { + "block_rows": 253632, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:29" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + "populated_blocks": 169004, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "29", + "state_population": 6154913, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:29" + ] + ] + }, + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + "zero_population_blocks": 84628 + }, + { + "block_rows": 88417, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:30" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + "populated_blocks": 43717, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "30", + "state_population": 1084225, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:30" + ] + ] + }, + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + "zero_population_blocks": 44700 + }, + { + "block_rows": 119103, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:31" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + "populated_blocks": 79779, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "31", + "state_population": 1961504, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:31" + ] + ] + }, + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + "zero_population_blocks": 39324 + }, + { + "block_rows": 57409, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:32" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + "populated_blocks": 35249, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "32", + "state_population": 3104614, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:32" + ] + ] + }, + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + "zero_population_blocks": 22160 + }, + { + "block_rows": 31948, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:33" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + "populated_blocks": 25317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "33", + "state_population": 1377529, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:33" + ] + ] + }, + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + "zero_population_blocks": 6631 + }, + { + "block_rows": 137972, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:34" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + "populated_blocks": 113212, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "34", + "state_population": 9288994, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:34" + ] + ] + }, + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + "zero_population_blocks": 24760 + }, + { + "block_rows": 107215, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:35" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + "populated_blocks": 56605, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "35", + "state_population": 2117522, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:35" + ] + ] + }, + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + "zero_population_blocks": 50610 + }, + { + "block_rows": 288819, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:36" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + "populated_blocks": 230339, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "36", + "state_population": 20201249, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:36" + ] + ] + }, + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + "zero_population_blocks": 58480 + }, + { + "block_rows": 236638, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:37" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + "populated_blocks": 174988, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "37", + "state_population": 10439388, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:37" + ] + ] + }, + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + "zero_population_blocks": 61650 + }, + { + "block_rows": 84566, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:38" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + "populated_blocks": 40198, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "38", + "state_population": 779094, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:38" + ] + ] + }, + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + "zero_population_blocks": 44368 + }, + { + "block_rows": 276428, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:39" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + "populated_blocks": 219669, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "39", + "state_population": 11799448, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:39" + ] + ] + }, + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + "zero_population_blocks": 56759 + }, + { + "block_rows": 180154, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:40" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + "populated_blocks": 121826, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "40", + "state_population": 3959353, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:40" + ] + ] + }, + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + "zero_population_blocks": 58328 + }, + { + "block_rows": 130807, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:41" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + "populated_blocks": 79081, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "41", + "state_population": 4237256, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:41" + ] + ] + }, + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + "zero_population_blocks": 51726 + }, + { + "block_rows": 336985, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:42" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + "populated_blocks": 275023, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "42", + "state_population": 13002700, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:42" + ] + ] + }, + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + "zero_population_blocks": 61962 + }, + { + "block_rows": 25649, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:44" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + "populated_blocks": 21382, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "44", + "state_population": 1097379, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:44" + ] + ] + }, + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + "zero_population_blocks": 4267 + }, + { + "block_rows": 146844, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:45" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + "populated_blocks": 105469, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "45", + "state_population": 5118425, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:45" + ] + ] + }, + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + "zero_population_blocks": 41375 + }, + { + "block_rows": 71383, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:46" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + "populated_blocks": 40766, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "46", + "state_population": 886667, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:46" + ] + ] + }, + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + "zero_population_blocks": 30617 + }, + { + "block_rows": 179717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:47" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + "populated_blocks": 133846, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "47", + "state_population": 6910840, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:47" + ] + ] + }, + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + "zero_population_blocks": 45871 + }, + { + "block_rows": 668757, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:48" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + "populated_blocks": 449085, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "48", + "state_population": 29145505, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:48" + ] + ] + }, + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + "zero_population_blocks": 219672 + }, + { + "block_rows": 71207, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:49" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + "populated_blocks": 46792, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "49", + "state_population": 3271616, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:49" + ] + ] + }, + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + "zero_population_blocks": 24415 + }, + { + "block_rows": 24611, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:50" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + "populated_blocks": 17976, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "50", + "state_population": 643077, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:50" + ] + ] + }, + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + "zero_population_blocks": 6635 + }, + { + "block_rows": 163491, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:51" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + "populated_blocks": 117361, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "51", + "state_population": 8631393, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:51" + ] + ] + }, + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + "zero_population_blocks": 46130 + }, + { + "block_rows": 158093, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:53" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + "populated_blocks": 111441, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "53", + "state_population": 7705281, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:53" + ] + ] + }, + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + "zero_population_blocks": 46652 + }, + { + "block_rows": 72558, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:54" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + "populated_blocks": 51850, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "54", + "state_population": 1793716, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:54" + ] + ] + }, + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + "zero_population_blocks": 20708 + }, + { + "block_rows": 203059, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:55" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + "populated_blocks": 144948, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "55", + "state_population": 5893718, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:55" + ] + ] + }, + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + "zero_population_blocks": 58111 + }, + { + "block_rows": 53769, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:56" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + "populated_blocks": 22857, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "56", + "state_population": 576851, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:56" + ] + ] + }, + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + "zero_population_blocks": 30912 + } + ], + "puma": { + "selected_state_tract_mappings": 84414, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960" + }, + "source_receipt_sha256": "9adaa4365fc4624dfcb80d456783d5a54806e672384e17b4e84ad08d10bde74b", + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "status": "source_control_passed", + "support_filename": "national-atomic-support.npz", + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960", + "support_size_bytes": 28862508, + "survey_frame_created": false + }, + "normalization_report_sha256": "1da2a391b58956de6541ac8e541d1b67f5d560b601da86caca3baca1859b8c45", + "peak_rss_bytes": 6813728768, + "receipt_sha256": "6c65dee6d9317e522a2076f690341889f867a28652ba18ddd6120bc6c2a80ad0", + "release_eligible": false, + "result": 0, + "scope": "One ordinary-acceptance national source-normalization case after separate root review and approval;104 population-only/geography pins, exact51 states, no survey Frame/source admission/release eligibility. Existing failed and accepted DE runs preserved.", + "source_and_owned_files": 498, + "source_control_accepted": true, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 82.97476216696668 +} diff --git a/packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py b/packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py new file mode 100644 index 000000000..87c922a6d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_national.py @@ -0,0 +1,502 @@ +"""Opt-in national original-source control; every native pin comes from a guard. + +Ordinary CI skips before source path access. The external fixture supplies +``states: {SS: {blocks: pin, total: pin}}``, ``cd: pin``, ``puma: pin`` and +``output_dir``. Each pin is exactly ``{path, sha256, size_bytes}``. There are 104 +sources, covering the 50 states plus DC. No native path or population value is +embedded. The independent population oracle retains only per-state digests; +the mapping check calls maintained parsers separately from the adapter result. +This is source normalization, without survey construction or source admission. +""" + +import hashlib +import json +import os +import re +import stat +import tempfile +from pathlib import Path + +import numpy as np +import pytest + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.build.us_runtime import atomic_block_api_sources as source +from microcosm.build.us_runtime import atomic_block_sources as source_bytes +from microcosm.build.us_runtime.block_ladder_sources import ( + US_STATES, + parse_national_cd_bef, +) +from microcosm.build.us_runtime.puma_ladder_sources import ( + parse_tract_to_puma_relationship, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +_STATES = tuple( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 " + "21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 " + "38 39 40 41 42 44 45 46 47 48 49 50 51 53 54 55 56".split() +) +_SOURCE_IDS = { + "population": "census-2020-dec-pl-api-P1_001N", + "district": "census-2025-cd119-NationalCD119.txt", + "puma": "census-2020-Census-Tract-to-2020-PUMA", +} +_CD_MEMBERS = ( + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt", +) +_MAX_SOURCE_BYTES = 64 * 1024**2 +_MAX_ACQUIRED_BYTES = 1024**3 +# Explicit selected-member cap from the separately reviewed CD metadata probe. +# This is a bound, not a native payload observation by this test's author. +_MAX_CD_MEMBER_BYTES = 163_499_110 +_MAX_SELECTED_BYTES = _MAX_ACQUIRED_BYTES + _MAX_CD_MEMBER_BYTES +_MAX_RESPONSE_ROWS = 2_000_000 +_BOUNDS = { + "max_source_bytes": _MAX_SOURCE_BYTES, + "max_member_bytes": _MAX_CD_MEMBER_BYTES, + "max_total_geography_bytes": _MAX_SELECTED_BYTES, + "max_zip_members": 6, + "max_line_bytes": 64 * 1024, + "max_response_rows": _MAX_RESPONSE_ROWS, +} + + +def _require(condition, reason): + assert condition, "NATIVE_NATIONAL_ATOMIC_" + reason + + +@pytest.fixture +def _native_case(request): + try: + return request.getfixturevalue("guarded_native_atomic_api_national_sources") + except pytest.FixtureLookupError: + pytest.skip("requires exact source pins from the external national guard") + + +def _pin(record, name, filename, maximum): + _require( + type(record) is dict and set(record) == {"path", "sha256", "size_bytes"}, + "SOURCE_PIN", + ) + path, digest, size = record["path"], record["sha256"], record["size_bytes"] + _require( + type(path) is str + and Path(path).is_absolute() + and Path(path).name == filename + and "\0" not in path + and type(digest) is str + and re.fullmatch(r"[0-9a-f]{64}", digest) is not None + and type(size) is int + and 0 < size <= maximum, + "SOURCE_PIN", + ) + return name, path, digest, size + + +def _snapshot(case): + _require( + type(case) is dict and set(case) == {"states", "cd", "puma", "output_dir"}, + "FIXTURE", + ) + states = case["states"] + _require(type(states) is dict and set(states) == set(_STATES), "STATE_ROSTER") + pins = [] + for state_code in _STATES: + pair = states[state_code] + _require(type(pair) is dict and set(pair) == {"blocks", "total"}, "STATE_PAIR") + for kind, maximum in (("blocks", _MAX_SOURCE_BYTES), ("total", 64 * 1024)): + pins.append( + _pin( + pair[kind], + state_code + "." + kind, + f"state-{state_code}.{kind}.P1_001N.json", + maximum, + ) + ) + pins.extend( + ( + _pin(case["cd"], "cd", "cd119.zip", _MAX_SOURCE_BYTES), + _pin( + case["puma"], + "puma", + "2020_Census_Tract_to_2020_PUMA.txt", + 4 * 1024**2, + ), + ) + ) + _require( + len(pins) == 104 + and len({pin[1] for pin in pins}) == 104 + and sum(pin[3] for pin in pins) <= _MAX_ACQUIRED_BYTES, + "SOURCE_ROSTER_OR_BYTES", + ) + output_dir = case["output_dir"] + _require( + type(output_dir) is str + and "\0" not in output_dir + and Path(output_dir).is_absolute(), + "OUTPUT_DIR", + ) + return tuple(pins), output_dir + + +def _file_stamp(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def _read_pinned(record): + _name, path, digest, size = record + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode) and before.st_size == size, "SOURCE_FILE") + payload = stream.read(size + 1) + after = os.fstat(stream.fileno()) + _require( + len(payload) == size + and hashlib.sha256(payload).hexdigest() == digest + and _file_stamp(before) == _file_stamp(after), + "SOURCE_BYTES", + ) + return payload + + +def _source_record(pin): + return source_bytes.AtomicBlockSourceBytes( + _read_pinned(pin), pin[2], _CD_MEMBERS if pin[0] == "cd" else () + ) + + +def _write_private(directory, filename, payload): + descriptor, partial = tempfile.mkstemp(prefix=".atomic-national-", dir=directory) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.link(partial, directory / filename, follow_symlinks=False) + finally: + os.unlink(partial) + + +def _digest_row(digest, geoid, population): + digest.update((geoid + "," + str(population) + "\n").encode("ascii")) + + +def _original_state_summary(block_payload, total_payload, state_code): + rows, totals = json.loads(block_payload), json.loads(total_payload) + _require( + type(rows) is list + and 1 < len(rows) <= _MAX_RESPONSE_ROWS + 1 + and rows[0] == ["P1_001N", "state", "county", "tract", "block"] + and type(totals) is list + and len(totals) == 2 + and totals[0] == ["P1_001N", "state"] + and type(totals[1]) is list + and len(totals[1]) == 2 + and totals[1][1] == state_code + and type(totals[1][0]) is str + and re.fullmatch(r"[0-9]{1,16}", totals[1][0]) is not None, + "ORIGINAL_HEADER_OR_TOTAL", + ) + expected_total = int(totals[1][0]) + _require(expected_total <= 2**53, "ORIGINAL_TOTAL_DOMAIN") + positive, seen, zero_count = {}, set(), 0 + for row in rows[1:]: + _require( + type(row) is list + and len(row) == 5 + and all(type(value) is str for value in row), + "ORIGINAL_ROW", + ) + population, state, county, tract, block = row + _require( + re.fullmatch(r"[0-9]{1,16}", population) is not None + and state == state_code + and re.fullmatch(r"[0-9]{3}", county) is not None + and re.fullmatch(r"[0-9]{6}", tract) is not None + and re.fullmatch(r"[0-9]{4}", block) is not None, + "ORIGINAL_DOMAIN", + ) + count, geoid = int(population), state + county + tract + block + _require(count <= 2**53 and geoid not in seen, "ORIGINAL_DUPLICATE_OR_COUNT") + seen.add(geoid) + if count: + positive[geoid] = count + else: + zero_count += 1 + _require( + bool(positive) and sum(positive.values()) == expected_total, "ORIGINAL_SUM" + ) + digest = hashlib.sha256() + for geoid in sorted(positive): + _digest_row(digest, geoid, positive[geoid]) + return { + "state_population": expected_total, + "block_rows": len(seen), + "populated_blocks": len(positive), + "zero_population_blocks": zero_count, + "ordered_positive_block_population_sha256": digest.hexdigest(), + } + + +def _population_responses(pins_by_name, summaries): + for state_code in _STATES: + blocks = _source_record(pins_by_name[state_code + ".blocks"]) + total = _source_record(pins_by_name[state_code + ".total"]) + summaries[state_code] = _original_state_summary( + blocks.payload, total.payload, state_code + ) + yield state_code, blocks, total + del blocks, total + + +def _check_complete_population(support, summaries): + _require( + set(support.arrays) + == {"area", "state", "county", "tract", "puma", "district", "population"}, + "SUPPORT_COLUMNS", + ) + digests = {state_code: hashlib.sha256() for state_code in _STATES} + counts, totals = dict.fromkeys(_STATES, 0), dict.fromkeys(_STATES, 0) + previous = "" + for geoid, state_code, county, tract, population in zip( + *( + support.arrays[name] + for name in ("area", "state", "county", "tract", "population") + ), + strict=True, + ): + geoid, state_code = str(geoid), str(state_code) + count = int(population) + _require( + re.fullmatch(r"[0-9]{15}", geoid) is not None + and previous < geoid + and state_code in digests + and state_code == geoid[:2] + and str(county) == geoid[:5] + and str(tract) == geoid[:11] + and count == population + and 0 < count <= 2**53, + "OUTPUT_BLOCK_IDENTITY", + ) + previous = geoid + _digest_row(digests[state_code], geoid, count) + counts[state_code] += 1 + totals[state_code] += count + for state_code in _STATES: + expected = summaries[state_code] + _require( + counts[state_code] == expected["populated_blocks"] + and totals[state_code] == expected["state_population"] + and digests[state_code].hexdigest() + == expected["ordered_positive_block_population_sha256"], + "COMPLETE_POSITIVE_BLOCK_PRESERVATION", + ) + + +def _check_complete_mapping_joins(support, pins_by_name, receipt): + # A separate maintained-parser pass checks every selected output join. + # This is not an independent interpretation of Census mapping conventions. + expanded = [0] + cd_payload, cd_pin = source_bytes._selected_zip_member( + _source_record(pins_by_name["cd"]), + "NationalCD119.txt", + bounds=_BOUNDS, + expanded=expanded, + ) + statistics, unassigned = {"source_records": 0, "delegate_records": 0}, set() + districts = parse_national_cd_bef( + source_bytes._cd_lines( + cd_payload, + max_line_bytes=_BOUNDS["max_line_bytes"], + statistics=statistics, + unassigned=unassigned, + ) + ) + _require( + not unassigned.intersection(districts) + and statistics["source_records"] == len(districts) + len(unassigned), + "CD_SOURCE_RECONCILIATION", + ) + del cd_payload, unassigned + puma_payload = _read_pinned(pins_by_name["puma"]) + _require( + expanded[0] + len(puma_payload) <= _MAX_SELECTED_BYTES, "JOIN_SOURCE_BYTES" + ) + pumas = parse_tract_to_puma_relationship( + source_bytes._lines( + puma_payload, encoding="utf-8", max_line_bytes=_BOUNDS["max_line_bytes"] + ), + allowed_state_fips=frozenset(_STATES), + ) + del puma_payload + for geoid, district, puma in zip( + *(support.arrays[name] for name in ("area", "district", "puma")), strict=True + ): + block = int(geoid) + _require( + block in districts + and block // 10000 in pumas + and str(district) == f"{districts[block]:04d}" + and str(puma) == f"{pumas[block // 10000]:07d}", + "COMPLETE_MAPPING_JOIN", + ) + _require( + all( + receipt["sources"]["district"][key] == value + for key, value in cd_pin.items() + ) + and receipt["sources"]["puma"]["selected_state_tract_mappings"] == len(pumas), + "MAPPING_RECEIPT", + ) + + +def test_native_national_api_sources_reconcile_and_round_trip(_native_case): + _require(tuple(row[0] for row in US_STATES) == _STATES, "MAINTAINED_STATE_ROSTER") + pins, output_path = _snapshot(_native_case) + pins_by_name = {pin[0]: pin for pin in pins} + directory = Path(output_path) + info = directory.lstat() + _require( + stat.S_ISDIR(info.st_mode) + and info.st_uid == os.getuid() + and info.st_mode & 0o077 == 0, + "PRIVATE_OUTPUT_DIR", + ) + summaries = {} + support_bytes, receipt_bytes = source.assemble_atomic_block_api_sources( + population_responses=_population_responses(pins_by_name, summaries), + cd_archive=_source_record(pins_by_name["cd"]), + tract_to_puma=_source_record(pins_by_name["puma"]), + state_fips=_STATES, + source_ids=_SOURCE_IDS, + **_BOUNDS, + ) + _require(0 < len(support_bytes) <= RAW_BYTES_MAX_BYTES, "SUPPORT_SIZE") + _require(0 < len(receipt_bytes) <= 256 * 1024, "RECEIPT_SIZE") + receipt, support = json.loads(receipt_bytes), decode_atomic_support(support_bytes) + _check_complete_population(support, summaries) + _check_complete_mapping_joins(support, pins_by_name, receipt) + state_totals = { + state_code: summaries[state_code]["state_population"] for state_code in _STATES + } + population_receipts = receipt["sources"]["population"] + _require( + receipt["state_fips"] == list(_STATES) + and receipt["source_ids"] == _SOURCE_IDS + and receipt["limits"] == _BOUNDS + and len(population_receipts) == 51 + and [row["state_fips"] for row in population_receipts] == list(_STATES) + and receipt["reconciliation"]["state_population"] == state_totals + and receipt["reconciliation"]["output_state_population"] == state_totals + and receipt["reconciliation"]["population_total"] == sum(state_totals.values()) + and receipt["reconciliation"]["populated_blocks"] + == sum(value["populated_blocks"] for value in summaries.values()) + and receipt["reconciliation"]["selected_geography_bytes"] + == sum(pin[3] for pin in pins if pin[0] != "cd") + + receipt["sources"]["district"]["selected_member_size_bytes"] + <= _MAX_SELECTED_BYTES + and receipt["support_sha256"] == hashlib.sha256(support_bytes).hexdigest(), + "RECONCILIATION", + ) + source_receipts = { + "cd": receipt["sources"]["district"], + "puma": receipt["sources"]["puma"], + } + for state_code, state_receipt in zip(_STATES, population_receipts, strict=True): + for key in ( + "state_population", + "block_rows", + "populated_blocks", + "zero_population_blocks", + ): + _require(state_receipt[key] == summaries[state_code][key], "STATE_RECEIPT") + source_receipts[state_code + ".blocks"] = state_receipt["blocks"] + source_receipts[state_code + ".total"] = state_receipt["state_total"] + for name, _path, digest, size in pins: + _require( + source_receipts[name]["sha256"] == digest + and source_receipts[name]["size_bytes"] == size, + "RECEIPT_SOURCE_PIN", + ) + _require( + receipt["sources"]["district"]["zip_members"] == list(_CD_MEMBERS) + and receipt["sources"]["district"]["crc_checked_members"] + == ["NationalCD119.txt"] + and receipt["sources"]["district"]["unselected_member_contents_read"] is False + and receipt["sources"]["district"]["district_relation"] == "official_tabulation" + and support.metadata["columns"]["district"]["relation"] == "official_tabulation" + and support.metadata["columns"]["population"]["basis"] == "2020_census_persons" + and receipt["exact_input_hashes_verified"] is True + and all( + receipt[name] is False + for name in ( + "publisher_provenance_established", + "request_origin_verified", + "source_admission_issued", + "release_eligible", + ) + ), + "AUTHORITY_BOUNDARY", + ) + support_path = directory / "national-atomic-support.npz" + _write_private(directory, support_path.name, support_bytes) + replay_bytes = _read_pinned( + ( + "support", + str(support_path), + hashlib.sha256(support_bytes).hexdigest(), + len(support_bytes), + ) + ) + _require(replay_bytes == support_bytes, "SUPPORT_READBACK") + replay = decode_atomic_support(replay_bytes) + _require(replay.metadata == support.metadata, "SUPPORT_METADATA_READBACK") + _require(set(replay.arrays) == set(support.arrays), "SUPPORT_COLUMNS_READBACK") + for name in support.arrays: + _require( + np.array_equal(replay.arrays[name], support.arrays[name]), + "SUPPORT_ARRAY_READBACK", + ) + del replay, replay_bytes + # Final reads reauthenticate all104 pinned originals without retaining them. + # The guard also checks these pins before and after the entire test process. + for pin in pins: + _read_pinned(pin) + _require(_snapshot(_native_case) == (pins, output_path), "FINAL_FIXTURE_PINS") + report = canonical_json( + { + "protocol": "microcosm.us.native-national-atomic-api-control.v1", + "status": "source_control_passed", + "state_fips": list(_STATES), + "source_count": 104, + "source_receipt": receipt, + "source_receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), + "population_oracle": summaries, + "mapping_oracle": "separate maintained-parser pass; complete selected joins", + "source_read_limits": { + "unique_acquired_bytes": _MAX_ACQUIRED_BYTES, + "selected_cd_member_bytes_per_pass": _MAX_CD_MEMBER_BYTES, + "selected_geography_bytes_per_adapter_pass": _MAX_SELECTED_BYTES, + "mapping_verification_passes": 1, + }, + "support_filename": support_path.name, + "support_sha256": hashlib.sha256(support_bytes).hexdigest(), + "support_size_bytes": len(support_bytes), + "exact_support_readback": True, + "national_coverage": True, + "survey_frame_created": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + _require(len(report) <= 256 * 1024, "REPORT_SIZE") + _write_private(directory, "national-atomic-normalization.json", report) From d3a0c3b5a7b6fcd02d1c988b63dac01ecac19685 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 20:19:02 -0400 Subject: [PATCH 040/255] Record financial successor CPU-limit failure --- ...cessor-positive-1-limit-stop-20260909.json | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 experiments/us-financial-successor-positive-1-limit-stop-20260909.json diff --git a/experiments/us-financial-successor-positive-1-limit-stop-20260909.json b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json new file mode 100644 index 000000000..c7b05b458 --- /dev/null +++ b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json @@ -0,0 +1,34 @@ +{ + "all_expected_source_code_and_model_pins_match_after_stop": true, + "automatic_retry": false, + "code_resources": 12, + "completed_test_count": null, + "configured_cpu_seconds": 600, + "configured_wall_seconds": 900, + "control_accepted": false, + "expected_test_count": 1, + "final_runtime_receipt_written": false, + "guard_sha256": "d65bd3f82a64791a0720dab30a2938456a90757b6c9aa6cfa4a30662edab1b6b", + "in_process_final_verification_completed": false, + "model_source_files": 5983, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_inputs_read": false, + "native_threads": 1, + "observed_cpu_seconds": null, + "observed_peak_rss_bytes": null, + "other_selections_executed": false, + "process_exit_code": 152, + "release_eligible": false, + "resource_map_sha256": "c6a557ab9d9231457a7c9f0d1c4a19d2ea354bcdeb7063812ddc88a3b3cf6ef4", + "result": "incomplete_resource_limit_stop", + "scope": "Positive invented financial successor cold/required graph, original budget, admitted financial ancestry, weight-only transition and unchanged default path. CPU-limit stop before final receipt; no pass or new successor/native acceptance. Next action is separately reviewed bounded profiling, not a budget increase.", + "source_files": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "termination_signal": "SIGXCPU", + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py": "613bf949a73ed4d0a6c9f92f01fb48a64b426db2825d719ccf023e938c5c18d7", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "packages/microcosm-build/tests/test_us_survey_financial_successor.py": "a42e7dadf3b77f6cf976db9c904243e787f8704bbc6d1a82bd820246f3c2e980" + } +} From 9757f4a57975e956e1a3cd43dcaefad4564d47df Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 00:10:34 -0400 Subject: [PATCH 041/255] Document measured financial fixture validation cost --- docs/us-launch-review.md | 23 +- ...us-financial-fixture-profile-20260910.json | 248 ++++++++++++++++++ 2 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 experiments/us-financial-fixture-profile-20260910.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 794cfc5f3..31c0db89e 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -18,8 +18,8 @@ The optional ten-node survey prefix now assigns a block before cloning, and its thirteen-node age-calibration extension passes cold execution and required replay on invented originals. A twenty-node extension now carries atomic geography through current financial imputation and required replay. Native -Delaware block support passes source normalization; complete national support -and national/CD acceptance remain pending. The earlier joint tract/district +Delaware and complete national block support pass source normalization; +national/CD acceptance of the enriched population remains pending. The earlier joint tract/district operator is a separate path. See [Geography assignment](geography-assignment.md) for the country contracts, implementation and source boundaries. @@ -220,6 +220,25 @@ receipts without reopening source bodies. This clears the national support normalization check; assigning households, evaluating geographic fit and calibrating the enriched survey remain separate acceptance steps. +The financial successor's positive control reached its 600 CPU-second limit +without a final receipt. A separate, bounded profile of the unchanged invented +cold/required financial fixture now passes its test, original teardown and all +final code/resource checks: 303.90 CPU seconds, 308.26 wall seconds and 594.3 MB +peak RSS. This diagnoses the fixture; it does not accept the financial successor. +See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). +Loaded ACS code verification accounts for 180.20 cumulative seconds inside the +298.15-second profiled fixture. Its nested function checker runs 2,009,108 times +and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. +Cumulative timings overlap and profiling adds overhead. The next change targets +repeated comparisons of immutable code objects within a single verification, +while preserving fresh source reads, aliases, globals, closure checks and final +mutation seals. That optimization, the remaining successor controls and the +native financial pilot have not yet passed. + +Current CI separately reports stale source-attested spec/seed fingerprints and +an imported-resource identity expectation. These failures remain open; neither +the component passes nor this profile establishes a green consolidation. + ## Related reviews - [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): diff --git a/experiments/us-financial-fixture-profile-20260910.json b/experiments/us-financial-fixture-profile-20260910.json new file mode 100644 index 000000000..964ba4c2e --- /dev/null +++ b/experiments/us-financial-fixture-profile-20260910.json @@ -0,0 +1,248 @@ +{ + "accepted": true, + "approval_sha256": "6e53647ed6bb1af53c120b2c8f105f3517d6264476bb767547ae2dda1cf69bc7", + "cpu_seconds": 303.897467, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "fixture_acquisition_completed": true, + "fixture_original_teardown_passed": true, + "guard_sha256": "e7972505720ee6ede8d3859e28e2c9171d4f568d477d90670371ca17a36167d1", + "native_acceptance": false, + "native_reads_stats_hashes": false, + "next_step": "Review and test per-producer code-comparison memoization without removing live globals, aliases, closure checks, source reads or final I/O seals. No optimization or larger run accepted by this diagnostic.", + "partial_snapshot_needed": false, + "peak_rss_bytes": 594296832, + "peer_source_review_sha256": "6b4614e182df7fa519a1079a6c5d99ded55bbdc45f021b46908e041c8895f4af", + "profile_scope": "fixture_acquisition_only", + "profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "profiled_implementation_sha256": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53" + }, + "profiled_seconds": 298.15339916600004, + "protocol": "microcosm.us.financial-fixture-profile-evidence.v1", + "receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "release_acceptance": false, + "runtime_result": 0, + "scope": "invented_financial_fixture_profile", + "successor_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_functions": [ + { + "calls": 1, + "cumulative_seconds": 298.15339916600004, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.004364749 + }, + { + "calls": 2, + "cumulative_seconds": 297.206106625, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.0062629930000000006 + }, + { + "calls": 110, + "cumulative_seconds": 212.956586461, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.004810231 + }, + { + "calls": 108, + "cumulative_seconds": 209.131314091, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.0005602070000000001 + }, + { + "calls": 652, + "cumulative_seconds": 180.19579690400002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 129, + "recursive_calls": 0, + "self_seconds": 5.073519417 + }, + { + "calls": 14, + "cumulative_seconds": 161.371121044, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.007554145000000001 + }, + { + "calls": 38464, + "cumulative_seconds": 151.50668402300002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 51, + "recursive_calls": 0, + "self_seconds": 9.209266879000001 + }, + { + "calls": 8, + "cumulative_seconds": 133.111351877, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.006935434000000001 + }, + { + "calls": 2, + "cumulative_seconds": 115.295601876, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001364704 + }, + { + "calls": 6, + "cumulative_seconds": 112.51926329300001, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.008178516 + }, + { + "calls": 15, + "cumulative_seconds": 109.250808543, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0037998010000000002 + }, + { + "calls": 348, + "cumulative_seconds": 103.018557294, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.14776667400000001 + }, + { + "calls": 2009108, + "cumulative_seconds": 95.553654135, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 86, + "recursive_calls": 51772, + "self_seconds": 91.045168105 + }, + { + "calls": 8, + "cumulative_seconds": 93.21181225100001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0020371490000000003 + }, + { + "calls": 150, + "cumulative_seconds": 92.18578382800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 394, + "recursive_calls": 0, + "self_seconds": 0.011203179 + }, + { + "calls": 116, + "cumulative_seconds": 69.695211884, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.0037891110000000004 + }, + { + "calls": 4, + "cumulative_seconds": 66.760753874, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.000840784 + }, + { + "calls": 110, + "cumulative_seconds": 66.121036298, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.00016753900000000002 + }, + { + "calls": 23, + "cumulative_seconds": 49.949269957000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.0047751980000000005 + }, + { + "calls": 2, + "cumulative_seconds": 41.477013541000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007954430000000001 + } + ], + "unexpected_refusals": {}, + "verified_after_run": { + "code_resource_files": 12, + "model_source_files": 5983, + "source_and_owned_files": 500 + }, + "wall_seconds": 308.2577062920318 +} From 56d9bf48456e8e8f14d1a53dfabd590a8df53910 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 00:31:19 -0400 Subject: [PATCH 042/255] Fix worker resource identity tests for lazy imports --- .../893-worker-resource-trace.fixed.md | 1 + .../tests/test_us_stacked_spine.py | 50 ++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 changelog.d/893-worker-resource-trace.fixed.md diff --git a/changelog.d/893-worker-resource-trace.fixed.md b/changelog.d/893-worker-resource-trace.fixed.md new file mode 100644 index 000000000..a48093a90 --- /dev/null +++ b/changelog.d/893-worker-resource-trace.fixed.md @@ -0,0 +1 @@ +Update the worker resource identity regression to follow the actual lazy import closure, and verify that opened JSON resources affect identity while unlisted resources do not. diff --git a/packages/microcosm-build/tests/test_us_stacked_spine.py b/packages/microcosm-build/tests/test_us_stacked_spine.py index b30c20272..381712f7f 100644 --- a/packages/microcosm-build/tests/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/test_us_stacked_spine.py @@ -5008,20 +5008,25 @@ def test_worker_identity_refuses_duplicate_backend_provider_distribution_identit @pytest.mark.usefixtures("live_worker_identity") -def test_worker_transitive_source_identity_binds_actual_imported_package_resource( +def test_worker_transitive_source_identity_binds_actual_imported_namespace_file( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - target_name = "soi_table_2_1_interest_components_ty2015.json" + target_resource = "microcosm/build/us_runtime/puf_qrf_chain.py" trace = worker_identity_module._clean_worker_import_trace() + # E19200 decomposition runs in the parent, outside worker startup. + assert all( + Path(path).name != "soi_table_2_1_interest_components_ty2015.json" + for path in trace["opened_files"] + ) + target_path = Path( + trace["module_origins"]["microcosm.build.us_runtime.puf_qrf_chain"] + ).resolve() before = worker_identity_module._worker_package_resource_rows(trace) stdlib_rows = worker_identity_module._worker_stdlib_import_rows(trace) assert any(row["module"] == "argparse" for row in stdlib_rows) - target_rows = [ - row for row in before if str(row.get("resource", "")).endswith(target_name) - ] + target_rows = [row for row in before if row["resource"] == target_resource] assert len(target_rows) == 1 - target_resource = target_rows[0]["resource"] target_sha256 = target_rows[0]["sha256"] _stub_worker_identity_static_closure(monkeypatch) runtime = tmp_path / "libpython-fixture.so" @@ -5043,7 +5048,7 @@ def test_worker_transitive_source_identity_binds_actual_imported_package_resourc def changed_resource_bytes(path: Path) -> bytes: raw = original_read_bytes(path) - if path.name == target_name: + if path.resolve() == target_path: return raw + b"\n" return raw @@ -5068,6 +5073,37 @@ def changed_resource_bytes(path: Path) -> bytes: ) +@pytest.mark.usefixtures("live_worker_identity") +def test_worker_namespace_trace_hashes_opened_json_and_ignores_unlisted_resource( + tmp_path: Path, +) -> None: + trace = _fixture_worker_import_trace(tmp_path) + namespace_root = Path(trace["namespace_roots"][0]) + opened = namespace_root / "build" / "fixture-opened.json" + unlisted = namespace_root / "build" / "fixture-unlisted.json" + original_bytes = b'{"fixture": 1}\n' + changed_bytes = b'{"fixture": 2}\n' + opened.write_bytes(original_bytes) + unlisted.write_bytes(b'{"unlisted": 1}\n') + trace["opened_files"] = (str(opened),) + resource = "microcosm/build/fixture-opened.json" + + before = worker_identity_module._worker_package_resource_rows(trace) + assert before == [ + {"resource": resource, "sha256": hashlib.sha256(original_bytes).hexdigest()} + ] + + unlisted.write_bytes(b'{"unlisted": 2}\n') + assert worker_identity_module._worker_package_resource_rows(trace) == before + + opened.write_bytes(changed_bytes) + after = worker_identity_module._worker_package_resource_rows(trace) + assert after == [ + {"resource": resource, "sha256": hashlib.sha256(changed_bytes).hexdigest()} + ] + assert after != before + + @pytest.mark.parametrize( "semantic_change", ( From 3a2edc82ac328619862ea2c266b70ed874620a5e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 00:56:08 -0400 Subject: [PATCH 043/255] Record guarded ACS memo comparison without adopting it --- docs/us-launch-review.md | 25 +- .../us-acs-code-memo-comparison-20260910.json | 217 ++++++++ ...s-acs-code-memo-not-adopted-20260910.patch | 497 ++++++++++++++++++ .../us-acs-loaded-code-controls-20260910.json | 33 ++ 4 files changed, 763 insertions(+), 9 deletions(-) create mode 100644 experiments/us-acs-code-memo-comparison-20260910.json create mode 100644 experiments/us-acs-code-memo-not-adopted-20260910.patch create mode 100644 experiments/us-acs-loaded-code-controls-20260910.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 31c0db89e..6a8d047d1 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -229,15 +229,22 @@ See the [profile evidence](../experiments/us-financial-fixture-profile-20260910. Loaded ACS code verification accounts for 180.20 cumulative seconds inside the 298.15-second profiled fixture. Its nested function checker runs 2,009,108 times and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. -Cumulative timings overlap and profiling adds overhead. The next change targets -repeated comparisons of immutable code objects within a single verification, -while preserving fresh source reads, aliases, globals, closure checks and final -mutation seals. That optimization, the remaining successor controls and the -native financial pilot have not yet passed. - -Current CI separately reports stale source-attested spec/seed fingerprints and -an imported-resource identity expectation. These failures remain open; neither -the component passes nor this profile establishes a green consolidation. +Cumulative timings overlap and profiling adds overhead. The invocation-local +code comparison memo now passes all 18 focused invented controls, including +changed source, aliases, globals, closures, mutable constants and the historical +catalogue caller. Source and resource checks pass before, after and in the +external postcheck. See the [control evidence](../experiments/us-acs-loaded-code-controls-20260910.json). +The paired full-fixture run also passed correctness and final checks, but took +319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility +work and demonstrated no speed improvement, so its implementation was not +adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) +remain recorded. Further profiling is narrowing the original verification cost; +the remaining successor controls and native financial pilot remain pending. + +Current CI separately reports stale source-attested spec/seed fingerprints. +The worker resource identity test now follows the actual lazy import closure +and includes an invented opened-JSON regression; its new CI result is pending. +Neither the component passes nor this profile establishes a green consolidation. ## Related reviews diff --git a/experiments/us-acs-code-memo-comparison-20260910.json b/experiments/us-acs-code-memo-comparison-20260910.json new file mode 100644 index 000000000..1cdec2fc8 --- /dev/null +++ b/experiments/us-acs-code-memo-comparison-20260910.json @@ -0,0 +1,217 @@ +{ + "baseline": { + "cpu_seconds": 303.897467, + "peak_rss_bytes": 594296832, + "wall_seconds": 308.2577062920318 + }, + "baseline_receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "correctness_controls": 18, + "cpu_ratio": 1.0405705290067455, + "date": "2026-09-10", + "decision": "not_adopted", + "experiment_patch_sha256": "bc2b245b712256578ad6587c7a242af38e4244eaeacc268b589b2ed0c38ae557", + "guard_sha256": "8a2cf87bd0a00c1de1d8e4263764394551efed8e1ecb3d3297eeb5b1ab0eaf5e", + "implementation_restored_to_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "native_speedup_established": false, + "optimized": { + "cpu_seconds": 316.226748, + "peak_rss_bytes": 601653248, + "wall_seconds": 319.78662233403884 + }, + "optimized_receipt_sha256": "49016ede5317b1e025e6d476364398b8233e3c589b7b80ef78781d154ab9f0d0", + "optimized_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "profile_and_original_teardown_passed": true, + "reason": "No measured speed improvement in the same instrumented fixture; added eligibility work exceeded the comparison savings.", + "release_acceptance": false, + "scope": "one paired invented cold/required fixture run, identical cProfile policy and dimensions", + "successor_controls_executed": false, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_optimized_records": [ + { + "calls": 1, + "cumulative_seconds": 309.704728666, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.0052826240000000005 + }, + { + "calls": 2, + "cumulative_seconds": 308.7401165, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.005962359 + }, + { + "calls": 110, + "cumulative_seconds": 224.98760203900002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.00429048 + }, + { + "calls": 108, + "cumulative_seconds": 220.871511795, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.000492867 + }, + { + "calls": 652, + "cumulative_seconds": 195.60535725600002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 223, + "recursive_calls": 0, + "self_seconds": 4.97942924 + }, + { + "calls": 14, + "cumulative_seconds": 169.824184456, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.006830769 + }, + { + "calls": 38464, + "cumulative_seconds": 168.34809192, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 137, + "recursive_calls": 0, + "self_seconds": 8.731628122 + }, + { + "calls": 8, + "cumulative_seconds": 139.864694334, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.0066878350000000005 + }, + { + "calls": 2, + "cumulative_seconds": 119.01038833400001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001211868 + }, + { + "calls": 6, + "cumulative_seconds": 116.761678708, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.007467228 + }, + { + "calls": 15, + "cumulative_seconds": 115.40236200000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0036097990000000003 + }, + { + "calls": 2010412, + "cumulative_seconds": 114.621920672, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 176, + "recursive_calls": 51772, + "self_seconds": 86.630530445 + }, + { + "calls": 348, + "cumulative_seconds": 111.58561966500001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.135475169 + }, + { + "calls": 150, + "cumulative_seconds": 98.955415883, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 490, + "recursive_calls": 0, + "self_seconds": 0.010436341 + }, + { + "calls": 8, + "cumulative_seconds": 98.24177225000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0019372500000000002 + }, + { + "calls": 116, + "cumulative_seconds": 75.394407754, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.003720891 + }, + { + "calls": 110, + "cumulative_seconds": 71.55709778800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.000156618 + }, + { + "calls": 4, + "cumulative_seconds": 70.568924458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.0008037900000000001 + }, + { + "calls": 23, + "cumulative_seconds": 52.046634627, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.003941233000000001 + }, + { + "calls": 2, + "cumulative_seconds": 42.917792458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007081860000000001 + } + ], + "wall_ratio": 1.037400252472796 +} diff --git a/experiments/us-acs-code-memo-not-adopted-20260910.patch b/experiments/us-acs-code-memo-not-adopted-20260910.patch new file mode 100644 index 000000000..246a86e19 --- /dev/null +++ b/experiments/us-acs-code-memo-not-adopted-20260910.patch @@ -0,0 +1,497 @@ +--- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py ++++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +@@ -48,13 +48,103 @@ + raise ACSNativeCoverageBindingError(code) + + +-def _live_code(module, compiled): ++def _immutable_code_constants(value, immutable_codes, depth=0): ++ """Recognize exact immutable compiler values without invoking custom equality. ++ ++ Runtime-created CodeType constants need not be compiler values. Unfamiliar ++ values and deeply nested trees take the original uncached equality path; ++ eligibility is never a new source-acceptance requirement. Positive code ++ entries retain the object itself so integer identity keys cannot be reused. ++ """ ++ if depth > 64: ++ return False ++ kind = type(value) ++ if any( ++ kind is scalar ++ for scalar in ( ++ type(None), ++ bool, ++ int, ++ float, ++ complex, ++ str, ++ bytes, ++ type(Ellipsis), ++ ) ++ ): ++ return True ++ if kind is tuple or kind is frozenset: ++ return all( ++ _immutable_code_constants(item, immutable_codes, depth + 1) ++ for item in value ++ ) ++ if kind is not CodeType: ++ return False ++ if immutable_codes.get(id(value)) is value: ++ return True ++ # Check all stored public CodeType fields, not merely co_consts: runtime ++ # constructors may admit scalar subclasses with custom comparison methods. ++ if not all( ++ type(getattr(value, name)) is int ++ for name in ( ++ "co_argcount", ++ "co_posonlyargcount", ++ "co_kwonlyargcount", ++ "co_nlocals", ++ "co_stacksize", ++ "co_flags", ++ "co_firstlineno", ++ ) ++ ) or not all( ++ type(getattr(value, name)) is str ++ for name in ("co_filename", "co_name", "co_qualname") ++ ): ++ return False ++ if not all( ++ type(getattr(value, name)) is bytes ++ for name in ("co_code", "co_linetable", "co_exceptiontable") ++ ): ++ return False ++ for name in ("co_names", "co_varnames", "co_freevars", "co_cellvars"): ++ items = getattr(value, name) ++ if type(items) is not tuple or any(type(item) is not str for item in items): ++ return False ++ if type(value.co_consts) is not tuple or not _immutable_code_constants( ++ value.co_consts, immutable_codes, depth + 1 ++ ): ++ return False ++ immutable_codes[id(value)] = value ++ return True ++ ++ ++def _same_loaded_code(live, reference, code_pairs, immutable_codes): ++ """Memoize equality only for strongly retained, immutable code pairs.""" ++ if type(live) is not CodeType or type(reference) is not CodeType: ++ return live == reference ++ key = (id(live), id(reference)) ++ retained = code_pairs.get(key) ++ if retained is not None and retained[0] is live and retained[1] is reference: ++ return True ++ eligible = _immutable_code_constants( ++ live, immutable_codes ++ ) and _immutable_code_constants(reference, immutable_codes) ++ matches = live == reference ++ if matches and eligible: ++ code_pairs[key] = (live, reference) ++ return matches ++ ++ ++def _live_code(module, compiled, code_pairs=None, immutable_codes=None): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ ++ if code_pairs is None: ++ code_pairs = {} ++ if immutable_codes is None: ++ immutable_codes = {} + path = getattr(module, "__file__", None) + if path is None: + return +@@ -102,15 +192,19 @@ + + visit(compile(Path(path).read_bytes(), path, "exec", dont_inherit=True)) + compiled[path] = codes ++ live = function.__code__ ++ _require(function.__globals__ is vars(origin), "LOADED_PRODUCER") + _require( +- function.__globals__ is vars(origin) +- and function.__code__ == compiled[path].get(function.__code__.co_qualname), ++ _same_loaded_code( ++ live, compiled[path].get(live.co_qualname), code_pairs, immutable_codes ++ ), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) ++ _require(function.__code__ is live, "LOADED_PRODUCER") + + for value in vars(module).values(): + if isinstance(value, FunctionType): +@@ -147,10 +241,12 @@ + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} ++ code_pairs = {} ++ immutable_codes = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: +- _live_code(module, compiled) ++ _live_code(module, compiled, code_pairs, immutable_codes) + return { + "preparation": manifest, + "coverage": coverage._producer(), +--- /dev/null ++++ b/packages/microcosm-build/tests/test_us_acs_loaded_code_validation.py +@@ -0,0 +1,351 @@ ++"""Invented Python source controls for invocation-local ACS code equality memo.""" ++ ++from __future__ import annotations ++ ++import sys ++from pathlib import Path ++from types import CodeType, FunctionType, ModuleType ++ ++import pytest ++ ++from microcosm.build.us_runtime import acs_native_coverage_binding as owner ++ ++ ++def _module(tmp_path, monkeypatch, name, source): ++ path = tmp_path / (name + ".py") ++ path.write_text(source, encoding="utf-8") ++ module = ModuleType(name) ++ module.__file__ = str(path) ++ module.__package__ = "" ++ monkeypatch.setitem(sys.modules, name, module) ++ exec(compile(source, str(path), "exec", dont_inherit=True), vars(module)) ++ return module ++ ++ ++def _code(source="def f():\n return 7\n"): ++ compiled = compile(source, "invented-code.py", "exec", dont_inherit=True) ++ return next(value for value in compiled.co_consts if type(value) is CodeType) ++ ++ ++def _state(): ++ return {}, {}, {} ++ ++ ++def _refuses(): ++ return pytest.raises(owner.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$") ++ ++ ++def _cell(value): ++ return (lambda: value).__closure__[0] ++ ++ ++def test_repeated_exact_pair_skips_only_second_deep_comparison(): ++ live, reference = _code(), _code() ++ pairs, immutable = {}, {} ++ eligibility_calls = [] ++ eligibility_code = owner._immutable_code_constants.__code__ ++ ++ def profile(frame, event, _arg): ++ if event == "call" and frame.f_code is eligibility_code: ++ eligibility_calls.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ first_calls = len(eligibility_calls) ++ assert first_calls > 0 ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert len(eligibility_calls) == first_calls ++ finally: ++ sys.setprofile(previous) ++ assert list(pairs) == [(id(live), id(reference))] ++ assert pairs[(id(live), id(reference))][0] is live ++ assert pairs[(id(live), id(reference))][1] is reference ++ assert immutable[id(live)] is live ++ assert immutable[id(reference)] is reference ++ ++ ++def test_wrong_strong_reference_does_not_authorize_an_identity_key(): ++ live, reference, changed = _code(), _code(), _code("def f():\n return 8\n") ++ pairs = {(id(changed), id(reference)): (live, reference)} ++ assert not owner._same_loaded_code(changed, reference, pairs, {}) ++ ++ ++def test_historical_two_argument_caller_keeps_fresh_validation(tmp_path, monkeypatch): ++ module = _module( ++ tmp_path, monkeypatch, "memo_legacy_call", "def f():\n return 7\n" ++ ) ++ compiled = {} ++ owner._live_code(module, compiled) ++ owner._live_code(module, compiled) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, compiled) ++ ++ ++def test_import_alias_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ origin = _module(tmp_path, monkeypatch, "memo_origin", "def f():\n return 7\n") ++ alias = _module(tmp_path, monkeypatch, "memo_alias", "from memo_origin import f\n") ++ state = _state() ++ owner._live_code(origin, *state) ++ owner._live_code(alias, *state) ++ alias.f = FunctionType(origin.f.__code__, vars(origin)) ++ with _refuses(): ++ owner._live_code(alias, *state) ++ ++ ++def test_equal_code_with_non_origin_globals_refuses_after_pair_hit( ++ tmp_path, monkeypatch ++): ++ module = _module( ++ tmp_path, monkeypatch, "memo_globals", "def f():\n return 7\nalias = f\n" ++ ) ++ state = _state() ++ owner._live_code(module, *state) ++ module.alias = FunctionType(module.f.__code__, {"__name__": module.__name__}) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_replaced_code_refuses_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_replaced", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++_CLOSURE_SOURCE = """def factory(): ++ def inner(): ++ return 7 ++ def wrapped(): ++ return inner() ++ return wrapped ++wrapped = factory() ++""" ++ ++ ++def test_replaced_closure_function_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ cell = module.wrapped.__closure__[0] ++ original = cell.cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ monkeypatch.setattr(cell, "cell_contents", bad) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_distinct_closure_with_same_outer_code_is_not_memoized_as_valid( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_second_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.wrapped.__closure__[0].cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ module.second = FunctionType( ++ module.wrapped.__code__, vars(module), closure=(_cell(bad),) ++ ) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_source_read_and_replacement_remain_fresh_within_shared_invocation_state( ++ tmp_path, monkeypatch ++): ++ source = "def f():\n return 7\n" ++ module = _module(tmp_path, monkeypatch, "memo_source", source) ++ path = Path(module.__file__) ++ state = _state() ++ reads = [] ++ reader = Path.read_bytes.__code__ ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "call" ++ and frame.f_code is reader ++ and frame.f_locals["self"] == path ++ ): ++ reads.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ # One AST read on each scan, plus the existing first code-compilation read. ++ assert len(reads) == 3 ++ changed = "def f():\n return 8\n" ++ fired = [] ++ ++ def change_after_scan(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is owner._live_code.__code__ ++ and frame.f_locals["module"] is module ++ and not fired ++ ): ++ fired.append(1) ++ path.write_text(changed, encoding="utf-8") ++ exec(compile(changed, str(path), "exec", dont_inherit=True), vars(module)) ++ ++ try: ++ sys.setprofile(change_after_scan) ++ owner._live_code(module, *state) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ assert fired == [1] ++ # A distinct producer invocation recompiles the new current source bytes. ++ owner._live_code(module, *_state()) ++ ++ ++def test_code_swap_on_pair_return_refuses_before_function_check_returns( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_return", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.f.__code__ ++ changed = _code("def f():\n return 8\n") ++ target = owner._same_loaded_code.__code__ ++ fired = [] ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is target ++ and frame.f_locals["live"] is original ++ and not fired ++ ): ++ fired.append(1) ++ module.f.__code__ = changed ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ module.f.__code__ = original ++ assert fired == [1] ++ ++ ++@pytest.mark.parametrize( ++ "container", [lambda x: x, lambda x: (x,), lambda x: frozenset((x,))] ++) ++def test_mutable_custom_constants_never_gain_pair_or_eligibility_authority(container): ++ class MutableEqual: ++ def __init__(self): ++ self.equal = True ++ ++ def __eq__(self, _other): ++ return self.equal ++ ++ __hash__ = object.__hash__ ++ ++ value = MutableEqual() ++ reference = _code() ++ live = reference.replace(co_consts=(None, container(value))) ++ pairs, immutable = {}, {} ++ for equal in (True, False, True): ++ value.equal = equal ++ # CPython may reject custom constant types before calling __eq__. ++ # Preserve the original decision; do not assume initial acceptance. ++ expected = live == reference ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is expected ++ assert not pairs ++ assert id(live) not in immutable ++ ++ ++def test_nested_code_with_mutable_constant_remains_uncached(): ++ mutable = [] ++ inner = _code().replace(co_consts=(None, mutable)) ++ outer = _code().replace(co_consts=(None, inner)) ++ reference = _code().replace(co_consts=(None, _code())) ++ pairs, immutable = {}, {} ++ for content in ((), (7,), (8,)): ++ mutable[:] = content ++ assert owner._same_loaded_code(outer, reference, pairs, immutable) is ( ++ outer == reference ++ ) ++ assert not pairs ++ assert id(inner) not in immutable ++ assert id(outer) not in immutable ++ ++ ++def test_constant_eligibility_never_calls_custom_metaclass_equality(): ++ class CustomType(type): ++ def __eq__(cls, _other): ++ raise AssertionError("constant eligibility called custom type equality") ++ ++ class CustomValue(metaclass=CustomType): ++ pass ++ ++ live = _code().replace(co_consts=(None, CustomValue())) ++ immutable = {} ++ assert owner._immutable_code_constants(live, immutable) is False ++ assert id(live) not in immutable ++ ++ ++def test_scalar_subclasses_and_deep_values_fall_back_without_new_refusal(): ++ class CustomInt(int): ++ pass ++ ++ deep = 7 ++ for _ in range(70): ++ deep = (deep,) ++ for value in (CustomInt(7), deep): ++ live = _code().replace(co_consts=(None, value)) ++ reference = live.replace() ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is ( ++ live == reference ++ ) ++ assert not pairs ++ ++ ++def test_compiler_constant_shapes_and_nested_functions_are_cacheable(): ++ source = """def f(x): ++ def inner(): ++ return (None, True, 7, 1.25, 2j, 'text', b'bytes', ...) ++ return x in {1, 2, 3}, inner ++""" ++ live, reference = _code(source), _code(source) ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert pairs[(id(live), id(reference))] == (live, reference) ++ assert all(type(code) is CodeType for code in immutable.values()) ++ ++ ++def test_source_class_methods_properties_and_dataclass_escape_keep_existing_behavior( ++ tmp_path, monkeypatch ++): ++ source = """class Example: ++ @classmethod ++ def named(cls): ++ return cls.__name__ ++ @staticmethod ++ def value(): ++ return 7 ++ @property ++ def amount(self): ++ return 7 ++""" ++ module = _module(tmp_path, monkeypatch, "memo_methods", source) ++ state = _state() ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ # Match only the pre-existing generated-code escape; it is not broadened. ++ module.generated = FunctionType( ++ _code().replace(co_filename=""), vars(module) ++ ) ++ owner._live_code(module, *state) diff --git a/experiments/us-acs-loaded-code-controls-20260910.json b/experiments/us-acs-loaded-code-controls-20260910.json new file mode 100644 index 000000000..5fec133ec --- /dev/null +++ b/experiments/us-acs-loaded-code-controls-20260910.json @@ -0,0 +1,33 @@ +{ + "accepted": true, + "acs_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "comparison_policy": "Memo only successful immutable CodeType identity pairs within one producer invocation; retain source, alias, globals and closure checks. Historical two-argument catalogue caller preserved.", + "cpu_seconds": 3.121793, + "date": "2026-09-10", + "financial_successor_acceptance": false, + "guard_sha256": "66d02d798933d7e2a677775c4037df9e16e4f5e6f45cd42363b78234daf56448", + "implementation_adopted": false, + "native_acceptance": false, + "peak_rss_bytes": 353026048, + "performance_gain_established": false, + "receipt_sha256": "da8cde54cb9c5b3ed909ed88b7ab12d6ab824148462593be798dfc86048b4bf1", + "release_acceptance": false, + "runtime": "Python 3.14.4; one numerical thread; no country engine, network, child process or native data access.", + "scope": "invented_acs_code_equality_controls", + "source_map_sha256": "3e5545835bf269a165738a6f59cdd9235171044120113bdd5568a030e59cdc66", + "subsequent_comparison": "us-acs-code-memo-comparison-20260910.json", + "test_sha256": "6d2e43acc5aea4533a99d12adfb94db00629c0f01f25b725d427bd47c7f1aa1f", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 4.655691749998368 +} From 640c1db49e6a70d47d58cea194e44df26c0762e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 00:57:30 -0400 Subject: [PATCH 044/255] Preserve exact archived experiment patch bytes --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..324641f92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve exact unified-diff context markers in the archived measured experiment. +experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace From a5a066f90046799fae06d634e5034d5f17b49b72 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 01:46:03 -0400 Subject: [PATCH 045/255] Capture bounded CI spec identities and isolate ACS compilation cost --- .github/workflows/test.yml | 43 +- CLAUDE.md | 6 + .../893-spec-identity-diagnostics.internal.md | 1 + docs/us-launch-review.md | 7 +- .../us-acs-producer-profile-20260910.json | 354 +++++++++ tools/spec_seed_identity_diagnostics.py | 717 ++++++++++++++++++ 6 files changed, 1126 insertions(+), 2 deletions(-) create mode 100644 changelog.d/893-spec-identity-diagnostics.internal.md create mode 100644 experiments/us-acs-producer-profile-20260910.json create mode 100644 tools/spec_seed_identity_diagnostics.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c424c145d..3bbad9075 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -202,7 +202,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14.4"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -215,6 +215,47 @@ jobs: mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + - name: Derive candidate spec and seed identities (diagnostic only) + id: spec_seed_diagnostics + if: ${{ always() && matrix.python-version == '3.14.4' }} + timeout-minutes: 15 + env: + DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} + DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + VECLIB_MAXIMUM_THREADS: "1" + NUMEXPR_NUM_THREADS: "1" + BLIS_NUM_THREADS: "1" + POPULACE_FIT_N_JOBS: "1" + POPULACE_FIT_PREDICT_WORKERS: "1" + run: | + export DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + export DIAG_UV_VERSION="$(uv --version)" + uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ + --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" + - name: Retain bounded candidate identity evidence + if: ${{ always() && matrix.python-version == '3.14.4' && steps.spec_seed_diagnostics.outcome != 'skipped' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 + path: | + ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json + ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json + ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json + if-no-files-found: error + retention-days: 3 + compression-level: 0 + include-hidden-files: false + engine-us: needs: changes if: github.event_name == 'push' || needs.changes.outputs.us == 'true' || needs.changes.outputs.shared == 'true' diff --git a/CLAUDE.md b/CLAUDE.md index ce1b71c3e..0e1e9840c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,6 +39,12 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert the wheel/import boundary and spec digests, and run the suite against installed wheels. +The shared engine lane also runs a bounded source/seed identity diagnostic on +Python 3.14.4 after its normal tests, including when those tests fail. Its six +JSON artifacts retain candidate digests and their complete canonical seed +records for review. They never replace the test assertions or certify coverage; +review the captured source and dependency identities before updating any pins. + `requires_us` and `requires_uk` are registered pytest markers. Mark new tests that need a live PolicyEngine engine with the appropriate marker; the root collection hook skips them when that engine is absent, and the marker also diff --git a/changelog.d/893-spec-identity-diagnostics.internal.md b/changelog.d/893-spec-identity-diagnostics.internal.md new file mode 100644 index 000000000..3eb49f5a0 --- /dev/null +++ b/changelog.d/893-spec-identity-diagnostics.internal.md @@ -0,0 +1 @@ +Capture bounded spec and seed identity evidence after shared CI tests so changed source fingerprints can be reviewed without weakening coverage assertions. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 6a8d047d1..6114ffb05 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -238,7 +238,12 @@ The paired full-fixture run also passed correctness and final checks, but took 319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility work and demonstrated no speed improvement, so its implementation was not adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) -remain recorded. Further profiling is narrowing the original verification cost; +remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer-profile-20260910.json) +now passes without constructing a fixture: the original producer takes 0.656 +profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including +0.122 seconds inside 103 AST parses; those overlapping times must not be added. +A bounded bytecode compilation cache is under source review, with fresh source +reads and every loaded-function check retained. No new optimization is adopted; the remaining successor controls and native financial pilot remain pending. Current CI separately reports stale source-attested spec/seed fingerprints. diff --git a/experiments/us-acs-producer-profile-20260910.json b/experiments/us-acs-producer-profile-20260910.json new file mode 100644 index 000000000..8c76e9386 --- /dev/null +++ b/experiments/us-acs-producer-profile-20260910.json @@ -0,0 +1,354 @@ +{ + "accepted_diagnostic": true, + "cprofile_options": { + "builtins": true, + "subcalls": false + }, + "current_reverified_files": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "date": "2026-09-10", + "decision": "Source-only experiment in bounded bytecode compilation reuse; preserve fresh reads and all loaded-function checks. No optimization adopted or native speedup established.", + "fixture_constructed": false, + "graph_executed": false, + "guard_sha256": "426cd0370e3e94d4d4e022fdcf954b89b373af7992da88e2d21a37e8670ec2a4", + "native_inputs_used": false, + "postcheck_sha256": "3426339877891cc0659118f788487fe54e7659094dcd454e193284cf6a6e60e7", + "producer_completed": true, + "producer_invocations_requested": 1, + "profile_sha256": "6b4508e6e96226746381a9862997c22eebb9880a328d50b24b9c19f9fd0e78a1", + "profiled_elapsed_cpu_seconds": 0.6406650000000003, + "profiled_elapsed_wall_seconds": 0.6561586250318214, + "protocol": "microcosm.acs-original-producer-profile.v1", + "receipt_sha256": "a956298f6bbe11438256ac5ad7183098c89ed889c457666602c68c32d4bc5e12", + "records": [ + { + "calls": 1, + "cumulative_seconds": 0.6560935, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "kind": "frozen_source", + "line": 129, + "recursive_calls": 0, + "self_seconds": 0.005839213 + }, + { + "calls": 1, + "cumulative_seconds": 0.42354929100000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "implementation_manifest", + "kind": "frozen_source", + "line": 411, + "recursive_calls": 0, + "self_seconds": 0.000314576 + }, + { + "calls": 59, + "cumulative_seconds": 0.36231329, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_contract", + "kind": "frozen_source", + "line": 303, + "recursive_calls": 0, + "self_seconds": 0.005696675 + }, + { + "calls": 59, + "cumulative_seconds": 0.355838413, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_details", + "kind": "frozen_source", + "line": 202, + "recursive_calls": 0, + "self_seconds": 0.023903589000000003 + }, + { + "calls": 44, + "cumulative_seconds": 0.212273085, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "kind": "frozen_source", + "line": 51, + "recursive_calls": 0, + "self_seconds": 0.021575394 + }, + { + "calls": 2586, + "cumulative_seconds": 0.128561793, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "kind": "frozen_source", + "line": 86, + "recursive_calls": 66, + "self_seconds": 0.0032278470000000003 + }, + { + "calls": 979, + "cumulative_seconds": 0.103954326, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_FunctionDef", + "kind": "frozen_source", + "line": 243, + "recursive_calls": 344, + "self_seconds": 0.000322374 + }, + { + "calls": 9482, + "cumulative_seconds": 0.052384176000000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Call", + "kind": "frozen_source", + "line": 260, + "recursive_calls": 2364, + "self_seconds": 0.002310004 + }, + { + "calls": 3, + "cumulative_seconds": 0.016132043000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_validate_package_roster", + "kind": "frozen_source", + "line": 332, + "recursive_calls": 0, + "self_seconds": 7.2842e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.014024958, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_producer", + "kind": "frozen_source", + "line": 124, + "recursive_calls": 0, + "self_seconds": 4.2163e-05 + }, + { + "calls": 36445, + "cumulative_seconds": 0.008270251000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Name", + "kind": "frozen_source", + "line": 252, + "recursive_calls": 0, + "self_seconds": 0.005725035000000001 + }, + { + "calls": 16, + "cumulative_seconds": 0.005612417000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_sha", + "kind": "frozen_source", + "line": 65, + "recursive_calls": 0, + "self_seconds": 8.211e-06 + }, + { + "calls": 1, + "cumulative_seconds": 0.004030375, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_package_roots", + "kind": "frozen_source", + "line": 180, + "recursive_calls": 0, + "self_seconds": 9.082e-06 + }, + { + "calls": 59, + "cumulative_seconds": 0.0036707920000000004, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_covered_imports", + "kind": "frozen_source", + "line": 286, + "recursive_calls": 0, + "self_seconds": 0.001535637 + }, + { + "calls": 3277, + "cumulative_seconds": 0.002018209, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "visit", + "kind": "frozen_source", + "line": 97, + "recursive_calls": 3223, + "self_seconds": 0.0012540350000000001 + }, + { + "calls": 1451, + "cumulative_seconds": 0.001515904, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 229, + "recursive_calls": 0, + "self_seconds": 0.000852642 + }, + { + "calls": 181, + "cumulative_seconds": 0.000619031, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_digest", + "kind": "frozen_source", + "line": 176, + "recursive_calls": 0, + "self_seconds": 6.2668e-05 + }, + { + "calls": 118, + "cumulative_seconds": 0.000576835, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_canonical", + "kind": "frozen_source", + "line": 170, + "recursive_calls": 0, + "self_seconds": 7.5121e-05 + }, + { + "calls": 555, + "cumulative_seconds": 0.00047688000000000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 289, + "recursive_calls": 0, + "self_seconds": 0.00047688000000000006 + }, + { + "calls": 1, + "cumulative_seconds": 0.000265083, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_inventory", + "kind": "frozen_source", + "line": 312, + "recursive_calls": 0, + "self_seconds": 4.0999000000000005e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.000205333, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_projections", + "kind": "frozen_source", + "line": 352, + "recursive_calls": 0, + "self_seconds": 4.0830000000000005e-06 + }, + { + "calls": 76, + "cumulative_seconds": 0.00019908300000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "columns", + "kind": "frozen_source", + "line": 355, + "recursive_calls": 74, + "self_seconds": 4.3078e-05 + }, + { + "calls": 3811, + "cumulative_seconds": 0.00012806000000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_require", + "kind": "frozen_source", + "line": 46, + "recursive_calls": 0, + "self_seconds": 0.00012806000000000002 + }, + { + "calls": 555, + "cumulative_seconds": 8.0924e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 290, + "recursive_calls": 0, + "self_seconds": 8.0924e-05 + }, + { + "calls": 303, + "cumulative_seconds": 4.1642000000000006e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 358, + "recursive_calls": 0, + "self_seconds": 3.0757000000000004e-05 + }, + { + "calls": 240, + "cumulative_seconds": 3.2792e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 296, + "recursive_calls": 0, + "self_seconds": 3.2792e-05 + }, + { + "calls": 1, + "cumulative_seconds": 8.400000000000001e-08, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_require", + "kind": "frozen_source", + "line": 60, + "recursive_calls": 0, + "self_seconds": 8.400000000000001e-08 + }, + { + "calls": 157, + "cumulative_seconds": 0.237641084, + "function": "builtins.compile", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 0.237641084 + }, + { + "calls": 1, + "cumulative_seconds": 2.92e-07, + "function": "time.perf_counter", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 2.92e-07 + }, + { + "calls": 1, + "cumulative_seconds": 3e-06, + "function": "time.process_time", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 3e-06 + }, + { + "calls": 103, + "cumulative_seconds": 0.12155241700000001, + "function": "ast.parse", + "kind": "stdlib", + "recursive_calls": 0, + "self_seconds": 4.1292000000000004e-05 + } + ], + "scope": "one_original_producer_invocation_after_normal_helper_imports", + "source_or_population_authority_issued": false, + "successor_or_release_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "normal imports are outside profiling and no fixture is requested", + "one producer call has no within-process repetition or warmup", + "cumulative times overlap and must not be summed", + "ast.parse wraps compile and their cumulative times overlap", + "cProfile adds overhead; this diagnostic cannot establish a speedup", + "source rows cover only the three declared owner/manifest modules", + "other external time remains within cumulative callers", + "compiled-reference visitation and deep code equality remain unmodified" + ], + "whole_guard": { + "cpu_seconds": 4.779278, + "peak_rss_bytes": 481591296, + "wall_seconds": 6.85519720899174 + } +} diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py new file mode 100644 index 000000000..0b53647b2 --- /dev/null +++ b/tools/spec_seed_identity_diagnostics.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +"""Emit bounded candidate spec identities; never replace a coverage assertion.""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import importlib.metadata +import json +import os +import re +import resource +import signal +import stat +import sys +import tempfile +import tomllib +from pathlib import Path + +LOCK_SHA256 = "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b" +UPLOAD_ACTION_SHA = "ea165f8d65b6e75b540449e92b4886f43607fa02" +CAPS = { + "candidate-digests.json": 64 * 1024, + "seed-protocol.json": 256 * 1024, + "seed-map.json": 256 * 1024, + "seed-bindings.json": 128 * 1024, + "environment-and-source.json": 128 * 1024, + "diagnostic-status.json": 16 * 1024, +} +MAX_TOTAL = 1024 * 1024 +THREAD_ENV = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS", + "BLIS_NUM_THREADS", + "POPULACE_FIT_N_JOBS", + "POPULACE_FIT_PREDICT_WORKERS", +) +DISTRIBUTIONS = ( + "jsonschema", + "jsonschema-specifications", + "microcosm-fit", + "numpy", + "policyengine-core", + "policyengine-us", + "pytest", + "pyyaml", + "quantile-forest", + "referencing", + "scikit-learn", + "torch", +) +# Filled from the reviewed source-only roster; no import is used to construct it. +SOURCE_PATHS = ( + "packages/microcosm-build/src/microcosm/build/country_spec.py", + "packages/microcosm-build/src/microcosm/build/frame_sampling.py", + "packages/microcosm-build/src/microcosm/build/source_runtime.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/canonical.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/compiler_ir.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/engine_abi.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/loader.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/model.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/schemas.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py", + "packages/microcosm-build/src/microcosm/build/spec_engine/typed_closure.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/adult_care.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/child_support.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/childcare.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_geography.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/disability_benefits.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/energy_subsidy.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/geography_ladder.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/medicaid_take_up.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/other_health_insurance.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/pregnancy.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_aggregate_records.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_capital_gains_tail.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_source_agi.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/retirement_contributions.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/retirement_distributions.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/scf_auto_loans.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/sipp_tips.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/sipp_vehicles.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/snap_discretionary_exemption.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/snap_state_take_up.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/snap_take_up.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/spine_agreement.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/ssi_disability_criteria.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/ssi_take_up.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/take_up.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/take_up_contract.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/weeks_unemployed.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/wic_claim.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/workers_compensation.py", + "packages/microcosm-build/tests/test_spec_engine_loader.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py", + "packages/microcosm-fit/src/microcosm/fit/qrf.py", + "packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py", + "packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py", +) +SEED_MODULES = ( + "microcosm.build.frame_sampling", + "microcosm.build.source_runtime", + "microcosm.build.us_runtime.acs_transfer", + "microcosm.build.us_runtime.adult_care", + "microcosm.build.us_runtime.child_support", + "microcosm.build.us_runtime.childcare", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.disability_benefits", + "microcosm.build.us_runtime.energy_subsidy", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.housing_inputs", + "microcosm.build.us_runtime.immigration", + "microcosm.build.us_runtime.medicaid_take_up", + "microcosm.build.us_runtime.other_health_insurance", + "microcosm.build.us_runtime.pregnancy", + "microcosm.build.us_runtime.prior_year_income", + "microcosm.build.us_runtime.puf_aggregate_records", + "microcosm.build.us_runtime.puf_capital_gains_tail", + "microcosm.build.us_runtime.puf_qrf_chain", + "microcosm.build.us_runtime.puf_source_agi", + "microcosm.build.us_runtime.puf_support", + "microcosm.build.us_runtime.puma_ladder", + "microcosm.build.us_runtime.retirement_contributions", + "microcosm.build.us_runtime.retirement_distributions", + "microcosm.build.us_runtime.scf_auto_loans", + "microcosm.build.us_runtime.scf_wealth", + "microcosm.build.us_runtime.sipp_financial_assets", + "microcosm.build.us_runtime.sipp_tips", + "microcosm.build.us_runtime.sipp_vehicles", + "microcosm.build.us_runtime.snap_discretionary_exemption", + "microcosm.build.us_runtime.snap_state_take_up", + "microcosm.build.us_runtime.snap_take_up", + "microcosm.build.us_runtime.source_runtime", + "microcosm.build.us_runtime.ssi_disability_criteria", + "microcosm.build.us_runtime.ssi_take_up", + "microcosm.build.us_runtime.take_up", + "microcosm.build.us_runtime.weeks_unemployed", + "microcosm.build.us_runtime.wic_claim", + "microcosm.build.us_runtime.workers_compensation", + "microcosm.calibrate.exact_k", + "microcosm.calibrate.solve", + "microcosm.fit.qrf", +) + + +class RefusalError(RuntimeError): + """A fixed-code refusal without source values or exception locals.""" + + +def require(condition: bool, code: str) -> None: + if not condition: + raise RefusalError(code) + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def encoded(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + "\n" + ).encode() + + +def coordinates() -> dict[str, str]: + names = { + "checkout_sha": "DIAG_CHECKOUT_SHA", + "event_sha": "GITHUB_SHA", + "workflow_sha": "DIAG_WORKFLOW_SHA", + "pr_head_sha": "DIAG_PR_HEAD_SHA", + "pr_base_sha": "DIAG_PR_BASE_SHA", + "event_merge_sha": "DIAG_MERGE_SHA", + } + result = {key: os.environ.get(name, "") for key, name in names.items()} + event = os.environ.get("GITHUB_EVENT_NAME", "") + require(event in {"push", "pull_request"}, "EVENT") + for key, value in result.items(): + optional = event == "push" and key.startswith(("pr_", "event_merge")) + require( + bool(re.fullmatch(r"[0-9a-f]{40}", value)) or (optional and not value), + "COORDINATES", + ) + require(result["checkout_sha"] == result["event_sha"], "CHECKOUT") + for key in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT"): + require(os.environ.get(key, "").isdigit(), "RUN_COORDINATES") + result[key.lower()] = os.environ[key] + result["event"] = event + result["repository"] = os.environ.get("GITHUB_REPOSITORY", "") + require(result["repository"] == "PolicyEngine/microcosm", "REPOSITORY") + return result + + +def install_boundary(root: Path, owned: Path, output: Path) -> list[str]: + refusals: list[str] = [] + blocked_files = ( + ".h5", + ".hdf", + ".hdf5", + ".npz", + ".npy", + ".parquet", + ".csv", + ".gz", + ".zip", + ".sas7bdat", + ".dta", + ".pkl", + ".pickle", + ) + read_roots = ( + root / "packages", + root / "tools", + Path(sys.prefix), + Path(sys.base_prefix), + owned, + output, + Path("/usr/share/zoneinfo"), + ) + read_files = { + root / "uv.lock", + root / ".github/workflows/test.yml", + Path("/etc/os-release"), + Path("/etc/localtime"), + Path("/dev/null"), + Path("/proc/cpuinfo"), + Path("/proc/meminfo"), + Path("/proc/self/maps"), + } + + def refuse(code: str) -> None: + if not refusals: + refusals.append(code) + raise RefusalError(code) + + def operation_path( + value: object, directory_fd: object = None, *, follow: bool = True + ) -> Path: + if isinstance(value, int): + # Linux is required; descriptor operations must identify an owned file. + path = Path(os.readlink(f"/proc/self/fd/{value}")) + else: + if not isinstance(value, (str, bytes)): + refuse("FILESYSTEM_PATH") + path = Path(os.fsdecode(value)) + if not path.is_absolute(): + if directory_fd not in (None, -1): + if not isinstance(directory_fd, int): + refuse("FILESYSTEM_DESCRIPTOR") + path = Path(os.readlink(f"/proc/self/fd/{directory_fd}")) / path + else: + path = Path.cwd() / path + # Unlink/rename operate on a final symlink itself, not its target. + return path.resolve() if follow else path.parent.resolve() / path.name + + def owned_mutation( + value: object, directory_fd: object = None, *, follow: bool = True + ) -> None: + path = operation_path(value, directory_fd, follow=follow) + if not any(path.is_relative_to(p) for p in (owned, output)): + refuse("WRITE_SCOPE") + + def audit(event: str, args: tuple[object, ...]) -> None: + # gethostname reads a local OS label; it opens no network connection. + if ( + (event.startswith("socket.") and event != "socket.gethostname") + or event.startswith("subprocess.") + or event + in { + "os.system", + "os.exec", + "os.fork", + "os.forkpty", + "os.posix_spawn", + "pty.spawn", + } + ): + refuse("NETWORK_OR_CHILD") + if event in {"os.link", "os.symlink"}: + refuse("FILESYSTEM_LINK") + if event == "os.rename": + owned_mutation(args[0], args[2], follow=False) + owned_mutation(args[1], args[3], follow=False) + elif event in {"os.remove", "os.rmdir"}: + owned_mutation(args[0], args[1], follow=False) + elif event in {"os.mkdir", "os.chmod"}: + owned_mutation(args[0], args[2]) + elif event in {"os.chown", "os.utime"}: + owned_mutation(args[0], args[3]) + elif event in {"os.truncate", "os.setxattr", "os.removexattr"}: + owned_mutation(args[0]) + if event == "open" and args: + path = operation_path(args[0]) + name = str(path) + if name.lower().endswith(blocked_files): + refuse("DATA_FILE") + flags = args[2] if len(args) > 2 else 0 + writing = isinstance(flags, int) and bool( + flags + & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_APPEND) + ) + if writing and name != os.devnull: + if not any(path.is_relative_to(p) for p in (owned, output)): + refuse("WRITE_SCOPE") + if ( + not writing + and path not in read_files + and not any(path.is_relative_to(p) for p in read_roots) + ): + refuse("READ_SCOPE") + + sys.addaudithook(audit) + return refusals + + +def source_stamps(root: Path) -> dict[str, str]: + paths = ( + *SOURCE_PATHS, + "tools/spec_seed_identity_diagnostics.py", + "uv.lock", + ".github/workflows/test.yml", + ) + result = {} + for name in paths: + path = root / name + require(path.is_file() and not path.is_symlink(), "SOURCE_PATH") + result[name] = digest(path.read_bytes()) + require(result["uv.lock"] == LOCK_SHA256, "LOCK") + return result + + +def check_loaded_origins(root: Path) -> None: + for name in SOURCE_PATHS: + if "/src/" not in name: + continue + module_name = name.split("/src/", 1)[1].removesuffix(".py").replace("/", ".") + module = sys.modules.get(module_name) + if module is not None: + require( + Path(module.__file__).resolve() == (root / name).resolve(), + "MODULE_ORIGIN", + ) + helper = sys.modules.get("test_spec_engine_loader") + if helper is not None: + require( + Path(helper.__file__).resolve() + == ( + root / "packages/microcosm-build/tests/test_spec_engine_loader.py" + ).resolve(), + "HELPER_ORIGIN", + ) + + +def versions(lock: dict) -> dict[str, str]: + actual = {} + for name in DISTRIBUTIONS: + expected = {p["version"] for p in lock["package"] if p["name"] == name} + require(len(expected) == 1, "LOCK_VERSION") + actual[name] = importlib.metadata.version(name) + require(actual[name] in expected, "INSTALLED_VERSION") + return actual + + +def thread_controls(torch) -> dict[str, object]: + controls = { + "environment": {name: os.environ.get(name) for name in THREAD_ENV}, + "torch_intra_op": torch.get_num_threads(), + "torch_inter_op": torch.get_num_interop_threads(), + } + require( + all(value == "1" for value in controls["environment"].values()) + and controls["torch_intra_op"] == 1 + and controls["torch_inter_op"] == 1, + "THREADS_CHANGED", + ) + return controls + + +def validate_payloads(payloads: dict[str, bytes]) -> None: + require(set(payloads) == set(CAPS), "OUTPUT_ROSTER") + require( + all(len(data) <= CAPS[name] for name, data in payloads.items()), "OUTPUT_CAP" + ) + require(sum(map(len, payloads.values())) <= MAX_TOTAL, "TOTAL_CAP") + + +def publish(output: Path, payloads: dict[str, bytes], *, thread_state=None) -> None: + validate_payloads(payloads) + retained = tuple((name, payloads[name]) for name in CAPS) + status_name = "diagnostic-status.json" + status_value = json.loads(payloads[status_name]) + if status_value.get("completed") is True: + require( + status_value.get("payload_sha256") + == { + name: digest(data) + for name, data in payloads.items() + if name != status_name + }, + "OUTPUT_SEAL", + ) + # A failed prior write can leave only one of these known owned temporaries. + # Remove it before using O_EXCL again; never recursively clean arbitrary paths. + for name in CAPS: + temporary = output / (name + ".part") + if temporary.exists() or temporary.is_symlink(): + info = temporary.lstat() + require( + stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode), "OUTPUT_PART" + ) + temporary.unlink() + for name, data in payloads.items(): + if name == status_name: + continue + temporary = output / (name + ".part") + with temporary.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(output / name) + # Prepare the terminal status, but keep the visible marker incomplete while + # every payload and the prepared status are checked through their actual files. + prepared = output / (status_name + ".part") + with prepared.open("xb") as stream: + stream.write(payloads[status_name]) + stream.flush() + os.fsync(stream.fileno()) + existing_status = output / status_name + expected_names = (set(CAPS) - {status_name}) | {prepared.name} + if existing_status.exists(): + expected_names.add(status_name) + info = existing_status.lstat() + require( + stat.S_ISREG(info.st_mode) and info.st_size <= CAPS[status_name], + "OUTPUT_STATUS", + ) + require( + json.loads(existing_status.read_bytes()).get("completed") is not True, + "OUTPUT_ALREADY_COMPLETE", + ) + require({p.name for p in output.iterdir()} == expected_names, "OUTPUT_FILES") + observed = {} + for name, cap in CAPS.items(): + path = prepared if name == status_name else output / name + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_size <= cap, "OUTPUT_FILE") + observed[name] = path.read_bytes() + require(observed == payloads, "OUTPUT_CHANGED") + validate_payloads(observed) + if thread_state is not None: + torch, controls, refusals = thread_state + require(thread_controls(torch) == controls, "THREADS_CHANGED") + require(not refusals, "BOUNDARY_REFUSAL") + require( + tuple((name, payloads[name]) for name in CAPS) == retained, "OUTPUT_CHANGED" + ) + # Last filesystem operation: success becomes visible only after verification. + # There is deliberately no post-commit read/check/cleanup in this function. + prepared.replace(existing_status) + + +def derive( + root: Path, + owned: Path, + run: dict[str, str], + before: dict[str, str], + installed: dict[str, str], + torch, + controls: dict[str, object], +) -> dict[str, bytes]: + # The exact helper is imported; pytest collection and its tests are never run. + sys.path.insert(0, str(root / "packages/microcosm-build/tests")) + from test_spec_engine_loader import SELECTION_KERNEL_IDS, _rich_minimal + + from microcosm.build.spec_engine import KernelRegistry, load_bundle + from microcosm.build.spec_engine.canonical import canonical_json_bytes, sha256_json + from microcosm.build.spec_engine.compiler_ir import compile_spec + from microcosm.build.spec_engine.seeds import source_inventory_sha256 + + check_loaded_origins(root) + countries = {country: load_bundle(country) for country in ("am", "be", "uk")} + minimal = load_bundle( + _rich_minimal(owned / "xx", note="first", store="local:a"), + kernel_registry=KernelRegistry.from_ids(SELECTION_KERNEL_IDS), + ) + us = load_bundle("us") + compiled = compile_spec(us) + protocol = us.seed_protocol.to_wire() + seed_map = compiled.seed_stream_map.to_wire() + bindings = [binding.to_wire() for binding in us.seed_site_bindings] + modules = tuple( + sorted({m for k in us.seed_protocol.kernels for m in k.source_modules}) + ) + require(modules == SEED_MODULES, "SEED_MODULE_ROSTER") + for kernel in us.seed_protocol.kernels: + require( + source_inventory_sha256(kernel.source_modules) == kernel.source_sha256, + "SEED_SOURCE_CHANGED", + ) + require( + ( + len(protocol["sites"]), + len(protocol["streams"]), + sum(len(s.owners) for s in compiled.seed_stream_map.sites), + len(seed_map["owners"]), + ) + == (53, 14, 112, 54), + "SEED_SHAPE", + ) + require( + compiled.seed_stream_map.implementation_sha256 + == us.seed_protocol.implementation_sha256, + "SEED_HEADER", + ) + candidates = { + "status": "candidate_values_only", + "coverage": "not_asserted_by_diagnostic", + "digests": { + **{country: value.spec_sha256 for country, value in countries.items()}, + "rich_minimal": minimal.spec_sha256, + "seed_protocol": us.seed_protocol.implementation_sha256, + "seed_map": sha256_json(seed_map), + }, + "shape": {"sites": 53, "streams": 14, "bindings": 112, "owners": 54}, + "locations": { + "am_be_uk": "packages/microcosm-build/tests/test_spec_engine_country_bundles.py", + "rich_minimal": "packages/microcosm-build/tests/test_spec_engine_loader.py", + "seed_protocol_seed_map": "packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:EXPECTED_HASHES", + }, + } + environment = { + "coordinates": run, + "python": sys.version, + "platform": sys.platform, + "runner_image": {k: os.environ.get(k, "") for k in ("ImageOS", "ImageVersion")}, + "uv_version": os.environ.get("DIAG_UV_VERSION", ""), + "lock_sha256": LOCK_SHA256, + "distributions": installed, + "thread_controls": controls, + "source_sha256": before, + "upload_action_commit": UPLOAD_ACTION_SHA, + "seed_modules": list(modules), + "compiler_source_inventory": list(compiled.compiler_ir_abi.source_inventory), + "limits": { + "cpu_seconds": 600, + "wall_seconds": 900, + "threads": 1, + "total_output_bytes": MAX_TOTAL, + "file_caps": CAPS, + }, + "boundary": "live_engine_metadata; no_native_data_network_children_or_fits", + } + # Complete canonical retained values precede the final source/metadata reads. + payloads = { + "candidate-digests.json": encoded(candidates), + "seed-protocol.json": canonical_json_bytes(protocol), + "seed-map.json": canonical_json_bytes(seed_map), + "seed-bindings.json": canonical_json_bytes(bindings), + "environment-and-source.json": encoded(environment), + } + lock = tomllib.loads((root / "uv.lock").read_text()) + require(versions(lock) == installed, "VERSION_CHANGED") + for kernel in us.seed_protocol.kernels: + require( + source_inventory_sha256(kernel.source_modules) == kernel.source_sha256, + "SEED_SOURCE_CHANGED", + ) + check_loaded_origins(root) + require(source_stamps(root) == before, "SOURCE_CHANGED") + require(thread_controls(torch) == controls, "THREADS_CHANGED") + payloads["diagnostic-status.json"] = encoded( + { + "completed": True, + "status": "candidate_values_only", + "coverage_pass": False, + "payload_sha256": {name: digest(data) for name, data in payloads.items()}, + } + ) + return payloads + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + output = args.output_dir.absolute() + runner_temp = Path(os.environ["RUNNER_TEMP"]).resolve() + require(output == runner_temp / "spec-seed-diagnostics", "OUTPUT_PATH") + require(not output.exists() and not output.is_symlink(), "OUTPUT_EXISTS") + output.mkdir(mode=0o700) + phase = "environment" + with tempfile.TemporaryDirectory( + prefix="spec-seed-owned-", dir=runner_temp + ) as temp: + owned = Path(temp) + require(sys.version_info[:3] == (3, 14, 4), "PYTHON") + require(sys.platform == "linux", "PLATFORM") + require(all(os.environ.get(name) == "1" for name in THREAD_ENV), "THREADS") + sys.dont_write_bytecode = True + resource.setrlimit(resource.RLIMIT_CPU, (600, 600)) + signal.signal( + signal.SIGALRM, lambda *_: (_ for _ in ()).throw(RefusalError("WALL")) + ) + signal.alarm(900) + refusals = install_boundary(root, owned, output) + # Dependency-created temporaries must stay inside this invocation's scope. + tempfile.tempdir = str(owned) + os.environ["TMPDIR"] = str(owned) + initial = {name: encoded({"status": "incomplete"}) for name in CAPS} + initial["diagnostic-status.json"] = encoded( + { + "completed": False, + "status": "in_progress", + "coverage_pass": False, + } + ) + publish(output, initial) + try: + run = coordinates() + before = source_stamps(root) + lock = tomllib.loads((root / "uv.lock").read_text()) + installed = versions(lock) + phase = "derive" + with ( + open(os.devnull, "w") as quiet, + contextlib.redirect_stdout(quiet), + contextlib.redirect_stderr(quiet), + ): + # Import/configure the locked numerical dependency within the same + # audit boundary, before any maintained project module is loaded. + import torch + + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + controls = thread_controls(torch) + payloads = derive(root, owned, run, before, installed, torch, controls) + require(not refusals, "BOUNDARY_REFUSAL") + exit_code = 0 + except Exception as error: + code = str(error) if type(error) is RefusalError else "DERIVATION_REFUSED" + payloads = {name: encoded({"status": "incomplete"}) for name in CAPS} + payloads["diagnostic-status.json"] = encoded( + { + "completed": False, + "status": "refused", + "phase": phase, + "code": code, + "coverage_pass": False, + } + ) + exit_code = 1 + retained = tuple((name, payloads[name]) for name in CAPS) + # Fixture cleanup has finished. No borrowed source/artifact I/O follows the + # terminal publication; a killed cleanup therefore leaves incomplete status. + require( + tuple((name, payloads[name]) for name in CAPS) == retained, "OUTPUT_CHANGED" + ) + try: + if exit_code == 0: + require(thread_controls(torch) == controls, "THREADS_CHANGED") + require(not refusals, "BOUNDARY_REFUSAL") + publish( + output, + dict(retained), + thread_state=(torch, controls, refusals) if exit_code == 0 else None, + ) + except Exception: + refused = {name: encoded({"status": "incomplete"}) for name in CAPS} + refused["diagnostic-status.json"] = encoded( + { + "completed": False, + "status": "refused", + "phase": "publish", + "code": "OUTPUT_REFUSED", + "coverage_pass": False, + } + ) + publish(output, refused) + signal.alarm(0) + return 1 + signal.alarm(0) + return exit_code + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception: + # Bootstrap failures have no locals, paths or exception text in CI logs. + print("spec-seed diagnostics: bootstrap refused", file=sys.stderr) + raise SystemExit(1) from None From bcc60e70c189240a9f2232b423236bb64e422670 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 02:23:42 -0400 Subject: [PATCH 046/255] Verify the financial successor positive control and retain source citations --- ...893-historical-zero-provenance.internal.md | 1 + docs/us-launch-review.md | 17 +++++--- ...financial-successor-positive-20260910.json | 41 +++++++++++++++++++ .../microcosm-build/tests/test_us_plan.py | 17 ++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 changelog.d/893-historical-zero-provenance.internal.md create mode 100644 experiments/us-financial-successor-positive-20260910.json diff --git a/changelog.d/893-historical-zero-provenance.internal.md b/changelog.d/893-historical-zero-provenance.internal.md new file mode 100644 index 000000000..5c7d83751 --- /dev/null +++ b/changelog.d/893-historical-zero-provenance.internal.md @@ -0,0 +1 @@ +Keep the exact historical ASEC zero-origin citations in the dependency scan while continuing to reject other incumbent data-package references in that module. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 6114ffb05..65893aa0c 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -220,11 +220,18 @@ receipts without reopening source bodies. This clears the national support normalization check; assigning households, evaluating geographic fit and calibrating the enriched survey remain separate acceptance steps. -The financial successor's positive control reached its 600 CPU-second limit -without a final receipt. A separate, bounded profile of the unchanged invented -cold/required financial fixture now passes its test, original teardown and all -final code/resource checks: 303.90 CPU seconds, 308.26 wall seconds and 594.3 MB -peak RSS. This diagnoses the fixture; it does not accept the financial successor. +The financial successor's unchanged positive control now passes with both +original fixture teardowns and final code/resource checks: 647.19 CPU seconds, +651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation +used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded +as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) +checks financial replay and a subsequent weight-only change. Seven refusal and +callback controls remain before accepting the complete admission path. + +A separate, bounded profile of the unchanged invented cold/required financial +fixture passes its test, original teardown and all final code/resource checks: +303.90 CPU seconds, 308.26 wall seconds and 594.3 MB peak RSS. This measures the +fixture rather than the additional financial and weight-only admission work. See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). Loaded ACS code verification accounts for 180.20 cumulative seconds inside the 298.15-second profiled fixture. Its nested function checker runs 2,009,108 times diff --git a/experiments/us-financial-successor-positive-20260910.json b/experiments/us-financial-successor-positive-20260910.json new file mode 100644 index 000000000..fa1a8d651 --- /dev/null +++ b/experiments/us-financial-successor-positive-20260910.json @@ -0,0 +1,41 @@ +{ + "all_successor_controls_passed": false, + "approval_sha256": "91efa2d737f38d6d6b11890241a1a9b5dbb1c6840da3a49f58966101e3f967ef", + "cpu_seconds": 647.185773, + "fixture_original_teardowns_passed": true, + "guard_sha256": "a9566eb1783d71dd1fc3c0468e2f1af763fbd5ed218f0a87d2cbca3ae1d4011b", + "interpretation": "The original financial replay and financial-to-weight-only successor behavior passes; remaining refusal/callback cases and native calibration admission are pending.", + "junit_sha256": "2af08e54f972192d299281c992dd4306dde99399699c4684037ad6bf60804d9a", + "limits": { + "cpu_seconds": 1200, + "numerical_threads": 1, + "wall_seconds": 1800 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_acs_owner_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "peak_rss_bytes": 570884096, + "postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "receipt_sha256": "96819afd6805e5cbcd6dfb2271ec1d8c4c3b72775135904a9efe1d97cbb79791", + "release_acceptance": false, + "remaining_controls": 7, + "resource_map_sha256": "088a55cd36f83e4b466947a216b0315a983d5888c603103e575f0cc7ae2ece38", + "schema_version": 1, + "scope": "one_invented_financial_successor_positive_control", + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "one_positive_control_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 651.2133005840005 +} diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index b2c7c8cff..e9688c654 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -1055,6 +1055,23 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: if rel in allowed_incumbent_references: continue text = path.read_text(encoding="utf-8") + if rel == ( + "packages/microcosm-build/src/microcosm/build/us_runtime/" + "asec_housing_status.py" + ): + # These two exact fields cite the inspected historical source + # of a frozen checkpoint's unresolved zero values. Preserve + # that evidence while still scanning the rest of this module + # for an incumbent import, dependency or other reference. + historical_fields = ( + ' "inspected_repository": "PolicyEngine/policyengine-' + 'us-data",\n', + ' "inspected_path": "policyengine_' + 'us_data/datasets/cps/census_cps.py",\n', + ) + for field in historical_fields: + assert text.count(field) == 1 + text = text.replace(field, "", 1) for needle in forbidden: if needle in text: offenders.append((rel, needle)) From 3bc7c8be65ab5dd8a6c9a4693ad773621f043a51 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 02:52:56 -0400 Subject: [PATCH 047/255] Repair age-count fixtures and run bounded identity diagnostics in parallel --- .github/workflows/test.yml | 22 +++- changelog.d/893-age-count-fixture.internal.md | 1 + ...893-diagnostic-refusal-context.internal.md | 1 + ...st_spec_seed_diagnostic_refusal_context.py | 82 ++++++++++++ .../tests/test_us_national_age_counts.py | 9 +- tools/spec_seed_identity_diagnostics.py | 122 +++++++++++++++++- 6 files changed, 225 insertions(+), 12 deletions(-) create mode 100644 changelog.d/893-age-count-fixture.internal.md create mode 100644 changelog.d/893-diagnostic-refusal-context.internal.md create mode 100644 packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3bbad9075..951e8fabd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -215,9 +215,19 @@ jobs: mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + spec-seed-diagnostics: + # Produce bounded diagnosis immediately, independently of the full suite. + # This remains required by ci-ok and never updates assertion goldens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14.4" + - name: Sync workspace with US and UK engines + run: uv sync --all-packages --locked --extra us --extra uk - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics - if: ${{ always() && matrix.python-version == '3.14.4' }} timeout-minutes: 15 env: DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -235,12 +245,13 @@ jobs: POPULACE_FIT_N_JOBS: "1" POPULACE_FIT_PREDICT_WORKERS: "1" run: | - export DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" - export DIAG_UV_VERSION="$(uv --version)" + DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + DIAG_UV_VERSION="$(uv --version)" + export DIAG_CHECKOUT_SHA DIAG_UV_VERSION uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" - name: Retain bounded candidate identity evidence - if: ${{ always() && matrix.python-version == '3.14.4' && steps.spec_seed_diagnostics.outcome != 'skipped' }} + if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 @@ -386,7 +397,7 @@ jobs: # dependency-skip lets a failed `changes` take the engine lanes down without # any required check going red: a lane skip is accepted only when # classification itself succeeded. - needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels] + needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels] if: always() runs-on: ubuntu-latest steps: @@ -401,6 +412,7 @@ jobs: require_success lint "${{ needs.lint.result }}" require_success fast "${{ needs.fast.result }}" require_success engine-shared "${{ needs['engine-shared'].result }}" + require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}" require_success wheels "${{ needs.wheels.result }}" for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do name="${lane%%:*}"; result="${lane#*:}" diff --git a/changelog.d/893-age-count-fixture.internal.md b/changelog.d/893-age-count-fixture.internal.md new file mode 100644 index 000000000..1e691423f --- /dev/null +++ b/changelog.d/893-age-count-fixture.internal.md @@ -0,0 +1 @@ +Use the maintained Frame store API in the national age-count cold and fresh-process replay tests after removal of the transfer example module. diff --git a/changelog.d/893-diagnostic-refusal-context.internal.md b/changelog.d/893-diagnostic-refusal-context.internal.md new file mode 100644 index 000000000..c6e330a2f --- /dev/null +++ b/changelog.d/893-diagnostic-refusal-context.internal.md @@ -0,0 +1 @@ +Record bounded code-path context for refused CI spec diagnostics while preserving the existing read restrictions and incomplete result status. diff --git a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py new file mode 100644 index 000000000..9a4500164 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py @@ -0,0 +1,82 @@ +"""Source-only context formatting: no audit install or denied file operation.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + root = Path(__file__).resolve().parents[3] + spec = importlib.util.spec_from_file_location( + "spec_seed_diagnostic_context_test", + root / "tools/spec_seed_identity_diagnostics.py", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("value", "scope", "shown"), + ( + ("/repo/tools/example.py", "repository", "tools/example.py"), + ( + "/repo/packages/example/schema.json", + "repository", + "packages/example/schema.json", + ), + ("/repo/data/invented.csv", "repository", None), + ("/repo/.env", "repository", None), + ("/repo/credentials.json", "repository", None), + ("/repo/example.err.log", "repository", None), + ("/unlisted/identity/example.py", "unlisted", "example.py"), + ("/opt/python/lib/example.py", "system_opt", "python/lib/example.py"), + ), +) +def test_path_classification_is_bounded_and_redacts_data_and_credentials( + diagnostic, value, scope, shown +): + result = diagnostic.refusal_path_context( + value, (("repository", Path("/repo")), ("system_opt", Path("/opt"))) + ) + assert result["scope"] == scope + assert result["path"] == shown + assert result["name_redacted"] is (shown is None) + + +def test_requested_and_resolved_code_scopes_remain_distinct_without_io(diagnostic): + # Literal examples only: no symlink or path is created, resolved or opened. + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/opt/python/lib/loader.py"), + "/environment/lib/loader.py", + (("environment", Path("/environment")), ("system_opt", Path("/opt"))), + ) + value = json.loads(payload) + assert value["requested"]["scope"] == "environment" + assert value["resolved"]["scope"] == "system_opt" + assert value["paths_are_metadata_only"] is True + assert len(payload) <= 4096 and len(value["frames"]) <= 8 + + +def test_context_never_serializes_locals_or_exception_text(diagnostic): + private_local = "invented-secret-value-that-must-never-be-exported" + error = RuntimeError(private_local) + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/repo/credentials.json"), + "/repo/credentials.json", + (("repository", Path("/repo")),), + ) + assert str(error).encode() not in payload + value = json.loads(payload) + assert value["requested"]["path"] is None + assert value["resolved"]["path"] is None + assert value["code"] == "READ_SCOPE" and value["event"] == "open" + assert all(set(frame) == {"file", "function", "line"} for frame in value["frames"]) + assert len(payload) <= 4096 diff --git a/packages/microcosm-build/tests/test_us_national_age_counts.py b/packages/microcosm-build/tests/test_us_national_age_counts.py index bff56b6e8..527de13b8 100644 --- a/packages/microcosm-build/tests/test_us_national_age_counts.py +++ b/packages/microcosm-build/tests/test_us_national_age_counts.py @@ -795,13 +795,16 @@ def run(self, context): def graph_fixture(output, warm=False): """Cold and warm run the same on-disk fixture and real production kernels.""" - from microcosm.build.transfer_example import _source_frame_path - output = Path(output) store = ContentStore(output / "store") if not warm: frame = fixture_frame(kind=WeightKind.DESIGN, scale=2.0) - path = _source_frame_path(store, frame) + # This key is local to the fresh invented fixture store. Warm replay + # consumes the same content-verified object through its persisted path. + source_key = hashlib.sha256( + b"test.us-national-age-counts.source.v1" + ).hexdigest() + path = store.put_frame(source_key, frame) (output / "source-path.txt").write_text(str(path)) path = Path((output / "source-path.txt").read_text()) count, calibration = national_age_calibration_nodes( diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 0b53647b2..d57bd4edd 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -216,7 +216,91 @@ def coordinates() -> dict[str, str]: return result -def install_boundary(root: Path, owned: Path, output: Path) -> list[str]: +def refusal_path_context(value: object, roots: tuple[tuple[str, Path], ...]) -> dict: + """Classify an already supplied path without resolving, opening or statting it. + + Only bounded source-like paths under named code/system anchors are shown. + Other names, data files and credential/log-like paths remain redacted. + This function changes neither path resolution nor permission decisions. + """ + if not isinstance(value, (str, bytes, Path)): + return {"scope": "descriptor_or_nonpath", "path": None} + path = Path(os.fsdecode(value)) + scope, relative = "unlisted", None + if not path.is_absolute(): + scope = "relative" + else: + for label, anchor in roots: + if path.is_relative_to(anchor): + scope, relative = label, path.relative_to(anchor).as_posix() + break + source_suffixes = { + ".py", + ".pyc", + ".so", + ".dylib", + ".json", + ".toml", + ".yaml", + ".yml", + } + suffix = path.suffix.lower() + text = relative if relative is not None else path.name + private_name = re.search( + r"(^|[/_.-])(secret|credentials?|password|tokens?|private|keys?|b19001|b25003|occupied)([/_.-]|$)", + text.lower(), + ) + shown = ( + text + if suffix in source_suffixes + and not private_name + and re.fullmatch(r"[A-Za-z0-9_./+@-]{1,240}", text) + and all(part not in ("", ".", "..") for part in Path(text).parts) + else None + ) + return { + "scope": scope, + "path": shown, + "path_form": "anchor_relative" if relative is not None else "basename_only", + "source_like": suffix in source_suffixes, + "name_redacted": shown is None, + } + + +def read_refusal_context(code, event, path, requested, roots) -> bytes: + """Bounded code coordinates only: no locals, source lines or exception text.""" + frames = [] + frame = sys._getframe(1) + visited = 0 + while frame is not None and visited < 32 and len(frames) < 8: + visited += 1 + filename = refusal_path_context(frame.f_code.co_filename, roots) + name = frame.f_code.co_name + if filename.get("path") is not None and ( + re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", name) + or name in {"", "", "", "", ""} + ): + frames.append({"file": filename, "function": name, "line": frame.f_lineno}) + frame = frame.f_back + result = { + "code": code, + "event": event, + "requested": refusal_path_context(requested, roots), + "resolved": refusal_path_context(path, roots), + "frames": frames, + "stack_truncated": frame is not None, + "paths_are_metadata_only": True, + } + del frame + payload = encoded(result) + if len(payload) > 4096: + payload = encoded({"code": code, "event": event, "context": "size_bound"}) + return payload + + +def install_boundary( + root: Path, owned: Path, output: Path, *, first_refusal: list[bytes] | None = None +) -> list[str]: refusals: list[str] = [] blocked_files = ( ".h5", @@ -253,9 +337,32 @@ def install_boundary(root: Path, owned: Path, output: Path) -> list[str]: Path("/proc/self/maps"), } - def refuse(code: str) -> None: + context_roots = ( + ("repository", root), + ("owned", owned), + ("output", output), + ("environment", Path(sys.prefix)), + ("base_environment", Path(sys.base_prefix)), + ("system_usr", Path("/usr")), + ("system_opt", Path("/opt")), + ("system_lib", Path("/lib")), + ("system_etc", Path("/etc")), + ) + + def refuse(code: str, *, event=None, path=None, requested=None) -> None: if not refusals: refusals.append(code) + if first_refusal is not None: + try: + first_refusal.append( + read_refusal_context( + code, event, path, requested, context_roots + ) + ) + except Exception: + first_refusal.append( + encoded({"code": code, "context": "unavailable"}) + ) raise RefusalError(code) def operation_path( @@ -332,7 +439,7 @@ def audit(event: str, args: tuple[object, ...]) -> None: and path not in read_files and not any(path.is_relative_to(p) for p in read_roots) ): - refuse("READ_SCOPE") + refuse("READ_SCOPE", event="open", path=path, requested=args[0]) sys.addaudithook(audit) return refusals @@ -628,7 +735,8 @@ def main() -> int: signal.SIGALRM, lambda *_: (_ for _ in ()).throw(RefusalError("WALL")) ) signal.alarm(900) - refusals = install_boundary(root, owned, output) + first_refusal: list[bytes] = [] + refusals = install_boundary(root, owned, output, first_refusal=first_refusal) # Dependency-created temporaries must stay inside this invocation's scope. tempfile.tempdir = str(owned) os.environ["TMPDIR"] = str(owned) @@ -671,6 +779,9 @@ def main() -> int: "status": "refused", "phase": phase, "code": code, + "first_boundary_refusal": ( + json.loads(first_refusal[0]) if first_refusal else None + ), "coverage_pass": False, } ) @@ -698,6 +809,9 @@ def main() -> int: "status": "refused", "phase": "publish", "code": "OUTPUT_REFUSED", + "first_boundary_refusal": ( + json.loads(first_refusal[0]) if first_refusal else None + ), "coverage_pass": False, } ) From 28d3b246e7fc64985da23dcbd9cfb3adfde0e775 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 02:59:22 -0400 Subject: [PATCH 048/255] Record original successor control acceptance and the remaining callback finding --- docs/us-launch-review.md | 18 ++++-- ...successor-remaining-controls-20260910.json | 61 +++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 experiments/us-financial-successor-remaining-controls-20260910.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 65893aa0c..fcf199077 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -225,8 +225,13 @@ original fixture teardowns and final code/resource checks: 647.19 CPU seconds, 651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) -checks financial replay and a subsequent weight-only change. Seven refusal and -callback controls remain before accepting the complete admission path. +checks financial replay and a subsequent weight-only change. The [seven remaining refusal and callback controls](../experiments/us-financial-successor-remaining-controls-20260910.json) +now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, +802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls +are accepted on the same source snapshot. A subsequent source review found an +additional detached-view callback gap in the budget and weight-only views; +targeted corrections and regression controls are in progress. Native calibration +remains separate. A separate, bounded profile of the unchanged invented cold/required financial fixture passes its test, original teardown and all final code/resource checks: @@ -249,9 +254,12 @@ remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer now passes without constructing a fixture: the original producer takes 0.656 profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including 0.122 seconds inside 103 AST parses; those overlapping times must not be added. -A bounded bytecode compilation cache is under source review, with fresh source -reads and every loaded-function check retained. No new optimization is adopted; -the remaining successor controls and native financial pilot remain pending. +A bounded bytecode compilation cache now passes 25 invented controls and final +code/resource checks, with fresh source reads and every loaded-function check +retained. Its full-fixture performance comparison remains pending; no new +optimization is adopted. The native financial cold pilot is running with the +original financial code after a narrow test-harness null-sink ordering fix; +its earlier startup refusal remains preserved and confers no data acceptance. Current CI separately reports stale source-attested spec/seed fingerprints. The worker resource identity test now follows the actual lazy import closure diff --git a/experiments/us-financial-successor-remaining-controls-20260910.json b/experiments/us-financial-successor-remaining-controls-20260910.json new file mode 100644 index 000000000..3180e591b --- /dev/null +++ b/experiments/us-financial-successor-remaining-controls-20260910.json @@ -0,0 +1,61 @@ +{ + "all_successor_controls_passed": true, + "approval_sha256": "9e4ed7c7924b130212a4580fa4ee4298bf7ea33d2ae3e5f2e9798226fb07baee", + "cpu_seconds": 798.6069409999999, + "guard_sha256": "51b3a5adad3c14e11a2f69b8566702abe5aecfffbc1c315dd6f08c412c5e671e", + "interpretation": "Together with the accepted positive control, all eight original financial successor cases pass. Native artifact replay, calibration admission in the completed graph and release verification remain separate.", + "junit_sha256": "6cffe5cac9baca685a7c2c0b032580edf83059bd653d4695b08aab75172d7a8c", + "limits": { + "cpu_seconds": 2400, + "native_threads": 1, + "qrf_fit_n_jobs": "1", + "qrf_predict_workers": "1", + "rss_monitor": false, + "rss_reporting": "final peak only", + "wall_seconds": 3600 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_and_teardowns": true, + "peak_rss_bytes": 563593216, + "peer_source_review_sha256": "12a541b3bcf620f0d931815f40f6b9fffac36e59064222c071c8128b9bdbab7d", + "positive_evidence": "us-financial-successor-positive-20260910.json", + "postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486", + "receipt_sha256": "e394516086ca0f4c773764027ad2d7270a8a9d82d038873f0640cd49e3db483d", + "release_acceptance": false, + "resource_map_sha256": "63de09c280f733c91d07e674bf753c3017b7a375901623a2200a8538f835a4a2", + "root_source_review_sha256": "db0d7f3424cd85bcbc58c690d375d0ef58c557e8d4fefad3dddc666101143c12", + "schema_version": 1, + "scope": "seven_unchanged_invented_financial_successor_refusal_and_callback_controls", + "selected_test_cases": [ + "test_us_survey_financial_successor.py::test_unissued_copy_and_wrong_original_budget_refuse", + "test_us_survey_financial_successor.py::test_arbitrary_nonweight_changes_still_refuse", + "test_us_survey_financial_successor.py::test_rehashed_graph_artifact_cannot_replace_issued_draw", + "test_us_survey_financial_successor.py::test_final_budget_support_borrow_cannot_mutate_financial_run", + "test_us_survey_financial_successor.py::test_final_support_borrow_cannot_forge_returned_view", + "test_us_survey_financial_successor.py::test_financial_borrow_cannot_relax_detached_budget_bounds", + "test_us_survey_financial_successor.py::test_run_borrow_cannot_forge_original_budget_digest" + ], + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "seven_remaining_controls_accepted", + "subsequent_source_review": { + "covered_by_these_eight_controls": false, + "finding": "Budget and weight-only checked views were constructed before the last support I/O, exposing returned detached values to callbacks.", + "status": "additional_gap_under_correction" + }, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 7 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 802.8923444160027 +} From 886eea7f59b1d784584c4813f1bead4bb4f0208a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:05:51 -0400 Subject: [PATCH 049/255] Distinguish terminal diagnostic refusal from optional import probes --- ...st_spec_seed_diagnostic_refusal_context.py | 48 +++++++++++ tools/spec_seed_identity_diagnostics.py | 85 +++++++++++++++---- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py index 9a4500164..16e62dd45 100644 --- a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py +++ b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py @@ -2,6 +2,7 @@ import importlib.util import json +import sys from pathlib import Path import pytest @@ -80,3 +81,50 @@ def test_context_never_serializes_locals_or_exception_text(diagnostic): assert value["code"] == "READ_SCOPE" and value["event"] == "open" assert all(set(frame) == {"file", "function", "line"} for frame in value["frames"]) assert len(payload) <= 4096 + + +@pytest.mark.parametrize( + ("path", "label"), + ( + ("/etc/os-release", "etc_os_release"), + ("/usr/lib/os-release", "usr_lib_os_release"), + ("/lib/os-release", "lib_os_release"), + ("/etc/localtime", "etc_localtime"), + ( + f"/runtime/lib/python{sys.version_info.major}{sys.version_info.minor}.zip", + "python_stdlib_zip_candidate", + ), + ("/runtime/lib/invented-data.zip", None), + ), +) +def test_fixed_os_metadata_labels_do_not_expose_data_path_text(diagnostic, path, label): + value = diagnostic.refusal_path_context(path, (("runtime", Path("/runtime")),)) + assert value["known_path_label"] == label + assert value["path"] is None and value["name_redacted"] is True + + +def test_terminal_refusal_retains_own_context_after_an_earlier_probe(diagnostic): + first = diagnostic.encoded({"code": "DATA_FILE", "event": "open"}) + last = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/usr/lib/os-release"), + "/etc/os-release", + (("system_usr", Path("/usr")), ("system_etc", Path("/etc"))), + ) + swallowed = diagnostic.RefusalError("DATA_FILE", boundary_context=first) + terminal = diagnostic.RefusalError("READ_SCOPE", boundary_context=last) + assert str(swallowed) == "DATA_FILE" and str(terminal) == "READ_SCOPE" + assert type(terminal.boundary_context) is bytes + assert terminal.boundary_context == last and swallowed.boundary_context == first + context = json.loads(terminal.boundary_context) + assert context["requested"]["known_path_label"] == "etc_os_release" + assert context["resolved"]["known_path_label"] == "usr_lib_os_release" + assert context["code"] != json.loads(swallowed.boundary_context)["code"] + assert diagnostic.RefusalError("WALL").boundary_context is None + + +@pytest.mark.parametrize("value", (bytearray(b"{}"), b"x" * 4097)) +def test_exception_context_requires_immutable_bounded_bytes(diagnostic, value): + with pytest.raises(TypeError, match="REFUSAL_CONTEXT"): + diagnostic.RefusalError("READ_SCOPE", boundary_context=value) diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index d57bd4edd..d741f71ae 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -163,7 +163,15 @@ class RefusalError(RuntimeError): - """A fixed-code refusal without source values or exception locals.""" + """A fixed-code refusal with optional immutable, bounded code metadata.""" + + def __init__(self, code: str, *, boundary_context: bytes | None = None): + if boundary_context is not None and ( + type(boundary_context) is not bytes or len(boundary_context) > 4096 + ): + raise TypeError("REFUSAL_CONTEXT") + super().__init__(code) + self.boundary_context = boundary_context def require(condition: bool, code: str) -> None: @@ -258,9 +266,20 @@ def refusal_path_context(value: object, roots: tuple[tuple[str, Path], ...]) -> and all(part not in ("", ".", "..") for part in Path(text).parts) else None ) + # Fixed labels identify these reviewed OS metadata candidates without + # showing arbitrary data/archive filenames or changing read permissions. + known_path_label = { + "/etc/os-release": "etc_os_release", + "/usr/lib/os-release": "usr_lib_os_release", + "/lib/os-release": "lib_os_release", + "/etc/localtime": "etc_localtime", + }.get(path.as_posix()) + if path.name == f"python{sys.version_info.major}{sys.version_info.minor}.zip": + known_path_label = "python_stdlib_zip_candidate" return { "scope": scope, "path": shown, + "known_path_label": known_path_label, "path_form": "anchor_relative" if relative is not None else "basename_only", "source_like": suffix in source_suffixes, "name_redacted": shown is None, @@ -299,7 +318,12 @@ def read_refusal_context(code, event, path, requested, roots) -> bytes: def install_boundary( - root: Path, owned: Path, output: Path, *, first_refusal: list[bytes] | None = None + root: Path, + owned: Path, + output: Path, + *, + first_refusal: list[bytes] | None = None, + distinct_refusals: list[str] | None = None, ) -> list[str]: refusals: list[str] = [] blocked_files = ( @@ -350,20 +374,24 @@ def install_boundary( ) def refuse(code: str, *, event=None, path=None, requested=None) -> None: + # Preserve the historical first-code refusal counter and decisions. if not refusals: refusals.append(code) - if first_refusal is not None: - try: - first_refusal.append( - read_refusal_context( - code, event, path, requested, context_roots - ) - ) - except Exception: - first_refusal.append( - encoded({"code": code, "context": "unavailable"}) - ) - raise RefusalError(code) + if ( + distinct_refusals is not None + and code not in distinct_refusals + and len(distinct_refusals) < 8 + ): + distinct_refusals.append(code) + try: + context = read_refusal_context(code, event, path, requested, context_roots) + except Exception: + context = encoded({"code": code, "context": "unavailable"}) + if first_refusal is not None and not first_refusal: + first_refusal.append(context) + # An optional import probe can swallow an earlier refusal. Attach the + # actual terminal denial's own bytes instead of reporting that probe. + raise RefusalError(code, boundary_context=context) def operation_path( value: object, directory_fd: object = None, *, follow: bool = True @@ -425,7 +453,7 @@ def audit(event: str, args: tuple[object, ...]) -> None: path = operation_path(args[0]) name = str(path) if name.lower().endswith(blocked_files): - refuse("DATA_FILE") + refuse("DATA_FILE", event="open", path=path, requested=args[0]) flags = args[2] if len(args) > 2 else 0 writing = isinstance(flags, int) and bool( flags @@ -736,7 +764,14 @@ def main() -> int: ) signal.alarm(900) first_refusal: list[bytes] = [] - refusals = install_boundary(root, owned, output, first_refusal=first_refusal) + distinct_refusals: list[str] = [] + refusals = install_boundary( + root, + owned, + output, + first_refusal=first_refusal, + distinct_refusals=distinct_refusals, + ) # Dependency-created temporaries must stay inside this invocation's scope. tempfile.tempdir = str(owned) os.environ["TMPDIR"] = str(owned) @@ -782,6 +817,14 @@ def main() -> int: "first_boundary_refusal": ( json.loads(first_refusal[0]) if first_refusal else None ), + "terminal_boundary_refusal": ( + json.loads(error.boundary_context) + if type(error) is RefusalError + and error.boundary_context is not None + else None + ), + "boundary_refusal_codes": list(distinct_refusals), + "boundary_refusal_code_limit": 8, "coverage_pass": False, } ) @@ -801,7 +844,7 @@ def main() -> int: dict(retained), thread_state=(torch, controls, refusals) if exit_code == 0 else None, ) - except Exception: + except Exception as error: refused = {name: encoded({"status": "incomplete"}) for name in CAPS} refused["diagnostic-status.json"] = encoded( { @@ -812,6 +855,14 @@ def main() -> int: "first_boundary_refusal": ( json.loads(first_refusal[0]) if first_refusal else None ), + "terminal_boundary_refusal": ( + json.loads(error.boundary_context) + if type(error) is RefusalError + and error.boundary_context is not None + else None + ), + "boundary_refusal_codes": list(distinct_refusals), + "boundary_refusal_code_limit": 8, "coverage_pass": False, } ) From c5a1db98bd1d21f641ce60baa1501ed9be0fd49e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:19:09 -0400 Subject: [PATCH 050/255] Reuse source compilation after paired fixture and mutation checks --- ...s-compilation-cache-adoption-20260910.json | 75 +++ .../us_runtime/acs_native_coverage_binding.py | 107 +++- .../tests/test_us_acs_source_compile_cache.py | 457 ++++++++++++++++++ 3 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 experiments/us-acs-compilation-cache-adoption-20260910.json create mode 100644 packages/microcosm-build/tests/test_us_acs_source_compile_cache.py diff --git a/experiments/us-acs-compilation-cache-adoption-20260910.json b/experiments/us-acs-compilation-cache-adoption-20260910.json new file mode 100644 index 000000000..157e4596a --- /dev/null +++ b/experiments/us-acs-compilation-cache-adoption-20260910.json @@ -0,0 +1,75 @@ +{ + "control_postcheck_sha256": "58f10a538ad6db8efa30a4029a6eefaa4b0451c1bbb325e21d13a4bd2cc39cd7", + "control_receipt_sha256": "53b578332356b5e5700e2d1a6c40b2b1ccab47440aa886e53db9f19746c8c074", + "control_tests": 25, + "date": "2026-09-10", + "decision": "Adopt bounded process-local reuse of compiled bytecode keyed by complete compilation inputs. Retain every fresh source read, AST check, and loaded function, alias, global and closure check.", + "limitations": [ + "One paired cProfile observation, not a generalized benchmark or a native-data speedup.", + "Cache bounds retained source keys, not total process memory.", + "Compiler warning and audit events occur on cache misses; fresh AST parsing still occurs on every loaded-code check.", + "Cache shares the existing trusted-process boundary; it does not authenticate coherent forgery of private process state.", + "The already running native pilot retains its frozen original implementation." + ], + "native_acceptance": false, + "paired_fixture": { + "comparison": { + "baseline_cpu_seconds": 303.897467, + "baseline_postcheck_sha256": "146aea7545077b933f921fc53a15340b5818f478408199c12d101c0c920477de", + "baseline_profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "baseline_wall_seconds": 308.2577062920318, + "cpu_reduction_fraction": 0.2935343617062791, + "same_original_fixture_scope": true, + "scope": "one_paired_profile_observation_not_generalized_benchmark", + "sole_production_delta_owner_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "wall_reduction_fraction": 0.2966248653600234 + }, + "cpu_seconds": 214.693118, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "guard_sha256": "a665ab886e7feeebadffc630175734434a0667cd6b5a0fe812c4cd2cb4f42ca2", + "junit_sha256": "3b17eabe18fb9d8e6d87cb7f84b5eae00cc331086ab4d90239456c00263f2639", + "peak_rss_bytes": 581271552, + "profile_sha256": "0348d21a7ed9edc1dd2152595b13fcc91d51e6347619d80b78b76b4c1a96e5a9", + "receipt_sha256": "ed856c409f4923e49379044a003f652ec3fcdfa29789be52dd8fee33802f01d2", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 216.82080566696823 + }, + "paired_postcheck_sha256": "fc2c232eee9904077e578ead9ea6e68f4d9d44c7ac9930d44cb1f08c6b67eba8", + "postcheck_correction": "The first checker incorrectly excluded the inherited root pyproject.toml source entry. The preserved original refused before physical verification; the corrected checker accepts that exact entry and passed. No fixture rerun was needed.", + "protocol": "microcosm.acs-source-compilation-cache-adoption.v1", + "release_acceptance": false, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "packages/microcosm-build/tests/test_us_acs_source_compile_cache.py": "3ee1adffa8a1420d6b95f0f6b8cc966f5193c6221420aed9b8dbf83b543d9dd6" + }, + "source_peer_review_sha256": "e5de7a750a3e33a985009a7e330f182a97c7c833bbc0a09e204125028203c244" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py index 569fe84a8..770fdc9bb 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -7,13 +7,15 @@ from __future__ import annotations import ast +import builtins import importlib.util import json import sys import tempfile +from _thread import RLock from dataclasses import InitVar, dataclass from pathlib import Path -from types import CodeType, FunctionType +from types import BuiltinFunctionType, CodeType, FunctionType from weakref import WeakKeyDictionary from microcosm.frame import Frame @@ -38,6 +40,16 @@ _TOKEN = object() _ISSUED = WeakKeyDictionary() +# Only bytecode compilation is reused across producers. Fresh source reads, +# AST checks, local code indexes and loaded-function checks remain mandatory. +_COMPILE_CACHE_MAX_ENTRIES = 128 +_COMPILE_CACHE_MAX_SOURCE_BYTES = 16 * 1024**2 +_COMPILE_CACHE_MAX_ENTRY_BYTES = 1024**2 +_COMPILE_CACHE_COMPILER = compile +_COMPILE_CACHE_DEFAULT_OPTIMIZE = sys.flags.optimize +_COMPILE_CACHE = {} +_COMPILE_CACHE_LOCK = RLock() + class ACSNativeCoverageBindingError(ValueError): """Static refusal without source values, paths or exception chains.""" @@ -48,6 +60,93 @@ def _require(condition, code): raise ACSNativeCoverageBindingError(code) +def _clear_compile_cache(): + """Clear only compiled-source outputs, for process-local test isolation.""" + with _COMPILE_CACHE_LOCK: + _COMPILE_CACHE.clear() + + +def _compile_source( + source, filename, mode="exec", *, flags=0, dont_inherit=True, optimize=-1 +): + """Reuse bounded immutable bytecode; never reuse loaded-function validity. + + Non-original compilers and context-dependent or non-exact inputs bypass the + cache. Compiler warning/audit events occur on misses; AST parsing still runs + on every _live_code call. This shares the trusted-process scope of that check. + """ + compiler = compile + eligible = ( + compiler is _COMPILE_CACHE_COMPILER + and type(compiler) is BuiltinFunctionType + and compiler.__module__ == "builtins" + and compiler.__name__ == "compile" + and compiler.__self__ is builtins + and type(source) is bytes + and type(filename) is str + and type(mode) is str + and mode in ("exec", "eval", "single") + and type(flags) is int + and flags >= 0 + and dont_inherit is True + and type(optimize) is int + and optimize in (-1, 0, 1, 2) + and len(source) <= _COMPILE_CACHE_MAX_ENTRY_BYTES + and len(source) <= _COMPILE_CACHE_MAX_SOURCE_BYTES + and _COMPILE_CACHE_MAX_ENTRIES > 0 + ) + key = None + if eligible: + effective_optimize = ( + _COMPILE_CACHE_DEFAULT_OPTIMIZE if optimize == -1 else optimize + ) + key = ( + source, + filename, + mode, + flags, + dont_inherit, + effective_optimize, + id(compiler), + ) + with _COMPILE_CACHE_LOCK: + entry = _COMPILE_CACHE.get(key) + if ( + type(entry) is tuple + and len(entry) == 3 + and type(entry[0]) is tuple + and entry[0] == key + and entry[1] is compiler + and type(entry[2]) is CodeType + ): + return entry[2] + _COMPILE_CACHE.pop(key, None) + + # Compile outside the lock: instrumentation/audit hooks may reenter, and a + # concurrent duplicate miss is harmless. Never retain a failed compilation. + code = compiler( + source, + filename, + mode, + flags=flags, + dont_inherit=dont_inherit, + optimize=optimize, + ) + if key is not None and type(code) is CodeType: + with _COMPILE_CACHE_LOCK: + # A nested/concurrent call may already have filled this key. FIFO + # eviction bounds retained source keys; it is not a hard RSS cap. + if key not in _COMPILE_CACHE: + while _COMPILE_CACHE and ( + len(_COMPILE_CACHE) >= _COMPILE_CACHE_MAX_ENTRIES + or sum(len(item[0]) for item in _COMPILE_CACHE) + len(source) + > _COMPILE_CACHE_MAX_SOURCE_BYTES + ): + del _COMPILE_CACHE[next(iter(_COMPILE_CACHE))] + _COMPILE_CACHE[key] = (key, compiler, code) + return code + + def _live_code(module, compiled): """Check loaded Python implementations against their current source bytes. @@ -100,7 +199,11 @@ def visit(code): if isinstance(value, CodeType): visit(value) - visit(compile(Path(path).read_bytes(), path, "exec", dont_inherit=True)) + visit( + _compile_source( + Path(path).read_bytes(), path, "exec", dont_inherit=True + ) + ) compiled[path] = codes _require( function.__globals__ is vars(origin) diff --git a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py new file mode 100644 index 000000000..6d108fe7a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py @@ -0,0 +1,457 @@ +"""Invented controls for the real ACS owner's source-only compilation cache.""" + +from __future__ import annotations +import __future__ + +import ast +import builtins +import sys +from contextlib import contextmanager +from pathlib import Path +from types import CodeType, FunctionType, ModuleType + +import pytest + +from microcosm.build.us_runtime import acs_native_coverage_binding as native + + +@pytest.fixture(autouse=True) +def empty_compile_cache(): + native._clear_compile_cache() + yield + native._clear_compile_cache() + + +@contextmanager +def _observed_calls(): + """Observe real compile/read calls without replacing attested functions.""" + assert sys.getprofile() is None + counts = {"compile": 0, "read": 0} + helper_code = native._compile_source.__code__ + read_code = Path.read_bytes.__code__ + + def observe(frame, event, arg): + if ( + event == "c_call" + and frame.f_code is helper_code + and arg is builtins.compile + ): + counts["compile"] += 1 + elif event == "call" and frame.f_code is read_code: + counts["read"] += 1 + + sys.setprofile(observe) + try: + yield counts + finally: + sys.setprofile(None) + + +def _module(tmp_path, monkeypatch, source, *, name="invented_acs_compile_cache"): + path = tmp_path / f"{name}.py" + path.write_bytes(source) + module = ModuleType(name) + module.__file__ = str(path) + module.__package__ = "" + monkeypatch.setitem(sys.modules, name, module) + exec(builtins.compile(source, str(path), "exec", dont_inherit=True), vars(module)) + return module, path + + +def _index(code): + result = {} + + def visit(item): + result[item.co_qualname] = item + for value in item.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(code) + return result + + +@pytest.mark.parametrize("optimize", [0, 1, 2]) +def test_plain_helper_preserves_compiled_code_and_qualified_name_order(optimize): + source = """# coding: utf-8 +from __future__ import annotations +label = "café" +def outer(value: Missing): + "docstring" + assert value is not None + def nested(): + return [item for item in range(value)] + def nested(): + return lambda: {item: item for item in range(value)} + return nested +class Example: + @property + def item(self): + return label + @item.setter + def item(self, value): + self._item = value +""".encode() + filename = "invented/café/../module.py" + expected = builtins.compile( + source, filename, "exec", dont_inherit=True, optimize=optimize + ) + with _observed_calls() as calls: + first = native._compile_source(source, filename, optimize=optimize) + second = native._compile_source(source, filename, optimize=optimize) + assert type(native._compile_source) is FunctionType + assert first is second + assert first == expected + assert first.co_filename == filename + assert _index(first) == _index(expected) + assert calls["compile"] == 1 + + +def test_complete_inputs_separate_cache_entries(): + source = b"42" + options = [ + ("literal/../source.py", "exec", 0, 0), + ("source.py", "exec", 0, 0), + ("source.py", "eval", 0, 0), + ("source.py", "exec", __future__.annotations.compiler_flag, 0), + ("source.py", "exec", 0, 1), + ("source.py", "exec", 0, 2), + ] + with _observed_calls() as calls: + for filename, mode, flags, optimize in options: + expected = builtins.compile( + source, + filename, + mode, + flags=flags, + dont_inherit=True, + optimize=optimize, + ) + first = native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + assert first == expected + assert first.co_filename == filename + assert ( + native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + is first + ) + assert calls["compile"] == len(options) + assert len(native._COMPILE_CACHE) == len(options) + with _observed_calls() as calls: + effective = native._compile_source( + source, "source.py", optimize=sys.flags.optimize + ) + assert native._compile_source(source, "source.py", optimize=-1) is effective + assert calls["compile"] == 0 + + +def test_same_literal_path_tracks_a_b_a_source_bytes(): + first_source, second_source = b"value = 1", b"value = 2" + assert len(first_source) == len(second_source) + with _observed_calls() as calls: + first = native._compile_source(first_source, "unchanged.py") + second = native._compile_source(second_source, "unchanged.py") + again = native._compile_source(first_source, "unchanged.py") + assert first is again + assert second != first + assert calls["compile"] == 2 + + +def test_changed_compiler_is_called_each_time_and_original_entry_remains(monkeypatch): + source = b"value = 1" + original = native._compile_source(source, "compiler.py") + replacements = [] + + def replacement(raw, filename, mode, **options): + replacements.append(raw) + return builtins.compile( + f"value = {len(replacements) + 1}".encode(), filename, mode, **options + ) + + with monkeypatch.context() as context: + context.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(source, "compiler.py") + second = native._compile_source(source, "compiler.py") + assert replacements == [source, source] + assert first != second + assert first != original + assert len(native._COMPILE_CACHE) == 1 + assert native._compile_source(source, "compiler.py") is original + + +@pytest.mark.parametrize( + "options", [{"dont_inherit": False}, {"flags": ast.PyCF_ONLY_AST}] +) +def test_context_dependent_or_mutable_compiler_outputs_are_not_retained(options): + with _observed_calls() as calls: + first = native._compile_source(b"value = 1", "bypass.py", **options) + second = native._compile_source(b"value = 1", "bypass.py", **options) + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_failures_are_not_memoized(): + with _observed_calls() as calls: + for _ in range(2): + with pytest.raises(SyntaxError): + native._compile_source(b"def invalid(:", "repair.py") + assert not native._COMPILE_CACHE + fixed = native._compile_source(b"value = 1", "repair.py") + assert calls["compile"] == 3 + assert type(fixed) is CodeType + assert len(native._COMPILE_CACHE) == 1 + + +def test_fifo_entry_limit_and_clear(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 2) + source = b"value = 1" + first = native._compile_source(source, "first.py") + second = native._compile_source(source, "second.py") + assert native._compile_source(source, "first.py") is first + native._compile_source(source, "third.py") + assert len(native._COMPILE_CACHE) == 2 + assert native._compile_source(source, "second.py") is second + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + native._clear_compile_cache() + assert not native._COMPILE_CACHE + with _observed_calls() as calls: + native._compile_source(source, "first.py") + assert calls["compile"] == 1 + + +def test_source_byte_budget_counts_distinct_filename_keys(monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_SOURCE_BYTES", len(source) * 2) + first = native._compile_source(source, "first.py") + native._compile_source(source, "second.py") + native._compile_source(source, "third.py") + assert sum(len(key[0]) for key in native._COMPILE_CACHE) == len(source) * 2 + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + + +@pytest.mark.parametrize( + "limit_name", ["_COMPILE_CACHE_MAX_ENTRY_BYTES", "_COMPILE_CACHE_MAX_SOURCE_BYTES"] +) +def test_oversize_source_is_compiled_without_cache(limit_name, monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, limit_name, len(source) - 1) + with _observed_calls() as calls: + first = native._compile_source(source, "oversize.py") + second = native._compile_source(source, "oversize.py") + assert first == second + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_live_check_reads_source_twice_even_after_warming(tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + with _observed_calls() as cold: + first_index = {} + native._live_code(module, first_index) + with _observed_calls() as warm: + second_index = {} + native._live_code(module, second_index) + assert cold == {"compile": 1, "read": 2} + assert warm == {"compile": 0, "read": 2} + assert first_index == second_index + assert first_index is not second_index + first_index[module.__file__]["value"] = None + native._live_code(module, {}) + assert second_index[module.__file__]["value"] is not None + + +def test_real_producer_keeps_read_counts_and_full_evidence_on_warm_cache(): + # Settle unrelated import/manifest memoization, then compare only compile + # cache cold versus warm with the real producer and unchanged source files. + native._producer() + native._clear_compile_cache() + with _observed_calls() as cold: + first = native._producer() + with _observed_calls() as warm: + second = native._producer() + assert cold["compile"] > 0 + assert warm["compile"] == 0 + assert cold["read"] > 0 + assert warm["read"] == cold["read"] + assert first == second + assert len(native._COMPILE_CACHE) <= native._COMPILE_CACHE_MAX_ENTRIES + assert ( + sum(len(key[0]) for key in native._COMPILE_CACHE) + <= native._COMPILE_CACHE_MAX_SOURCE_BYTES + ) + + +@pytest.mark.parametrize("drift", ["code", "globals", "declaration"]) +def test_warm_cache_does_not_hide_loaded_function_drift(drift, tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + native._live_code(module, {}) + if drift == "code": + module.value.__code__ = module.value.__code__.replace(co_consts=(None, 2)) + elif drift == "globals": + module.value = FunctionType(module.value.__code__, dict(vars(module))) + else: + del module.value + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_warm_cache_preserves_imported_alias_identity(tmp_path, monkeypatch): + origin, _ = _module( + tmp_path, + monkeypatch, + b"def value():\n return 1\n", + name="invented_acs_origin", + ) + consumer, _ = _module( + tmp_path, + monkeypatch, + b"from invented_acs_origin import value\n", + name="invented_acs_consumer", + ) + native._live_code(consumer, {}) + consumer.value = FunctionType(origin.value.__code__, vars(origin)) + assert consumer.value.__code__ is origin.value.__code__ + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(consumer, {}) + + +def test_warm_cache_checks_distinct_functions_with_shared_code_and_closures( + tmp_path, monkeypatch +): + source = b"""def make(): + def target(): + return 1 + def entry(): + return target() + return entry +first = make() +second = make() +""" + module, _ = _module(tmp_path, monkeypatch, source) + native._live_code(module, {}) + assert module.first.__code__ is module.second.__code__ + first_target = module.first.__closure__[0].cell_contents + second_target = module.second.__closure__[0].cell_contents + assert first_target is not second_target + second_target.__code__ = second_target.__code__.replace(co_consts=(None, 2)) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_second_source_read_can_invalidate_a_warm_entry(tmp_path, monkeypatch): + original = b"def value():\n return 1\n" + changed = b"def value():\n return 2\n" + module, path = _module(tmp_path, monkeypatch, original) + native._live_code(module, {}) + read_bytes = Path.read_bytes + calls = 0 + + def change_on_second_read(self): + nonlocal calls + if self == path: + calls += 1 + if calls == 2: + path.write_bytes(changed) + return read_bytes(self) + + monkeypatch.setattr(Path, "read_bytes", change_on_second_read) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + assert calls == 2 + + +def test_malformed_cache_entry_is_recompiled_and_replaced(): + source = b"value = 1" + first = native._compile_source(source, "malformed.py") + key = next(iter(native._COMPILE_CACHE)) + native._COMPILE_CACHE[key] = None + with _observed_calls() as calls: + repaired = native._compile_source(source, "malformed.py") + again = native._compile_source(source, "malformed.py") + assert repaired == first + assert repaired is not first + assert again is repaired + assert calls["compile"] == 1 + + +def test_nested_compilation_keeps_current_fifo_bound(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 1) + assert sys.getprofile() is None + nested = [] + helper_code = native._compile_source.__code__ + + def after_compile(frame, event, arg): + if ( + not nested + and event == "c_return" + and frame.f_code is helper_code + and arg is builtins.compile + ): + nested.append(True) + native._compile_source(b"nested = 2", "nested.py") + + sys.setprofile(after_compile) + try: + outer = native._compile_source(b"outer = 1", "outer.py") + finally: + sys.setprofile(None) + assert nested == [True] + assert outer == builtins.compile( + b"outer = 1", "outer.py", "exec", dont_inherit=True + ) + assert len(native._COMPILE_CACHE) == 1 + assert next(iter(native._COMPILE_CACHE))[1] == "outer.py" + + +def test_swapped_valid_entries_are_recompiled_for_their_complete_inputs(): + first_source, second_source = b"value = 1", b"value = 2" + first = native._compile_source(first_source, "first.py") + second = native._compile_source(second_source, "second.py") + first_key, second_key = tuple(native._COMPILE_CACHE) + native._COMPILE_CACHE[first_key], native._COMPILE_CACHE[second_key] = ( + native._COMPILE_CACHE[second_key], + native._COMPILE_CACHE[first_key], + ) + with _observed_calls() as calls: + repaired_first = native._compile_source(first_source, "first.py") + repaired_second = native._compile_source(second_source, "second.py") + assert native._compile_source(first_source, "first.py") is repaired_first + assert native._compile_source(second_source, "second.py") is repaired_second + assert calls["compile"] == 2 + assert repaired_first == first and repaired_first is not first + assert repaired_second == second and repaired_second is not second + assert repaired_first.co_filename == "first.py" + assert repaired_second.co_filename == "second.py" + + +def test_python_compiler_captured_as_initial_compiler_still_bypasses(monkeypatch): + # This is the exact captured/current state after a pre-import replacement; + # leave all issued-owner modules loaded and exercise the real cache helper. + calls = [] + + def replacement(source, filename, mode, **options): + calls.append(source) + return builtins.compile( + f"value = {len(calls)}".encode(), filename, mode, **options + ) + + # Textual labels alone must not classify a Python replacement as builtin. + replacement.__module__ = "builtins" + replacement.__name__ = "compile" + replacement.__self__ = builtins + monkeypatch.setattr(native, "_COMPILE_CACHE_COMPILER", replacement) + monkeypatch.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(b"value = 0", "captured.py") + second = native._compile_source(b"value = 0", "captured.py") + assert calls == [b"value = 0", b"value = 0"] + assert first != second + assert not native._COMPILE_CACHE From 23866bc8441919fe4a27ec677006ec9c78e2f59d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:29:45 -0400 Subject: [PATCH 051/255] Merge disjoint PUF raw routes before whole-cohort finalization --- ...55-two-route-values-controls-20260910.json | 38 +++ .../us_runtime/puf55_route_finalization.py | 226 ++++++++++++++ .../tests/test_us_puf55_route_finalization.py | 291 ++++++++++++++++++ 3 files changed, 555 insertions(+) create mode 100644 experiments/us-puf55-two-route-values-controls-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py create mode 100644 packages/microcosm-build/tests/test_us_puf55_route_finalization.py diff --git a/experiments/us-puf55-two-route-values-controls-20260910.json b/experiments/us-puf55-two-route-values-controls-20260910.json new file mode 100644 index 000000000..5db3a77bc --- /dev/null +++ b/experiments/us-puf55-two-route-values-controls-20260910.json @@ -0,0 +1,38 @@ +{ + "attachment_performed": false, + "cpu_seconds": 3.8935099999999996, + "date": "2026-09-10", + "finalization_performed": false, + "fitted_model_or_donor_authority_verified": false, + "guard_sha256": "bde8f7ce2742b45a750e7705be6fc899b7f04f3d913d933d69594ff6724ca6e9", + "junit_sha256": "1619f4b57c5a39df909ee03f7c54d8dd05f634bdda68fe59d12185d012bf4431", + "native_acceptance": false, + "native_access": false, + "next_step": "Bind actual recipient and donor/model owners, execute both real routes, merge raw targets, finalize the whole clone cohort once, and attach once.", + "peak_rss_bytes": 438517760, + "peer_source_review_sha256": "df628b7d237decebbf252f0c51cc215d2801fa67a355068e05a0a0ae21ee70b1", + "postcheck_sha256": "15732db6c91ba14f8c9c012a0f4251c37e6cd1793287958ce0d7f92739e588c4", + "protocol": "microcosm.puf55-two-route-values-controls.v1", + "raw_merge_only": true, + "receipt_sha256": "d52648c33cf48d786a6ee7d4a29838a03e936700db8de05cb4ae0a5b476cac1e", + "release_acceptance": false, + "scope": "18_invented_puf55_raw_route_merge_controls", + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py": "5ce739283cd80d37c17a3063dc0f8585f303411c21b2f5d87cc92ffb6345f9cd", + "packages/microcosm-build/tests/test_us_puf55_route_finalization.py": "f3638ceb41953af57f50c2a43b58eac7fb9dcc4f959832d4921da9ac236041d9" + }, + "source_or_population_admission": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "typed_chain_metadata_synthetic": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 505 + }, + "wall_seconds": 4.931140874978155 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py new file mode 100644 index 000000000..ed35d3552 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py @@ -0,0 +1,226 @@ +"""Validate and merge the two PUF55 raw chains before one finalization. + +This is a values-only boundary. The host must freshly qualify the recipient +matrices against its actual financial run, authenticate every typed graph edge, +and verify donor/model ownership before using the result. Caller bytes and this +receipt issue no source, Population, donor, fit or release authority. No model +pickle is loaded, no fit is performed, and no output column is attached here. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input + +from . import full_puf_enrichment as full +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.puf55-route-raw-merge.v1" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_ROUTE_" + reason) + + +@dataclass(frozen=True) +class Puf55RouteDraws: + """Descriptive typed-chain bytes, never an authenticated producer capsule.""" + + profile: full.PufOutputProfile + matrix: bytes + matrix_producer_key: str + raw_draws: tuple[tuple[str, bytes], ...] + apply_state: bytes + training_state: bytes + + +def _route_snapshot(route): + _require(type(route) is Puf55RouteDraws, "INPUT_TYPE") + # Frozen dataclasses can still be changed with object.__setattr__. Read each + # field once and use only these immutable retained values during decoding. + result = ( + route.profile, + route.matrix, + route.matrix_producer_key, + route.raw_draws, + route.apply_state, + route.training_state, + ) + profile, matrix, key, raw, apply_state, training_state = result + _require(type(profile) is full.PufOutputProfile and profile in PROFILES, "PROFILE") + _require( + all(type(value) is bytes for value in (matrix, apply_state, training_state)) + and type(key) is str + and full.codec._hash(key), + "ARTIFACT_TYPES", + ) + _require( + type(raw) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in raw + ) + and tuple(pair[0] for pair in raw) == profile.targets, + "RAW_ROSTER", + ) + return result + + +def _receiving_axis(frame): + table = frame.table("tax_unit") + column = provenance.support_clone_index_column("tax_unit") + _require( + table.columns.is_unique + and table.index.is_unique + and column in table + and table[column].dtype == np.dtype("int64") + and bool(table[column].isin((0, 1)).all()), + "CLONE_AXIS", + ) + all_ids = full._ids(table.tax_unit_id, "route_receiving") + _require(len(set(all_ids.tolist())) == len(all_ids), "RECEIVING_IDS") + mask = table[column].eq(1).to_numpy() + ids = all_ids[mask] + _require(len(ids) > 0, "NO_RECIPIENTS") + return ids, table.index[mask].copy() + + +def merge_puf55_route_draws(frame, *, recipient_matrices, route_draws, seed): + """Return all 55 raw targets in complete receiving pandas row order. + + ``recipient_matrices`` is the exact immutable matrix roster freshly derived + by the host's authenticated recipient qualifier. Empty routes are omitted + in nine/eight order. This function checks values, not that upstream duty. + Finalize the complete returned table once, after checking both donor/model + bindings; never finalize subsets or attach a route over another route. + """ + _require(type(seed) is int and seed >= 0, "SEED") + _require( + type(recipient_matrices) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in recipient_matrices + ), + "MATRIX_ROSTER", + ) + _require(type(route_draws) is tuple and 1 <= len(route_draws) <= 2, "ROUTE_ROSTER") + retained = tuple(_route_snapshot(route) for route in route_draws) + names = tuple(values[0].value for values in retained) + _require( + names == tuple(p.value for p in PROFILES if p.value in names) + and tuple(name for name, _ in recipient_matrices) == names, + "ROUTE_ORDER", + ) + ids, row_index = _receiving_axis(frame) + tables, seen, evidence, raw_seals = [], set(), [], [] + for values, (_, expected_matrix) in zip(retained, recipient_matrices, strict=True): + profile, matrix, key, raw, apply_state, training_state = values + _require(matrix == expected_matrix, "RECIPIENT_MATRIX_CHANGED") + prepared = model_input.decode_recipient_matrix(matrix) + route_ids = tuple(prepared.entity_ids.tolist()) + _require( + prepared.entity == "tax_unit" + and tuple(prepared.features.columns) == profile.predictors + and prepared.features.index.name == "tax_unit_id", + "MATRIX_PROFILE", + ) + _require(not seen.intersection(route_ids), "OVERLAPPING_IDS") + seen.update(route_ids) + decoded = full.decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=key, + raw_draws=dict(raw), + apply_state=apply_state, + training_state=training_state, + seed=seed, + profile=profile, + ) + # Check the detached decoder result against the immutable input bytes; + # comparing two mutable returned tables would not anchor its values. + _require( + type(decoded) is pd.DataFrame + and tuple(decoded.columns) == profile.targets + and decoded.index.equals(pd.Index(route_ids, name="tax_unit_id")) + and all(dtype == np.dtype("float64") for dtype in decoded.dtypes) + and all( + full.codec.encode_raw_target( + decoded[target].to_numpy(), target=target, index=decoded.index + ) + == payload + for target, payload in raw + ), + "DECODED_RAW_CHANGED", + ) + tables.append(decoded.copy(deep=True)) + raw_seals.append((route_ids, raw)) + evidence.append( + { + "profile": profile.value, + "rows": len(route_ids), + "matrix_sha256": full.codec.sha(matrix), + "matrix_producer_key": key, + "apply_state_sha256": full.codec.sha(apply_state), + "training_state_sha256": full.codec.sha(training_state), + "raw_target_sha256": {target: full.codec.sha(p) for target, p in raw}, + } + ) + _require(seen == set(ids.tolist()), "INCOMPLETE_RECIPIENT_UNION") + combined = ( + pd.concat(tables, axis=0).loc[ids.tolist(), list(PROFILES[0].targets)].copy() + ) + # The IDs above establish the one explicit bridge to the existing pandas + # row index required by the maintained whole-cohort finalizer. + combined.index = row_index.copy() + fresh_ids, fresh_index = _receiving_axis(frame) + _require( + np.array_equal(ids, fresh_ids) and row_index.identical(fresh_index), + "RECEIVING_AXIS_CHANGED", + ) + receipt = full.codec.encode_json( + { + "protocol": PROTOCOL, + "routes": evidence, + "recipient_rows": len(ids), + "target_order": list(PROFILES[0].targets), + "ordered_recipient_ids_sha256": full.codec.sha(ids.tobytes()), + "merge": "disjoint complete clone-one union in receiving order", + "finalization_performed": False, + "donor_model_binding_verified_here": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + _require(len(receipt) <= 128 * 1024, "RECEIPT_SIZE") + # A later route decoder could have changed an earlier detached table. Check + # every final route slice against its original bytes after all decoding. + positions = {value: position for position, value in enumerate(ids.tolist())} + _require( + tuple(combined.columns) == PROFILES[0].targets + and combined.index.identical(row_index) + and all(dtype == np.dtype("float64") for dtype in combined.dtypes) + and all( + full.codec.encode_raw_target( + combined[target].to_numpy()[[positions[value] for value in route_ids]], + target=target, + index=pd.Index(route_ids, name="tax_unit_id"), + ) + == payload + for route_ids, raw in raw_seals + for target, payload in raw + ), + "MERGED_RAW_CHANGED", + ) + return combined, receipt diff --git a/packages/microcosm-build/tests/test_us_puf55_route_finalization.py b/packages/microcosm-build/tests/test_us_puf55_route_finalization.py new file mode 100644 index 000000000..c60420da1 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf55_route_finalization.py @@ -0,0 +1,291 @@ +"""Invented real codec packets; no fitted model, source or graph authority. + +These controls exercise only the route-value boundary. Actual train/apply, +donor/producer ownership and full attachment require separate guarded tests. +""" + +import json +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import puf55_route_finalization as routes +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +full = routes.full +SEED = 21 + + +def _frame(reverse=False): + ids = np.array([11, 12, 13, 101, 202, 303], dtype=np.int64) + person = pd.DataFrame({"person_id": ids}) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + person["social_security_source_value"] = [-0.0, np.nan, 17.0, np.nan, 0.0, 2.0] + person["source_component_allowed"] = [True, False, True, False, False, True] + tables = {"person": person} + for entity in US_SCHEMA.group_entities: + tables[entity] = pd.DataFrame({entity + "_id": ids}) + tax_unit = tables["tax_unit"] + tax_unit[routes.provenance.support_clone_index_column("tax_unit")] = np.array( + [0, 0, 0, 1, 1, 1], dtype=np.int64 + ) + # Group entity IDs stay sorted, as required by Frame linkage validation. + # Nonmonotonic pandas labels exercise the distinct receiving row axis. + labels = [10, 20, 50, 70, 30, 90] + tax_unit.index = pd.Index(labels[::-1] if reverse else labels, name="original_row") + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.ones(len(ids)), WeightKind.DESIGN)}, + metadata={"source_admission": False, "scope": "invented_codec_values"}, + ) + + +def _route(profile, ids, offset): + """Build canonical synthetic chain metadata with the maintained codecs.""" + index = pd.Index(np.asarray(ids, dtype=np.int64), name="tax_unit_id") + features = pd.DataFrame(1.0, index=index, columns=profile.predictors) + matrix = model_input.encode_recipient_matrix( + features, entity="tax_unit", entity_ids=index.to_numpy() + ) + key = full.codec.sha(("invented-matrix:" + profile.value).encode()) + raw = tuple( + ( + target, + full.codec.encode_raw_target( + np.asarray(ids, dtype=np.float64) + offset + j / 100, + target=target, + index=index, + ), + ) + for j, target in enumerate(profile.targets) + ) + state = qrf.QRFChainState( + predictors=profile.predictors, + targets=profile.targets, + completed_targets=profile.targets, + entity="tax_unit", + weight_kind="design", + weight_sha256="1" * 64, + n_estimators=2, + zero_atol=1e-8, + max_samples_leaf=None, + max_samples_leaf_kind="none", + seed=SEED, + fit_n_jobs=1, + donor_index=qrf._index_identity(pd.Index([1, 2], name="RECID")), + recipient_index=qrf._index_identity(index), + fit_rng_state_json=json.dumps(np.random.PCG64(1).state), + draw_rng_state_json=json.dumps(np.random.PCG64(2).state), + ) + models = [ + { + "target": target, + "sha256": full.codec.sha((profile.value + target).encode()), + "training_id": full.codec.sha( + ("training:" + profile.value + target).encode() + ), + } + for target in profile.targets + ] + training = full.codec.encode_json( + { + "schema_version": 1, + "state": qrf_target.LegacyQRFTrainingState.from_chain(state).to_dict(), + "models": models, + } + ) + application = { + "schema_version": 1, + "state": state.to_dict(), + "seed": SEED, + "models": models, + "raw_targets": [ + {"target": target, "sha256": full.codec.sha(payload)} + for target, payload in raw + ], + } + applied = full.codec.encode_json( + { + "schema_version": 1, + "matrix_sha256": full.codec.sha(matrix), + "matrix_producer_key": key, + "application": application, + } + ) + return routes.Puf55RouteDraws(profile, matrix, key, raw, applied, training) + + +def _inputs(): + return ( + _route(full.PUF55_SURVEY_SS, [202, 303], 1000), + _route(full.PUF55_SURVEY_SS_NO_TOTAL, [101], 2000), + ) + + +def _merge(frame, packets, **kwargs): + arguments = { + "recipient_matrices": tuple((p.profile.value, p.matrix) for p in packets), + "route_draws": packets, + "seed": SEED, + **kwargs, + } + return routes.merge_puf55_route_draws(frame, **arguments) + + +@pytest.mark.parametrize("reverse", (False, True)) +def test_merge_restores_complete_receiving_order_and_preserves_source_values(reverse): + frame, packets = _frame(reverse), _inputs() + before = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + person_bits = frame.person.social_security_source_value.to_numpy().tobytes() + actual, payload = _merge(frame, packets) + units = frame.table("tax_unit") + selected = units.loc[units.tax_unit_id.isin([101, 202, 303])] + expected = pd.DataFrame( + { + target: [ + value + (2000 if value == 101 else 1000) + j / 100 + for value in selected.tax_unit_id + ] + for j, target in enumerate(full.PUF55_SURVEY_SS.targets) + }, + index=selected.index, + ) + pd.testing.assert_frame_equal(actual, expected, check_exact=True) + assert actual.shape == (3, 55) + receipt = full.codec.decode_json(payload) + assert receipt["recipient_rows"] == 3 + assert [r["rows"] for r in receipt["routes"]] == [2, 1] + assert not any( + receipt[name] + for name in ( + "finalization_performed", + "donor_model_binding_verified_here", + "source_admission_issued", + "population_admission_issued", + "release_eligible", + ) + ) + for entity in frame.entities: + pd.testing.assert_frame_equal( + frame.table(entity), before[entity], check_exact=True + ) + assert frame.person.social_security_source_value.to_numpy().tobytes() == person_bits + + +@pytest.mark.parametrize("profile", routes.PROFILES) +def test_single_nonempty_route_omits_other_route(profile): + packet = _route(profile, [202, 101, 303], 1000) + actual, payload = _merge(_frame(), (packet,)) + assert actual.index.tolist() == [70, 30, 90] + assert len(full.codec.decode_json(payload)["routes"]) == 1 + + +@pytest.mark.parametrize("defect", ("overlap", "missing", "extra", "clone_zero")) +def test_refuses_inexact_recipient_partition(defect): + nine_ids, eight_ids = { + "overlap": ([202, 303], [101, 202]), + "missing": ([202], [101]), + "extra": ([202, 303], [101, 999]), + "clone_zero": ([202, 303], [101, 11]), + }[defect] + packets = ( + _route(full.PUF55_SURVEY_SS, nine_ids, 1000), + _route(full.PUF55_SURVEY_SS_NO_TOTAL, eight_ids, 2000), + ) + with pytest.raises( + ValueError, match="PUF55_ROUTE_(OVERLAPPING_IDS|INCOMPLETE_RECIPIENT_UNION)" + ): + _merge(_frame(), packets) + + +@pytest.mark.parametrize("defect", ("matrix", "order", "mutable_bytes", "profile")) +def test_refuses_changed_matrix_route_or_storage(defect): + packets = _inputs() + expected = tuple((p.profile.value, p.matrix) for p in packets) + if defect == "matrix": + expected = ( + (expected[0][0], _route(packets[0].profile, [303, 202], 1000).matrix), + expected[1], + ) + elif defect == "order": + packets = packets[::-1] + expected = expected[::-1] + elif defect == "mutable_bytes": + packets = (replace(packets[0], matrix=bytearray(packets[0].matrix)), packets[1]) + else: + packets = (replace(packets[0], profile=full.PUF59), packets[1]) + with pytest.raises(ValueError, match="PUF55_ROUTE_"): + _merge(_frame(), packets, recipient_matrices=expected) + + +@pytest.mark.parametrize("defect", ("seed", "raw", "model_history", "target_history")) +def test_complete_chain_history_is_checked_before_merge(defect): + packets = _inputs() + if defect == "seed": + with pytest.raises(ValueError, match="PUF_FULL_CHAIN_IDENTITY"): + _merge(_frame(), packets, seed=SEED + 1) + return + packet = packets[0] + if defect == "raw": + target, _ = packet.raw_draws[0] + changed = full.codec.encode_raw_target( + [99.0, 98.0], + target=target, + index=model_input.decode_recipient_matrix(packet.matrix).features.index, + ) + packet = replace(packet, raw_draws=((target, changed), *packet.raw_draws[1:])) + else: + applied = full.codec.decode_json(packet.apply_state) + if defect == "model_history": + applied["application"]["models"][0]["sha256"] = "e" * 64 + else: + applied["application"]["raw_targets"].reverse() + packet = replace(packet, apply_state=full.codec.encode_json(applied)) + with pytest.raises( + ValueError, match="PUF_RAW_HISTORY|PUF_FULL_CHAIN_IDENTITY|Legacy QRF history" + ): + _merge(_frame(), (packet, packets[1])) + + +def test_changed_detached_decoder_values_refuse(): + old, fired = sys.getprofile(), False + + def observe(call, event, arg): + nonlocal fired + if event == "return" and call.f_code is full.decode_full_puf_draws.__code__: + fired = True + arg.iloc[0, 0] += 1 + + try: + sys.setprofile(observe) + with pytest.raises(ValueError, match="PUF55_ROUTE_DECODED_RAW_CHANGED"): + _merge(_frame(), _inputs()) + finally: + sys.setprofile(old) + assert fired + + +def test_later_decoder_cannot_change_an_earlier_route_table(): + old, calls, fired = sys.getprofile(), 0, False + + def observe(call, event, arg): + nonlocal calls, fired + if event == "return" and call.f_code is full.decode_full_puf_draws.__code__: + calls += 1 + if calls == 2: + call.f_back.f_locals["tables"][0].iloc[0, 0] += 1 + fired = True + + try: + sys.setprofile(observe) + with pytest.raises(ValueError, match="PUF55_ROUTE_MERGED_RAW_CHANGED"): + _merge(_frame(), _inputs()) + finally: + sys.setprofile(old) + assert calls == 2 and fired From b47209e2b877c826944ae864ad26bac6d60fdea9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:34:38 -0400 Subject: [PATCH 052/255] Keep identity diagnostics on the explicit CPU-only Torch path --- .github/workflows/test.yml | 8 + .../test_spec_seed_identity_cpu_bootstrap.py | 226 ++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 105 +++++++- 3 files changed, 335 insertions(+), 4 deletions(-) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 951e8fabd..58c73cf6d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -226,6 +226,14 @@ jobs: python-version: "3.14.4" - name: Sync workspace with US and UK engines run: uv sync --all-packages --locked --extra us --extra uk + - name: Check CPU-only diagnostic bootstrap + timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: >- + uv run --no-sync pytest --noconftest -p no:cacheprovider -q + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics timeout-minutes: 15 diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py new file mode 100644 index 000000000..9c31d579c --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py @@ -0,0 +1,226 @@ +"""Invented bootstrap controls; no Torch, Microcosm, engines or data imported.""" + +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + # Normal stdlib-only module loading; main/audit/derive are never called. + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("cpu_bootstrap_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fresh_import_state(monkeypatch): + # Keep the process import roster isolated without importing any dependency. + for name in tuple(sys.modules): + if ( + name == "torch" + or name == "cuda.bindings" + or name.startswith("cuda.bindings.") + ): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(sys, "meta_path", list(sys.meta_path)) + + +@pytest.mark.parametrize( + "name", ["json", "torch", "cuda", "cuda.pathfinder", "cuda.bindings_other"] +) +def test_non_cuda_bindings_imports_pass_through(diagnostic, name): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + assert finder.find_spec(name, ["invented"], object()) is None + assert finder.attempts == 0 + + +def test_only_exact_optional_bindings_are_excluded(diagnostic): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + with pytest.raises(ModuleNotFoundError) as error: + finder.find_spec("cuda.bindings") + assert error.value.name == "cuda.bindings" + assert finder.attempts == 1 + + +@pytest.mark.parametrize("fails", [False, True]) +def test_finder_removed_on_success_and_failure(diagnostic, fresh_import_state, fails): + before = tuple(sys.meta_path) + other = object() + evidence = {} + + class InventedFailureError(Exception): + pass + + try: + with diagnostic.cpu_only_torch_import(evidence): + finder = sys.meta_path[0] + assert type(finder) is diagnostic._CpuOnlyCudaBindingsFinder + assert tuple(sys.meta_path[1:]) == before + sys.meta_path.append(other) + with pytest.raises(ModuleNotFoundError): + finder.find_spec("cuda.bindings") + if fails: + raise InventedFailureError + except InventedFailureError: + assert fails + assert tuple(sys.meta_path) == (*before, other) + assert evidence == { + "cuda_bindings_import_attempts": 1, + "temporary_finder_removed": True, + } + + +@pytest.mark.parametrize("name", ["torch", "cuda.bindings", "cuda.bindings.runtime"]) +def test_preloaded_optional_path_refuses_before_finder( + diagnostic, fresh_import_state, monkeypatch, name +): + monkeypatch.setitem(sys.modules, name, SimpleNamespace()) + before = tuple(sys.meta_path) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_BOOTSTRAP_PRELOADED$"): + with diagnostic.cpu_only_torch_import({}): + pytest.fail("preloaded dependency was accepted") + assert tuple(sys.meta_path) == before + + +def test_actual_fallback_contract_is_required( + diagnostic, fresh_import_state, monkeypatch +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + diagnostic.verify_cpu_only_torch_import(evidence) + assert evidence["torch_optional_bindings_fallback_verified"] is True + + +@pytest.mark.parametrize("bindings_present", [False, True]) +def test_unconfirmed_or_loaded_bindings_refuse( + diagnostic, fresh_import_state, monkeypatch, bindings_present +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1 if bindings_present else 0, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + if bindings_present: + monkeypatch.setitem(sys.modules, "cuda.bindings", SimpleNamespace()) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_CPU_FALLBACK$"): + diagnostic.verify_cpu_only_torch_import(evidence) + + +def _candidate(monkeypatch): + monkeypatch.setattr(sys, "base_prefix", "/invented-bootstrap-prefix") + return str( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}{sys.version_info.minor}.zip" + ) + + +def test_only_exact_confirmed_absent_zip_entry_is_removed(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + others = ["", "/invented/code", "/invented/unrelated.zip", candidate + "-other"] + monkeypatch.setattr(sys, "path", [candidate, *others, candidate]) + checked = [] + + def absent(path): + checked.append(path) + raise FileNotFoundError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", absent) + assert diagnostic.omit_absent_stdlib_zip() is True + assert checked == [candidate] + assert sys.path == others + + +@pytest.mark.parametrize("kind", ["regular", "broken_symlink"]) +def test_existing_zip_or_symlink_remains_subject_to_original_guard( + diagnostic, monkeypatch, kind +): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate, "other"]) + # Any successful lstat, including a broken symlink, prevents omission. + monkeypatch.setattr(diagnostic.os, "lstat", lambda path: SimpleNamespace(kind=kind)) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == [candidate, "other"] + + +def test_other_stat_errors_do_not_justify_omission(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate]) + + def denied(path): + raise PermissionError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", denied) + with pytest.raises(PermissionError): + diagnostic.omit_absent_stdlib_zip() + assert sys.path == [candidate] + + +def test_unlisted_stdlib_candidate_is_not_probed(diagnostic, monkeypatch): + _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", ["other"]) + monkeypatch.setattr( + diagnostic.os, "lstat", lambda path: pytest.fail("unexpected metadata probe") + ) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == ["other"] + + +def test_real_non_cuda_import_keeps_original_loader( + diagnostic, fresh_import_state, monkeypatch +): + name = "invented_non_cuda_bootstrap_control" + monkeypatch.delitem(sys.modules, name, raising=False) + loaded = [] + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + loaded.append(module.__name__) + module.marker = "ordinary_loader" + + loader = Loader() + + class Finder: + def find_spec(self, fullname, path=None, target=None): + if fullname == name: + return importlib.machinery.ModuleSpec(fullname, loader) + return None + + finder = Finder() + sys.meta_path.insert(0, finder) + before = tuple(sys.meta_path) + try: + with diagnostic.cpu_only_torch_import({}): + module = importlib.import_module(name) + assert module.marker == "ordinary_loader" + assert module.__loader__ is loader + assert loaded == [name] + assert tuple(sys.meta_path) == before + finally: + sys.modules.pop(name, None) diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index d741f71ae..15aa3550e 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -521,6 +521,84 @@ def versions(lock: dict) -> dict[str, str]: return actual +def omit_absent_stdlib_zip() -> bool: + """Omit only CPython's exact absent ZIP search entry; admit no archive read.""" + candidate = str( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}{sys.version_info.minor}.zip" + ) + if candidate not in sys.path: + return False + try: + os.lstat(candidate) + except FileNotFoundError: + # lstat, rather than exists, distinguishes an absent path from a broken + # symlink. All other entries and their order remain unchanged. + sys.path[:] = [entry for entry in sys.path if entry != candidate] + return True + return False + + +class _CpuOnlyCudaBindingsFinder: + """Use Torch's real optional ImportError fallback during its import only.""" + + def __init__(self): + self.attempts = 0 + + def find_spec(self, fullname, path=None, target=None): + if fullname != "cuda.bindings": + return None + self.attempts += 1 + require(self.attempts <= 8, "CUDA_BINDINGS_PROBE_LIMIT") + raise ModuleNotFoundError( + "cuda.bindings excluded by CPU-only diagnostic bootstrap", + name="cuda.bindings", + ) + + +@contextlib.contextmanager +def cpu_only_torch_import(bootstrap: dict): + """Temporarily exclude optional CUDA bindings, never replace a Torch module.""" + require( + "torch" not in sys.modules + and not any( + name == "cuda.bindings" or name.startswith("cuda.bindings.") + for name in sys.modules + ), + "TORCH_BOOTSTRAP_PRELOADED", + ) + finder = _CpuOnlyCudaBindingsFinder() + sys.meta_path.insert(0, finder) + try: + yield + finally: + # Remove this exact finder on both success and failure, retaining any + # legitimate finder changes made by ordinary dependency imports. + sys.meta_path[:] = [entry for entry in sys.meta_path if entry is not finder] + bootstrap["cuda_bindings_import_attempts"] = finder.attempts + bootstrap["temporary_finder_removed"] = not any( + entry is finder for entry in sys.meta_path + ) + + +def verify_cpu_only_torch_import(bootstrap: dict) -> None: + utilities = sys.modules.get("torch.cuda._utils") + require( + bootstrap["temporary_finder_removed"] is True + and 1 <= bootstrap["cuda_bindings_import_attempts"] <= 8 + and utilities is not None + and getattr(utilities, "_HAS_CUDA_BINDINGS", None) is False + and getattr(utilities, "_cuda_bindings_runtime", False) is None + and not any( + name == "cuda.bindings" or name.startswith("cuda.bindings.") + for name in sys.modules + ), + "TORCH_CPU_FALLBACK", + ) + bootstrap["torch_optional_bindings_fallback_verified"] = True + + def thread_controls(torch) -> dict[str, object]: controls = { "environment": {name: os.environ.get(name) for name in THREAD_ENV}, @@ -627,6 +705,7 @@ def derive( installed: dict[str, str], torch, controls: dict[str, object], + bootstrap: dict[str, object], ) -> dict[str, bytes]: # The exact helper is imported; pytest collection and its tests are never run. sys.path.insert(0, str(root / "packages/microcosm-build/tests")) @@ -697,6 +776,7 @@ def derive( "lock_sha256": LOCK_SHA256, "distributions": installed, "thread_controls": controls, + "dependency_bootstrap": bootstrap, "source_sha256": before, "upload_action_commit": UPLOAD_ACTION_SHA, "seed_modules": list(modules), @@ -731,6 +811,7 @@ def derive( payloads["diagnostic-status.json"] = encoded( { "completed": True, + "dependency_bootstrap": bootstrap, "status": "candidate_values_only", "coverage_pass": False, "payload_sha256": {name: digest(data) for name, data in payloads.items()}, @@ -784,7 +865,17 @@ def main() -> int: } ) publish(output, initial) + bootstrap = { + "policy": "cpu_only_torch_optional_cuda_bindings_exclusion_v1", + "finder_scope": "torch_import_only", + "stdlib_zip_entry": "exact_base_prefix_lib_python_version_zip", + "absent_stdlib_zip_omitted": False, + "cuda_bindings_import_attempts": 0, + "temporary_finder_removed": True, + "torch_optional_bindings_fallback_verified": False, + } try: + bootstrap["absent_stdlib_zip_omitted"] = omit_absent_stdlib_zip() run = coordinates() before = source_stamps(root) lock = tomllib.loads((root / "uv.lock").read_text()) @@ -795,14 +886,18 @@ def main() -> int: contextlib.redirect_stdout(quiet), contextlib.redirect_stderr(quiet), ): - # Import/configure the locked numerical dependency within the same - # audit boundary, before any maintained project module is loaded. - import torch + # Keep real locked Torch/thread controls under the same audit + # boundary, explicitly taking its optional CUDA bindings fallback. + with cpu_only_torch_import(bootstrap): + import torch + verify_cpu_only_torch_import(bootstrap) torch.set_num_threads(1) torch.set_num_interop_threads(1) controls = thread_controls(torch) - payloads = derive(root, owned, run, before, installed, torch, controls) + payloads = derive( + root, owned, run, before, installed, torch, controls, bootstrap + ) require(not refusals, "BOUNDARY_REFUSAL") exit_code = 0 except Exception as error: @@ -814,6 +909,7 @@ def main() -> int: "status": "refused", "phase": phase, "code": code, + "dependency_bootstrap": bootstrap, "first_boundary_refusal": ( json.loads(first_refusal[0]) if first_refusal else None ), @@ -852,6 +948,7 @@ def main() -> int: "status": "refused", "phase": "publish", "code": "OUTPUT_REFUSED", + "dependency_bootstrap": bootstrap, "first_boundary_refusal": ( json.loads(first_refusal[0]) if first_refusal else None ), From 290082a86050373383a7c79f0cd21dfe90825017 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:34:39 -0400 Subject: [PATCH 053/255] Document adopted validation optimization and remaining boundary proposals --- docs/validation-cost-and-borrow-boundaries.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/validation-cost-and-borrow-boundaries.md diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md new file mode 100644 index 000000000..88441dbfe --- /dev/null +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -0,0 +1,39 @@ +# Validation cost and borrow boundaries + +The US graph must detect changed sources, substituted artifacts and changes to retained Populations without repeating the same expensive work at every nested call. This note records the current decision and the proposals still under review. It does not change dataset release requirements. + +## Adopted: reuse source compilation + +The ACS native coverage owner now keeps a bounded process-local cache of compiled code. Its key includes exact source bytes, filename, mode, flags, inheritance setting, effective optimization level and compiler identity. Every source read, AST check and loaded-function, global, alias and closure check remains fresh. Cached code is neither source authority nor a portable receipt. + +The change passes 25 focused controls and the original invented financial fixture, including cold execution, required replay and final checks. One paired profile took 216.82 wall seconds versus 308.26 seconds before the change, a 29.66% reduction. CPU time fell from 303.90 to 214.69 seconds. These instrumented observations establish an improvement for that fixture; they do not establish native-data performance. See [the experiment](../experiments/us-acs-compilation-cache-adoption-20260910.json) for exact source, test and verification identities. + +The native pilot already in progress retains its original frozen source snapshot. + +## Next: remove duplicate work within a check + +A source review confirmed nine direct Frame storage traversals in a healthy `verify_acs_native_coverage` call: three calls to `_verify_frame`, each performing three storage hashes. Some repetitions occur across supplemental axis and resolved-weight checks, so collapsing all nine into three is not automatically equivalent under the existing mutation checks. + +The next candidate will share the first storage calculation with the supplemental fingerprint calculation, while preserving the independent final prepared-Frame storage pass and all three outer verification points. That would reduce nine direct traversals to six. It needs a regression that mutates a cell during the supplemental check, unchanged receipt and fingerprint comparisons, and a paired measurement before adoption. + +## Keep separate: code, files and retained objects + +Three different facts currently appear together in nested validation: + +- Loaded-code identity: the interpreter's functions, code, globals, aliases and closures. +- Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. +- Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. + +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. That is a proposed contract change, not an adopted optimization. Moving file checks would change when an altered archive is detected. It requires a documented public-entry contract and tests at every relevant callback, file borrow and emission boundary before implementation. + +Similarly, an AST cache would hold mutable objects, unlike compiled code. It needs its own ownership design and evidence; the compilation-cache result does not establish its safety or benefit. + +## Composition review + +The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. + +The immediate implementation priorities remain the budget returned-view correction, actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. + +## Review scope + +Fable 5.1 completed a read-only source review on September 10, 2026, using the saved Hivesight login. Its requested scratch-cache read was denied and that limitation was disclosed in its result. It supplied architectural advice, not runtime approval or a release verdict. The recommendations above are distinguished from independently verified source findings and adopted changes. From a8368ebfec2d9358249b9fad684d3f0767d48759 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 03:54:51 -0400 Subject: [PATCH 054/255] Expose bounded diagnostic failures without changing build permissions --- .github/workflows/test.yml | 3 +- .../test_spec_seed_identity_outer_failure.py | 107 +++++++++++++++ tools/spec_seed_identity_diagnostics.py | 129 +++++++++++++++++- 3 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58c73cf6d..ffcbaa81c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -226,7 +226,7 @@ jobs: python-version: "3.14.4" - name: Sync workspace with US and UK engines run: uv sync --all-packages --locked --extra us --extra uk - - name: Check CPU-only diagnostic bootstrap + - name: Check diagnostic bootstrap and failure reporting timeout-minutes: 2 env: PYTHONDONTWRITEBYTECODE: "1" @@ -234,6 +234,7 @@ jobs: run: >- uv run --no-sync pytest --noconftest -p no:cacheprovider -q packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics timeout-minutes: 15 diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py new file mode 100644 index 000000000..a24788b54 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py @@ -0,0 +1,107 @@ +"""Formatting-only outer-failure controls; no main/audit/dependency execution.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("outer_failure_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("kind", [TypeError, PermissionError]) +def test_outer_error_keeps_fixed_type_and_code_coordinates_without_message( + diagnostic, kind +): + try: + raise kind("secret-password.csv user-input-payload") + except kind as error: + payload = diagnostic.outer_failure_context(error) + value = json.loads(payload) + assert value["exception_type"] == kind.__name__ + assert value["code"] == "OUTER_FAILURE" + assert value["completed"] is value["coverage_pass"] is False + assert value["artifact_status_not_modified"] is True + assert b"secret-password" not in payload + assert b"user-input-payload" not in payload + assert len(payload) <= 8192 + + +def test_actual_short_named_traceback_frame_is_retained(diagnostic): + def cleanup_probe(): + raise TypeError("not emitted") + + try: + cleanup_probe() + except TypeError as error: + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["traceback_code_frames"][-1]["function"] == "cleanup_probe" + assert result["traceback_code_frames"][-1]["line"] > 0 + + +def test_fixed_refusal_retains_attached_sanitized_context(diagnostic): + context = {"code": "READ_SCOPE", "event": "open", "requested": {"path": None}} + error = diagnostic.RefusalError( + "READ_SCOPE", boundary_context=diagnostic.encoded(context) + ) + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["exception_type"] == "RefusalError" + assert result["code"] == "READ_SCOPE" + assert result["boundary_refusal"] == context + + +def test_unknown_refusal_and_exception_names_are_fixed_labels(diagnostic): + class SecretNamedError(Exception): + pass + + errors = ( + diagnostic.RefusalError("arbitrary-secret-code"), + SecretNamedError("secret-message"), + ) + for error in errors: + payload = diagnostic.outer_failure_context(error) + assert b"secret" not in payload.lower() + assert b"SecretNamedError" not in payload + result = json.loads(payload) + assert result["code"] in {"OUTER_FAILURE", "OTHER_FIXED_REFUSAL"} + + +def test_traceback_scan_and_output_are_bounded(diagnostic): + def recurse(depth): + if depth: + recurse(depth - 1) + else: + raise ValueError("unprinted") + + try: + recurse(40) + except ValueError as error: + payload = diagnostic.outer_failure_context(error) + result = json.loads(payload) + assert result["traceback_truncated"] is True + assert len(result["traceback_code_frames"]) <= 8 + assert len(payload) <= 8192 + + +def test_exception_only_metadata_does_not_open_or_resolve_files( + diagnostic, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("outer report attempted file I/O or resolution") + + monkeypatch.setattr(Path, "read_bytes", forbidden) + monkeypatch.setattr(Path, "read_text", forbidden) + monkeypatch.setattr(Path, "resolve", forbidden) + monkeypatch.setattr(Path, "stat", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + payload = diagnostic.outer_failure_context(TypeError("no traceback")) + assert json.loads(payload)["traceback_code_frames"] == [] diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 15aa3550e..b34ba6f2b 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -970,10 +970,133 @@ def main() -> int: return exit_code +def outer_failure_context(error: Exception) -> bytes: + """Record fixed exception labels and code coordinates without file I/O. + + This is stderr-only evidence. In particular, failed fixture cleanup must + leave the initial incomplete artifact intact rather than publish success. + """ + labels = { + "AssertionError", + "AttributeError", + "FileNotFoundError", + "ImportError", + "IndexError", + "IsADirectoryError", + "KeyError", + "MemoryError", + "ModuleNotFoundError", + "NameError", + "NotADirectoryError", + "OSError", + "OverflowError", + "PermissionError", + "RecursionError", + "RuntimeError", + "TypeError", + "UnboundLocalError", + "UnicodeDecodeError", + "ValueError", + } + kind = type(error).__name__ + exception_type = ( + kind + if type(error).__module__ == "builtins" and kind in labels + else "other_exception" + ) + code, boundary = "OUTER_FAILURE", None + if type(error) is RefusalError: + exception_type = "RefusalError" + value = error.args[0] if len(error.args) == 1 else None + code = ( + value + if value + in { + "READ_SCOPE", + "WRITE_SCOPE", + "DATA_FILE", + "FILESYSTEM_PATH", + "FILESYSTEM_DESCRIPTOR", + "FILESYSTEM_LINK", + "NETWORK_OR_CHILD", + "WALL", + "OUTPUT_CHANGED", + "THREADS_CHANGED", + "BOUNDARY_REFUSAL", + "OUTPUT_CAP", + "OUTPUT_FILE", + "OUTPUT_FILES", + "OUTPUT_ROSTER", + "OUTPUT_SEAL", + "OUTPUT_PART", + "OUTPUT_STATUS", + "OUTPUT_ALREADY_COMPLETE", + } + else "OTHER_FIXED_REFUSAL" + ) + if error.boundary_context is not None: + boundary = json.loads(error.boundary_context) + roots = ( + ("repository", Path(__file__).absolute().parents[1]), + ("environment", Path(sys.prefix)), + ("base_environment", Path(sys.base_prefix)), + ("system_usr", Path("/usr")), + ("system_opt", Path("/opt")), + ("system_lib", Path("/lib")), + ) + frames, visited = [], 0 + trace = error.__traceback__ + while trace is not None and visited < 32: + visited += 1 + code_object = trace.tb_frame.f_code + filename = refusal_path_context(code_object.co_filename, roots) + name = code_object.co_name + if filename.get("path") is not None and ( + re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", name) + or name in {"", "", "", "", ""} + ): + frames.append({"file": filename, "function": name, "line": trace.tb_lineno}) + frames = frames[-8:] + trace = trace.tb_next + truncated = trace is not None or visited > 8 + del trace + result = { + "diagnostic_outer_failure": True, + "completed": False, + "coverage_pass": False, + "exception_type": exception_type, + "code": code, + "boundary_refusal": boundary, + "traceback_code_frames": frames, + "traceback_truncated": truncated, + "artifact_status_not_modified": True, + "message_locals_source_lines_emitted": False, + } + payload = encoded(result) + if len(payload) > 8192: + payload = encoded( + { + "diagnostic_outer_failure": True, + "completed": False, + "coverage_pass": False, + "exception_type": exception_type, + "code": code, + "context": "size_bound", + "artifact_status_not_modified": True, + } + ) + return payload + + if __name__ == "__main__": try: raise SystemExit(main()) - except Exception: - # Bootstrap failures have no locals, paths or exception text in CI logs. - print("spec-seed diagnostics: bootstrap refused", file=sys.stderr) + except Exception as error: + # No second publication after failed cleanup. Emit only bounded metadata; + # never exception messages, locals, source lines or an unfiltered trace. + try: + context = outer_failure_context(error) + except Exception: + context = b'{"diagnostic_outer_failure":true,"completed":false,"context":"unavailable"}\n' + print(context.decode("ascii").rstrip("\n"), file=sys.stderr) raise SystemExit(1) from None From df10f246b45b11680a63dbe1151e5cd65ee659f4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 04:09:47 -0400 Subject: [PATCH 055/255] Bind financial successors to issued runs and seal returned budget views --- docs/validation-cost-and-borrow-boundaries.md | 6 +- ...get-detached-view-correction-20260910.json | 39 +++ .../graph_atomic_survey_financial.py | 326 ++++++++++++++++++ .../us_runtime/survey_financial_successor.py | 254 ++++++++++++++ .../build/us_runtime/survey_origin_budget.py | 251 ++++++++++++-- .../test_us_survey_financial_successor.py | 309 +++++++++++++++++ ..._us_survey_origin_budget_detached_views.py | 207 +++++++++++ 7 files changed, 1369 insertions(+), 23 deletions(-) create mode 100644 experiments/us-budget-detached-view-correction-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py create mode 100644 packages/microcosm-build/tests/test_us_survey_financial_successor.py create mode 100644 packages/microcosm-build/tests/test_us_survey_origin_budget_detached_views.py diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md index 88441dbfe..f88c2cf4c 100644 --- a/docs/validation-cost-and-borrow-boundaries.md +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -32,7 +32,11 @@ Similarly, an AST cache would hold mutable objects, unlike compiled code. It nee The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. -The immediate implementation priorities remain the budget returned-view correction, actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. +The budget returned-view correction is now adopted. Detached documents and grouped bounds are constructed after the last borrowed I/O, then checked against the issued payload and retained owners. Four targeted controls passed, including nine callback branches; the post-run verification rechecked 501 source/owned files, 5,983 model-code files and 12 resources. The earlier eight financial-successor controls remain evidence for their original source revision. These are separate component runs, not twelve tests of one integrated revision. See [the correction experiment](../experiments/us-budget-detached-view-correction-20260910.json). + +A second Fable source review found no confirmed correctness or ownership bug in the public PUF recipient interface. It recommended explicit cold-cache reuse assertions, exact comparisons with the original issuers, and clearer documentation of the original-source reads required by the recipient kernels. The frozen two-positive pilot remains unchanged; those additions belong to subsequent controls. + +The immediate implementation priorities remain actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. ## Review scope diff --git a/experiments/us-budget-detached-view-correction-20260910.json b/experiments/us-budget-detached-view-correction-20260910.json new file mode 100644 index 000000000..894069e3b --- /dev/null +++ b/experiments/us-budget-detached-view-correction-20260910.json @@ -0,0 +1,39 @@ +{ + "adopted": true, + "behavior": "Move detached checked-view construction after last borrowed I/O; seal fresh document and complete maintained GroupedUpperBounds state, including stable-ID buckets, canonical digest and immutable storage, against issued payload and pure owner state.", + "budget_owner_sha256": "e1faffa1b133878b3607e888fe0c1fe0503f23d46be52eeddc054cf6c28a1c2c", + "callback_branches": { + "new_detached_values": 7, + "old_population_mutations": 2 + }, + "cpu_seconds": 540.759706, + "guard_sha256": "8c0b29efd7dc6c272b94463c996a9a6d35bb81c16857046bc75f95d11fb69087", + "historical_original_eight": { + "budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "positive_postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "remaining_postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486" + }, + "native_acceptance": false, + "new_test_sha256": "4a8078e93d95cd6c807218fe021128614ce0567490ef216b7c83555a25b65a0e", + "peak_rss_bytes": 452624384, + "performance_claim": false, + "public_recipient_pilot_uses_original_frozen_budget": true, + "release_acceptance": false, + "root_postcheck_sha256": "2f94929c903c86633c199d3570625dbfda29fe57a84dab9b52502b25fe1236ed", + "scope": "Original eight financial-successor controls and four corrected budget-view controls are separate versioned runs; this is not whole-tree or native certification.", + "source_candidate_peer_sha256": "5f4ef9c2ecb9b19803b3439ec26b0bfdd92b19acc09e1579a6cc216240d35801", + "source_map_sha256": "9f5858b3bd7aceb9e9ee5ca0a2b5b4033ab9356a845364e9b07a11373de75b5f", + "status": "adopted_after_exact_component_verification", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 4 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 501 + }, + "wall_seconds": 543.4429369170102 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py index fb2c55b1d..fbfaa9081 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -9,6 +9,7 @@ import json import sys +import weakref from dataclasses import dataclass, replace from types import FunctionType, SimpleNamespace @@ -31,6 +32,8 @@ survey = atomic.survey reconstruction = atomic.reconstruction require = values.require +RUN_PROTOCOL = "microcosm.us.atomic-survey-financial-run.v1" +_ISSUED_RUNS = {} @dataclass(frozen=True) @@ -47,6 +50,316 @@ class AtomicSurveyFinancialRunValues: projection: bytes matrix: bytes + def checked_view(self): + """Recheck a completed actual run; public dataclass copies are unissued.""" + return check_atomic_survey_financial_run(self) + + +@dataclass(frozen=True) +class CheckedAtomicSurveyFinancialRun: + """Descriptive values; authority remains in the actual issued run handle.""" + + payload: bytes + digest: str + population: Population + + +@dataclass(frozen=True) +class _FinancialRunState: + prefix: object + prefix_objects: tuple + preparation_entry: tuple + financial_population: Population + populations: tuple + manifest: object + manifest_bytes: bytes + prefix_manifest_bytes: bytes + compiled: object + declaration: str + prefix_declaration: str + store: object + kernels: object + source_items: tuple + config_bytes: bytes + projection: bytes + matrix: bytes + pins: bytes + n_estimators: int + demographic_conditioning: bool + source_keys: tuple + keys: tuple + implementations: tuple + artifact_hashes: tuple + manifest_populations: tuple + prefix_manifest_populations: tuple + live: dict + + +def _manifest_population_seals(manifest, compiled): + """Include transient attached Frames, which portable JSON deliberately omits.""" + return tuple( + ( + version, + reconstruction._population_stamp( + Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ) + ), + ) + for version in sorted(set(compiled.versions.values())) + ) + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is AtomicSurveyFinancialRunValues + and entry is not None + and entry[0]() is run, + "UNISSUED_FINANCIAL_RUN", + ) + return entry + + +def _run_document(run, state): + """Portable ancestry omits timing/cache hits and private physical seals.""" + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(state.declaration.encode()), + "manifest_key": state.manifest.key, + "preparation_sha256": codec.sha(state.preparation_entry[1]), + "geography_config_sha256": codec.sha(state.config_bytes), + "projection_sha256": codec.sha(state.projection), + "matrix_sha256": codec.sha(state.matrix), + "host_edges": codec.decode_json(state.pins), + "node_keys": dict(state.keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "financial_frame_sha256": values.source._frame_identity( + run.financial_population.frame + ), + "financial_version": run.financial_population.version, + "financial_owners": sorted( + (e, c, writer) + for (e, c), writer in run.financial_population.owners.items() + ), + "owned_columns": list(values.OUTPUTS), + "demographic_conditioning": state.demographic_conditioning, + "n_estimators": state.n_estimators, + "release_eligible": False, + } + ) + + +def _pure_run(run, entry): + """Check retained graph/source/Population state after all external I/O.""" + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + state = entry[2] + prefix = state.prefix + require( + run.prefix is prefix + and run.financial_population is state.financial_population + and run.manifest is state.manifest + and run.compiled is state.compiled + and run.store is state.store + and run.kernels is state.kernels + and run.projection == state.projection + and run.matrix == state.matrix + and tuple(sorted(run.sources.items())) == state.source_items + and tuple(sorted(prefix.sources.items())) == state.source_items + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + state.prefix_objects, + strict=True, + ) + ) + and values.host.survey_budget._config_payload(prefix.geography_config) + == state.config_bytes + and run.manifest.to_json_bytes() == state.manifest_bytes + and prefix.manifest.to_json_bytes() == state.prefix_manifest_bytes + and graph_to_json(run.compiled.graph) == state.declaration + and graph_to_json(prefix.compiled.graph) == state.prefix_declaration + and run.compiled == compile_graph(run.compiled.graph) + and prefix.compiled == compile_graph(prefix.compiled.graph), + "FINANCIAL_RUN_BINDINGS_CHANGED", + ) + require( + _manifest_population_seals(run.manifest, run.compiled) + == state.manifest_populations + and _manifest_population_seals(prefix.manifest, prefix.compiled) + == state.prefix_manifest_populations, + "FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED", + ) + require( + values.source._ISSUED.get(id(prefix.preparation)) is state.preparation_entry + and prefix.preparation.payload == state.preparation_entry[1], + "FINANCIAL_RUN_SOURCE_CHANGED", + ) + values.source._pure_final(state.preparation_entry[2]) + for name, population, stamp in state.populations: + actual = ( + run.financial_population if name == "financial" else getattr(prefix, name) + ) + require( + actual is population and reconstruction._population_stamp(actual) == stamp, + "FINANCIAL_RUN_POPULATION_CHANGED", + ) + require( + _run_document(run, state) == entry[1] and _live() == state.live, + "FINAL_FINANCIAL_RUN_SEAL", + ) + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + + +def check_atomic_survey_financial_run(run): + """Requalify existing artifacts and live owners without fitting or execution.""" + entry = _run_entry(run) + _pure_run(run, entry) + state, prefix = entry[2], entry[2].prefix + _, source_keys = _source_paths_and_keys(run.compiled, run.sources, run.store) + keys, implementations = _all_node_keys(run.compiled, run.kernels, source_keys) + require( + tuple(sorted(source_keys.items())) == state.source_keys + and tuple(sorted(keys.items())) == state.keys + and tuple(sorted(implementations.items())) == state.implementations, + "FINANCIAL_RUN_IMPLEMENTATIONS_CHANGED", + ) + loaded = _artifacts( + run.manifest, run.compiled, run.store, run.kernels, keys, implementations + ) + require( + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ) + == state.artifact_hashes, + "FINANCIAL_RUN_ARTIFACT_CHANGED", + ) + # These exact fitted artifacts were checked at actual execution/required + # replay before issuance. Rechecking their identities needs no new pickle + # decode or fit; the materialized verifier freshly derives current values. + financial.verify_materialized_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + population=run.financial_population, + projection=state.projection, + matrix=state.matrix, + matrix_producer_key=keys[financial.PROJECTION_NODE], + raw_draws=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ), + apply_states=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ), + host_pins=codec.decode_json(state.pins), + n_estimators=state.n_estimators, + demographic_conditioning=state.demographic_conditioning, + geography_config=prefix.geography_config, + ) + result = CheckedAtomicSurveyFinancialRun( + entry[1], codec.sha(entry[1]), run.financial_population + ) + _pure_run(run, entry) + return result + + +def _issue_run( + result, + *, + preparation_entry, + pins, + n_estimators, + demographic_conditioning, + source_keys, + keys, + implementations, + loaded, + live, +): + """Called only after this runner's complete materialization/replay checks.""" + prefix = result.prefix + populations = tuple( + (name, population, reconstruction._population_stamp(population)) + for name, population in ( + *( + (name, getattr(prefix, name)) + for name in ( + "allocated_population", + "geography_population", + "clone_population", + ) + ), + ("financial", result.financial_population), + ) + ) + state = _FinancialRunState( + prefix, + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + preparation_entry, + result.financial_population, + populations, + result.manifest, + result.manifest.to_json_bytes(), + prefix.manifest.to_json_bytes(), + result.compiled, + graph_to_json(result.compiled.graph), + graph_to_json(prefix.compiled.graph), + result.store, + result.kernels, + tuple(sorted(result.sources.items())), + prefix.geography_config.to_bytes(), + result.projection, + result.matrix, + codec.encode_json(pins), + n_estimators, + demographic_conditioning, + tuple(sorted(source_keys.items())), + tuple(sorted(keys.items())), + tuple(sorted(implementations.items())), + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ), + _manifest_population_seals(result.manifest, result.compiled), + _manifest_population_seals(prefix.manifest, prefix.compiled), + live, + ) + identifier = id(result) + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED_RUNS[identifier] = (reference, _run_document(result, state), state) + _pure_run(result, _run_entry(result)) + def _live(): """Pure final fence over this composition and its existing owner closure.""" @@ -80,6 +393,7 @@ def _live(): ) result["financial_contract"] = values.source._runtime_marker( ( + RUN_PROTOCOL, values.FEATURES, values.DEMOGRAPHIC_FEATURES, values.TARGETS, @@ -553,4 +867,16 @@ def observe(node_id, population): "ATOMIC_FINAL_LEDGER", ) survey._check_node_states(manifest, states) + _issue_run( + result, + preparation_entry=entry, + pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + source_keys=source_keys, + keys=keys, + implementations=implementations, + loaded=loaded, + live=live, + ) return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py new file mode 100644 index 000000000..c689b746f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py @@ -0,0 +1,254 @@ +"""Admit a checked financial graph result under its original sampling budget. + +This narrow IMPORTANCE-to-IMPORTANCE binding changes only the seven current +financial leaves authenticated by the actual twenty-node runner. It neither +changes the original budget's initial population nor admits arbitrary Frames. + +Age composition still needs an explicit dependency on financial ATTACH and a +checked binding to the actual receiving graph's Population before calibration +admission. This module does not supply those graph edges or a new age runner. +""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass + +import numpy as np + +from microcosm.graph import ArtifactType + +PROTOCOL = "microcosm.us.sampling-origin-financial-successor.v1" +FINANCIAL_SUCCESSOR_TYPE = ArtifactType( + "microcosm.us.sampling_origin_financial_successor", 1 +) +MAX_PAYLOAD_BYTES = 256 * 1024 +_ISSUED = {} + + +class SurveyFinancialSuccessorError(ValueError): + """A financial result lacks its exact original budget/run ancestry.""" + + +def _require(condition, reason): + if not condition: + raise SurveyFinancialSuccessorError("SURVEY_FINANCIAL_SUCCESSOR_" + reason) + + +def _entry(value): + entry = _ISSUED.get(id(value)) + _require( + type(value) is SamplingOriginFinancialSuccessor + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +@dataclass(frozen=True) +class _State: + budget: object + budget_entry: tuple + financial_run: object + run_entry: tuple + previous: object + current: object + previous_identity: tuple + current_identity: tuple + + +def _document(state, budget_view, run_view): + # Local imports avoid a budget -> financial owner -> runner -> host cycle. + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + run, prefix = state.financial_run, state.financial_run.prefix + original = state.budget_entry[2] + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry + and runner._run_entry(run) is state.run_entry + and original.preparation is prefix.preparation + and original.allocated is prefix.allocated_population + and original.expanded is prefix.clone_population + and original.geography_config is prefix.geography_config + and budget_view.preparation is prefix.preparation + and budget_view.allocated_population is prefix.allocated_population + and budget_view.initial_population is state.previous is prefix.clone_population + and budget_view.payload == state.budget_entry[1] + and budget_view.digest == budget._sha(state.budget_entry[1]) + and run_view.population is state.current is run.financial_population + and run_view.payload == state.run_entry[1] + and run_view.digest == budget._sha(state.run_entry[1]), + "ORIGINAL_BUDGET_RUN_ANCESTRY", + ) + previous, current = state.previous, state.current + _require( + previous.version == current.version == runner.atomic.clone.COMBINED_CLONE_NODE + and current.mass_ledger == previous.mass_ledger + and current.weight_kind == previous.weight_kind + and np.array_equal( + current.frame.weights_for("household").values, + previous.frame.weights_for("household").values, + ), + "UNCHANGED_SAMPLING_MASS", + ) + budget.graph._check_design_anchors(current, previous.design_weights["household"]) + owners = { + **previous.owners, + **{ + ("person", column): runner.financial.ATTACH_NODE + for column in runner.values.OUTPUTS + }, + } + budget.graph._check_population_state( + current, + version=previous.version, + owners=owners, + kind=budget.WeightKind.IMPORTANCE, + ledger=previous.mass_ledger, + ) + # The checked run freshly compares the complete result to source-derived + # current values and its pinned raw draws; no caller column table is used. + return budget._json( + { + "protocol": PROTOCOL, + "budget_sha256": budget._sha(state.budget_entry[1]), + "financial_run_sha256": budget._sha(state.run_entry[1]), + "financial_run": json.loads(state.run_entry[1]), + "previous_version": previous.version, + "current_version": current.version, + "previous_frame_sha256": budget.source._frame_identity(previous.frame), + "current_frame_sha256": budget.source._frame_identity(current.frame), + "owned_columns": list(runner.values.OUTPUTS), + "sampling_bounds_changed": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_state(state): + """Final retained-owner checks; never opens an artifact or source file.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry, + "FINAL_BUDGET_ISSUANCE", + ) + runner._pure_run(state.financial_run, state.run_entry) + _require( + budget._population_identity(state.previous) == state.previous_identity + and budget._population_identity(state.current) == state.current_identity, + "FINAL_POPULATION_SEAL", + ) + _require(budget._live() == budget._LIVE, "FINAL_PRODUCER_SEAL") + + +def _final_state(state): + from . import survey_origin_budget as budget + + # Complete the last budget/support I/O before the financial owner's pure + # seal, so that borrow cannot invalidate the already-checked financial run. + budget._final_budget_state(state.budget_entry[2]) + _pure_state(state) + + +@dataclass(frozen=True) +class SamplingOriginFinancialSuccessor: + payload: bytes + + def __post_init__(self): + raise SurveyFinancialSuccessorError("NO_PUBLIC_FINANCIAL_SUCCESSOR_CONSTRUCTOR") + + def checked_view(self): + from . import graph_atomic_survey_financial as runner + + entry = _entry(self) + state = entry[2] + _pure_state(state) + budget_view = state.budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(state.financial_run) + _require(_document(state, budget_view, run_view) == entry[1], "RECONSTRUCTION") + _final_state(state) + _require(_entry(self) is entry, "FINAL_ISSUANCE") + # Do not expose a detached view to the final support-borrow callback. + # All owner/file I/O and retained-state seals are complete above. + return CheckedSamplingOriginFinancialSuccessor( + entry[1], + runner.codec.sha(entry[1]), + state.budget, + state.financial_run, + state.previous, + state.current, + ) + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginFinancialSuccessor: + """Values borrowed from a checked handle, never decoded authority.""" + + payload: bytes + digest: str + budget: object + financial_run: object + previous: object + current: object + + +def admit_survey_financial_population(budget, *, financial_run, candidate=None): + """Admit only a live completed graph run under its own original budget.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budgets + + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), + "CANDIDATE_TYPE_OR_BOUND", + ) + # An unissued descriptive dataclass must fail before any owner/file borrow. + run_entry = runner._run_entry(financial_run) + budget_entry = budgets._entry(budget, budgets.SamplingOriginBudget) + budget_view = budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(financial_run) + state = _State( + budget, + budget_entry, + financial_run, + run_entry, + budget_entry[2].expanded, + financial_run.financial_population, + budgets._population_identity(budget_entry[2].expanded), + budgets._population_identity(financial_run.financial_population), + ) + payload = _document(state, budget_view, run_view) + _require(len(payload) <= MAX_PAYLOAD_BYTES, "PAYLOAD_BOUND") + _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") + result = object.__new__(SamplingOriginFinancialSuccessor) + object.__setattr__(result, "payload", payload) + identifier = id(result) + + def forget(reference): + entry = _ISSUED.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED[identifier] = (reference, payload, state) + entry = _entry(result) + _final_state(state) + _require(_entry(result) is entry, "FINAL_ISSUANCE") + return result + + +def verify_survey_financial_successor(binding): + _entry(binding) + binding.checked_view() + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py index ef15217ca..00ac974bc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -42,6 +42,7 @@ from . import graph_combined_clone as clone from . import graph_survey_population as graph from . import survey_atomic_geography as geography +from . import survey_financial_successor as financial_successor from . import survey_population_preparation as source from .graph_sources import frame_column_declarations from .support_provenance import support_clone_index_column, support_source_id_column @@ -138,6 +139,7 @@ def _modules(): sys.modules[support_source_id_column.__module__], sys.modules[clone.clone_us_frame_for_puf_support.__module__], geography, + financial_successor, geography.blocks, geography.atomic, geography.atomic_graph, @@ -224,6 +226,12 @@ def _live(): ), ) ) + result["financial_successor_contract"] = ( + financial_successor.PROTOCOL, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.name, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.schema_version, + financial_successor.MAX_PAYLOAD_BYTES, + ) return result @@ -623,6 +631,11 @@ def _final_budget_state(state): # Finish support I/O before the same pure owner/Population final seals. # Re-reading exact pinned bytes grants no publisher provenance. geography._read_support(state.geography_config) + _pure_budget_state(state) + + +def _pure_budget_state(state): + """Check retained source/Population state without a new file borrow.""" entry = _preparation_entry( state.preparation, state.preparation_payload, state.preparation_entry ) @@ -734,6 +747,71 @@ def forget(reference): return value +def _check_view_constraint(constraint, document): + """Bind every derived numerical field to an already requalified issued view.""" + reason = "FINAL_BUDGET_VIEW_CONSTRAINT" + _require(type(constraint) is group_bounds.GroupedUpperBounds, reason) + ids = tuple(document["household_ids"]) + groups = document["group_indices"] + bounds_hex = [row["upper_float64_hex"] for row in document["origins"]] + # This is the maintained constraint's stable-ID reduction order. These + # expected buckets come from the issued document, not constructor output. + buckets = [[] for _ in bounds_hex] + for index in sorted( + range(len(ids)), key=lambda i: (isinstance(ids[i], str), ids[i]) + ): + buckets[groups[index]].append(index) + + def immutable_array(value, expected, dtype): + payload = np.asarray(expected, dtype=dtype).tobytes() + return ( + type(value) is np.ndarray + and value.dtype.str == dtype + and value.shape == (len(expected),) + and value.strides == (8,) + and not value.flags.writeable + and not value.flags.owndata + and type(value.base) is bytes + and value.base == payload + and value.tobytes() == payload + ) + + digest_payload = { + "contract": "stable-household-fsum-group-upper-v1", + "household_ids": ids, + "group_indices": groups, + "absolute_bounds_hex": bounds_hex, + } + expected_digest = _sha( + json.dumps( + digest_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + ) + _require( + type(constraint.household_ids) is tuple + and len(constraint.household_ids) == len(ids) + and all( + type(actual) is type(expected) and actual == expected + for actual, expected in zip(constraint.household_ids, ids, strict=True) + ) + and immutable_array(constraint.group_indices, groups, " Date: Thu, 10 Sep 2026 04:10:08 -0400 Subject: [PATCH 056/255] Retain diagnostic scratch space for runner cleanup --- .github/workflows/test.yml | 1 + .../test_spec_seed_identity_owned_temp.py | 60 +++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 27 +++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffcbaa81c..3d0279ae7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -235,6 +235,7 @@ jobs: uv run --no-sync pytest --noconftest -p no:cacheprovider -q packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics timeout-minutes: 15 diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py new file mode 100644 index 000000000..2fe9d1739 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py @@ -0,0 +1,60 @@ +"""Owned temporary-directory controls; no audit, main, Torch or Microcosm run.""" + +import importlib.util +import stat +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("owned_temp_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("refuses", [False, True]) +def test_owned_fixture_survives_success_and_exact_refusal( + diagnostic, tmp_path, refuses +): + context = b'{"code":"READ_SCOPE","event":"open"}\n' + error = diagnostic.RefusalError("READ_SCOPE", boundary_context=context) + caught = None + try: + with diagnostic.retained_owned_directory(tmp_path) as owned: + assert owned.parent == tmp_path + assert owned.name.startswith("spec-seed-owned-") + assert stat.S_IMODE(owned.stat().st_mode) == 0o700 + fixture = owned / "invented-spec.txt" + fixture.write_text("invented fixture only") + if refuses: + raise error + except diagnostic.RefusalError as actual: + caught = actual + assert caught is (error if refuses else None) + if caught is not None: + assert caught.boundary_context is context + assert fixture.read_text() == "invented fixture only" + assert owned.is_dir() + + +def test_each_workspace_is_distinct_and_has_no_implicit_cleanup( + diagnostic, tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("implicit TemporaryDirectory cleanup was registered") + + monkeypatch.setattr(diagnostic.tempfile, "TemporaryDirectory", forbidden) + with diagnostic.retained_owned_directory(tmp_path) as first: + (first / "invented.txt").write_text("first") + with diagnostic.retained_owned_directory(tmp_path) as second: + (second / "invented.txt").write_text("second") + assert first != second + assert first.parent == second.parent == tmp_path + assert (first / "invented.txt").read_text() == "first" + assert (second / "invented.txt").read_text() == "second" diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index b34ba6f2b..9cfb2295f 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -820,6 +820,17 @@ def derive( return payloads +@contextlib.contextmanager +def retained_owned_directory(runner_temp: Path): + """Retain one private invented-fixture workspace for CI runner cleanup. + + CPython's descriptor-relative rmtree opens omit dir_fd in the audit event. + Keeping the workspace avoids weakening the diagnostic's path restrictions. + This directory is not one of the six uploaded diagnostic artifacts. + """ + yield Path(tempfile.mkdtemp(prefix="spec-seed-owned-", dir=runner_temp)) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-dir", required=True, type=Path) @@ -831,10 +842,7 @@ def main() -> int: require(not output.exists() and not output.is_symlink(), "OUTPUT_EXISTS") output.mkdir(mode=0o700) phase = "environment" - with tempfile.TemporaryDirectory( - prefix="spec-seed-owned-", dir=runner_temp - ) as temp: - owned = Path(temp) + with retained_owned_directory(runner_temp) as owned: require(sys.version_info[:3] == (3, 14, 4), "PYTHON") require(sys.platform == "linux", "PLATFORM") require(all(os.environ.get(name) == "1" for name in THREAD_ENV), "THREADS") @@ -873,6 +881,13 @@ def main() -> int: "cuda_bindings_import_attempts": 0, "temporary_finder_removed": True, "torch_optional_bindings_fallback_verified": False, + "owned_temporary_storage": { + "policy": "retain_private_directory_until_ci_runner_cleanup", + "parent": "RUNNER_TEMP", + "directories_per_invocation": 1, + "contents": "invented_spec_fixture_and_dependency_temporaries", + "uploaded": False, + }, } try: bootstrap["absent_stdlib_zip_omitted"] = omit_absent_stdlib_zip() @@ -926,8 +941,8 @@ def main() -> int: ) exit_code = 1 retained = tuple((name, payloads[name]) for name in CAPS) - # Fixture cleanup has finished. No borrowed source/artifact I/O follows the - # terminal publication; a killed cleanup therefore leaves incomplete status. + # The private fixture workspace remains for CI runner cleanup. No borrowed + # source/artifact I/O follows the terminal publication. require( tuple((name, payloads[name]) for name in CAPS) == retained, "OUTPUT_CHANGED" ) From ed96fb9272e3a99a24d5b853a9159afd4ae7e7b2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 04:19:40 -0400 Subject: [PATCH 057/255] Prepare PUF recipient routes from the issued financial survey graph --- ...5-public-recipient-positives-20260910.json | 48 ++ .../graph_puf55_survey_recipients.py | 292 +++++++++ .../us_runtime/puf55_survey_recipients.py | 574 ++++++++++++++++++ .../test_us_graph_puf55_survey_recipients.py | 258 ++++++++ .../tests/test_us_puf55_survey_recipients.py | 500 +++++++++++++++ 5 files changed, 1672 insertions(+) create mode 100644 experiments/us-puf55-public-recipient-positives-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py create mode 100644 packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py create mode 100644 packages/microcosm-build/tests/test_us_puf55_survey_recipients.py diff --git a/experiments/us-puf55-public-recipient-positives-20260910.json b/experiments/us-puf55-public-recipient-positives-20260910.json new file mode 100644 index 000000000..51cf6e145 --- /dev/null +++ b/experiments/us-puf55-public-recipient-positives-20260910.json @@ -0,0 +1,48 @@ +{ + "accepted_compilation_cache_overlay_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "actual_issued_recipient_positive": true, + "actual_twenty_two_node_cold_required_replay": true, + "cpu_seconds": 331.268576, + "fable_strengthened_issuer_comparisons_included": false, + "frozen_budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "guard_sha256": "45e7faf460d74210613e2898c4471d226a2b8d2e9f241d4c6945fcb7ec7cdb09", + "head": "640c1db49e6a70d47d58cea194e44df26c0762e5", + "junit_case_seconds": { + "test_actual_issued_financial_run_retains_source_reports_and_current_money": "104.759", + "test_actual_twenty_two_node_cold_required_replay_keeps_complete_population": "223.518" + }, + "junit_sha256": "de2befa8fe7f3878a681f4de9e4c81ad467d51fa7d75c33494258b0fc8c46d60", + "junit_timing_scope": "pytest per-case durations include assigned fixture setup/teardown; not isolated issuer or graph compute timings", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_teardowns_passed": true, + "original_twenty_node_cold_hit_assertion_present": false, + "peak_rss_bytes": 541048832, + "receipt_sha256": "ae02970da176ad92efacd831de8cf78e18ea1d96a811c124f2d268e6c5da71e4", + "release_acceptance": false, + "remaining_public_cases_executed": 0, + "remaining_public_controls": 24, + "resource_map_sha256": "c7e4193c952bd0d28d81db4371b51b51462870326fe59d5d26c13e4617f3e5b7", + "root_postcheck_sha256": "465b347efcc12e5f717a846207f2db1703b3c31648abefbe355d4274c0c61eab", + "scope": "2_invented_actual_public_puf55_recipient_issuer_and_graph_controls", + "scope_note": "Actual issuers and graph over invented survey originals; no native donor fitting, whole-cohort PUF finalization or enriched calibration.", + "selected_test_cases": [ + "test_us_puf55_recipients_public_positive.py::test_actual_issued_financial_run_retains_source_reports_and_current_money", + "test_us_puf55_recipients_public_positive.py::test_actual_twenty_two_node_cold_required_replay_keeps_complete_population" + ], + "source_count": 498, + "source_map_sha256": "504e045e6551895fabf8bc377fdfce1c69da9e70aa17989d2b771d5b66176c4b", + "status": "two_original_public_positives_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 333.96333820803557 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py new file mode 100644 index 000000000..1f328aa20 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py @@ -0,0 +1,292 @@ +"""Typed artifact-only PUF55 route preparation after current financial leaves. + +These two nodes perform no fit, attachment or structural/weight change. A host +must retain the actual financial run and verify materialized typed artifacts +on cold execution and replay; decoded projection receipts grant no authority. +""" + +from __future__ import annotations + +import sys + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + source_hash, +) +from microcosm.graph.keys import opaque_artifact_key + +from . import puf55_survey_recipients as values + +financial = values.financial +source_graph = financial.survey +host = financial.financial.host +codec, model_input = financial.codec, values.model_input +PROJECTION_NODE = "survey_puf55.recipient_projection" +MATRIX_NODE = "survey_puf55.recipient_matrices" +PROJECTION_TYPE = ArtifactType("microcosm.us.puf55_survey_recipient_projection", 1) +_NAMES = { + values.PROFILES[0].value: "matrix_nine", + values.PROFILES[1].value: "matrix_eight", +} + + +def _edges(): + return ( + ArtifactInput( + "preparation", + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + ), + ArtifactInput( + "financial_projection", + financial.financial.PROJECTION_NODE, + "projection", + financial.financial.PROJECTION_TYPE, + ), + ArtifactInput( + "financial_matrix", + financial.financial.PROJECTION_NODE, + "matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ), + ) + + +def _pins(run): + entry = financial._run_entry(run) + keys = dict(entry[2].keys) + hashes = {(node, name): digest for node, name, digest in entry[2].artifact_hashes} + return { + edge.name: { + "producer_key": keys[edge.producer], + "artifact_key": opaque_artifact_key(keys[edge.producer], edge.artifact), + "payload_sha256": hashes[edge.producer, edge.artifact], + } + for edge in _edges() + } + + +def _check_values(qualified): + values._require( + type(qualified) is values.Puf55SurveyRecipients, "GRAPH_VALUES_TYPE" + ) + values._result_stamp(qualified) + run = qualified.financial_run + entry = financial._run_entry(run) + financial._pure_run(run, entry) + document = codec.decode_json(qualified.receipt) + values._require( + document["protocol"] == values.PROTOCOL + and document["financial_run_sha256"] == codec.sha(entry[1]) + and document["preparation_sha256"] == codec.sha(entry[2].preparation_entry[1]) + and document["person_projection_sha256"] + == values._table_digest(qualified.person) + and document["tax_unit_projection_sha256"] + == values._table_digest(qualified.tax_unit) + and tuple(row["profile"] for row in document["routes"]) + == tuple(name for name, _ in qualified.matrices) + and tuple(row["matrix_sha256"] for row in document["routes"]) + == tuple(codec.sha(p) for _, p in qualified.matrices), + "GRAPH_VALUES_BINDING", + ) + names = tuple(name for name, _ in qualified.matrices) + values._require( + names and names == tuple(p.value for p in values.PROFILES if p.value in names), + "GRAPH_ROUTE_ROSTER", + ) + return entry, document + + +def puf55_survey_recipient_nodes(qualified): + """Bind expected projections; executing kernels retain/requalify the live run.""" + entry, _ = _check_values(qualified) + params = { + "protocol": values.PROTOCOL, + "financial_run_sha256": codec.sha(entry[1]), + "preparation_sha256": codec.sha(entry[2].preparation_entry[1]), + "projection_sha256": codec.sha(qualified.receipt), + "host_edges": codec.encode_json(_pins(qualified.financial_run)).decode(), + "route_matrices": codec.encode_json( + [[name, codec.sha(p)] for name, p in qualified.matrices] + ).decode(), + } + common = { + "population": entry[2].financial_population.version, + # Reading the seven financial-owned leaves creates the actual attach + # dependency, even though neither node owns any receiving cells. + "inputs": financial.financial._inputs(entry[2].financial_population.frame), + "params": params, + } + projection = Node( + PROJECTION_NODE, + Puf55SurveyRecipientProjectionKernel.ref, + **common, + artifact_inputs=_edges(), + artifact_outputs=(ArtifactOutput("projection", PROJECTION_TYPE),), + ) + matrix = Node( + MATRIX_NODE, + Puf55SurveyRecipientMatrixKernel.ref, + **common, + artifact_inputs=( + *_edges(), + ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE), + ), + artifact_outputs=tuple( + ArtifactOutput(_NAMES[name], model_input.RECIPIENT_MATRIX_TYPE) + for name, _ in qualified.matrices + ), + ) + return projection, matrix + + +class _Kernel(KernelBase): + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=financial.financial._Kernel.capabilities.dependencies, + ) + + def __init__(self, financial_run): + self._financial_run = financial_run + + def implementation_hash(self): + # Bind the actual current-money graph owner closure plus each additional + # numerical/source module. No receipt string stands in for source code. + return codec.sha( + codec.encode_json( + { + "financial": source_hash( + financial, dependencies=self.capabilities.dependencies + ), + "source": source_graph._Kernel.implementation_hash(self), + "recipient": source_hash( + sys.modules[__name__], + values, + values.ss, + values.ss.reports, + values.ss.reports.basis_owner, + values.puf_source, + values.full, + values.support, + values.provenance, + model_input, + *host.survey_budget._modules(), + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _context(self, context, qualified): + entry, _ = _check_values(qualified) + expected = {node.id: node for node in puf55_survey_recipient_nodes(qualified)} + values._require( + context.node.id in expected + and context.node == expected[context.node.id] + and dict(context.params) == dict(context.node.params), + "GRAPH_DECLARATION", + ) + values._require( + not context.sources + and set(context.artifacts) + == {e.name for e in context.node.artifact_inputs}, + "GRAPH_ARTIFACT_ROSTER", + ) + host._current_context_frame(context, entry[2].financial_population.frame) + payloads = { + "preparation": entry[2].preparation_entry[1], + "financial_projection": entry[2].projection, + "financial_matrix": entry[2].matrix, + } + for edge in _edges(): + artifact = host.shared.artifact(context, edge.name, edge.type) + values._require( + { + "producer_key": artifact.producer_key, + "artifact_key": artifact.key, + "payload_sha256": codec.sha(artifact.payload), + } + == _pins(qualified.financial_run)[edge.name] + and artifact.payload == payloads[edge.name], + "GRAPH_HOST_ARTIFACT", + ) + if context.node.id == MATRIX_NODE: + artifact = host.shared.artifact(context, "projection", PROJECTION_TYPE) + values._require( + artifact.payload == qualified.receipt, "GRAPH_PROJECTION_ARTIFACT" + ) + + def run(self, context): + run = self._financial_run + qualified = values.qualify_puf55_survey_recipients(run) + entry = financial._run_entry(run) + self._context(context, qualified) + outputs = ( + {"projection": qualified.receipt} + if context.node.id == PROJECTION_NODE + else {_NAMES[name]: payload for name, payload in qualified.matrices} + ) + output_seal = tuple(sorted(outputs.items())) + receipt = { + "protocol": values.PROTOCOL, + "projection_sha256": codec.sha(qualified.receipt), + "artifact_sha256": {n: codec.sha(p) for n, p in outputs.items()}, + "population_changed": False, + "release_eligible": False, + } + receipt_bytes = codec.encode_json(receipt) + result = KernelResult(artifacts=dict(outputs), receipt=receipt) + # Finish the live run's source/store/support I/O, then seal detached + # context/outputs and owners without another external read. + financial.check_atomic_survey_financial_run(run) + financial._pure_run(run, entry) + self._context(context, qualified) + values._require( + self._financial_run is run + and all( + type(name) is str and type(payload) is bytes + for name, payload in result.artifacts.items() + ) + and tuple(sorted(result.artifacts.items())) == output_seal + and codec.encode_json(result.receipt) == receipt_bytes + and not result.columns + and result.frame is result.keep is result.expand is result.weights is None + and result.strata is None, + "GRAPH_FINAL_RESULT", + ) + return result + + +class Puf55SurveyRecipientProjectionKernel(_Kernel): + ref = "us.survey_puf55.recipient_projection@1" + + +class Puf55SurveyRecipientMatrixKernel(_Kernel): + ref = "us.survey_puf55.recipient_matrices@1" + + +def verify_materialized_puf55_survey_recipients(financial_run, *, projection, matrices): + """Check host-pinned artifact bytes on replay; this is not an artifact issuer. + + The caller must first verify actual producer keys/types/implementation hashes + against its complete compiled graph and retained receiving Population. + """ + expected = values.qualify_puf55_survey_recipients(financial_run) + values._require( + type(projection) is bytes + and projection == expected.receipt + and type(matrices) is tuple + and matrices == expected.matrices, + "MATERIALIZED_ARTIFACTS", + ) + return expected diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py new file mode 100644 index 000000000..7f3965be2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py @@ -0,0 +1,574 @@ +"""Disjoint PUF55 recipient values from an actual, authenticated financial run. + +Preparation only: no donor, fit, attachment or Population admission. Retained +modeled return roles are not observed filers or a current-money reconstruction. +The six existing money features retain their all-member aggregation contract. +Source SS reports/components/masks are detached projections on both clone arms; +the receiving Frame and its canonical benefit cells are never modified. +""" + +from __future__ import annotations + +import csv +import hashlib +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import graph_atomic_survey_financial as financial +from . import puf55_survey_ss_measurement as ss +from . import puf_full_source as puf_source +from . import support_provenance as provenance + +source, full, support = ss.source, ss.full, ss.full.support +PROTOCOL = "microcosm.us.puf55-survey-recipients.v1" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) +MONEY_PREDICTORS = full.PUF59.predictors[2:] +MONEY_COLUMNS = ( + ("employment_income_before_lsr",), + ("self_employment_income_before_lsr",), + ("taxable_interest_income",), + ("non_qualified_dividend_income", "qualified_dividend_income"), + ("short_term_capital_gains",), + ("long_term_capital_gains_before_response",), +) +_STATUS = { + "SINGLE": 1, + "JOINT": 2, + "SEPARATE": 3, + "HEAD_OF_HOUSEHOLD": 4, + "SURVIVING_SPOUSE": 5, +} +MAX_RECEIPT_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_RECIPIENTS_" + reason) + + +def _table_digest(table): + """Exact axes, storage domains, values and unknowns for our detached tables.""" + _require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE") + digest = hashlib.sha256() + digest.update( + source._encode( + [ + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + [ + [ + str(d), + getattr(d, "storage", None), + str(getattr(d, "na_value", "")), + ] + for d in table.dtypes + ], + ] + ) + ) + for row in table.itertuples(index=True, name=None): + for value in row: + payload = source._frame_cell_encode(value) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + for series in (pd.Series(table.index), *(table[c] for c in table)): + if series.dtype == np.dtype("object"): + # These projections use object storage only for non-null strings. + # Their bytes were encoded above; ndarray.tobytes() here would hash + # process-local Python pointers and make equivalent replay unstable. + _require( + series.map(lambda value: type(value) is str).all(), + "OBJECT_STRING_STORAGE", + ) + continue + for part in financial.reconstruction._storage_parts( + series, np.ones(len(series), dtype=np.bool_) + ): + digest.update(len(part).to_bytes(8, "big")) + digest.update(part) + return digest.hexdigest() + + +def _literal_snapshot(state): + """Capture only the selected literal rows against the actual closed pin. + + Immutable strings come from the independently hashed original CSV stream, + not from a detached qualifier result or carried reason/allocation columns. + The existing coverage capture owns file handling; this adds no source issuer. + """ + owner = ss.reports.coverage + native = state.native[1] + entry = source.asec_native._ISSUED.get(id(native)) + _require( + entry is not None and entry[0]() is native and entry[1] == native.payload, + "LITERAL_NATIVE_OWNER", + ) + pins = tuple(pin for pin in owner._MEMBER_PINS if pin[0] == 2024) + _require(len(pins) == 1, "LITERAL_PIN") + year, member, archive, digest, rows, size = pins[0] + retained = [ + row + for row in entry[2].coverage.receipt["sources"] + if row["source_year"] == year + ] + _require( + len(retained) == 1 + and all( + retained[0][key] == value + for key, value in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "LITERAL_NATIVE_PIN", + ) + people = state.frame.person + keys = set( + people.loc[ + people[provenance.support_channel_column("person")].eq("asec"), "PERIDNUM" + ] + ) + reader = ss.reports.source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "LITERAL_CSV_READER") + captured_rows, count, total = {}, 0, 0 + hashed = hashlib.sha256() + with tempfile.TemporaryDirectory(prefix="microcosm-puf55-ss-literals-") as tmp: + path = Path(tmp) / member + owner._capture( + state.root / "asec" / member, + path, + size=size, + digest=digest, + budget=[owner._BODY_MAX], + ) + with path.open("rb") as stream: + + def lines(): + nonlocal total + while chunk := stream.readline(owner._ROW_MAX + 1): + _require( + len(chunk) <= owner._ROW_MAX and total + len(chunk) <= size, + "LITERAL_BYTES", + ) + first = total == 0 + total += len(chunk) + hashed.update(chunk) + yield chunk.decode("utf-8-sig" if first else "utf-8") + + records = reader(lines(), strict=True) + header = next(records, []) + _require( + len(header) == len(set(header)) + and set(ss.reports.READ_COLUMNS) <= set(header), + "LITERAL_HEADER", + ) + positions = [header.index(c) for c in ss.reports.READ_COLUMNS] + for record in records: + count += 1 + _require(count <= rows and len(record) == len(header), "LITERAL_ROWS") + selected = tuple(record[i] for i in positions) + if selected[0] in keys: + _require(selected[0] not in captured_rows, "LITERAL_DUPLICATE") + captured_rows[selected[0]] = selected + _require( + count == rows + and total == size + and hashed.hexdigest() == digest + and set(captured_rows) == keys, + "LITERAL_SOURCE_BYTES", + ) + return pins[0], tuple(captured_rows[key] for key in sorted(captured_rows)) + + +def _source_report(state, snapshot): + """Pure full projection from pinned literals and retained native source cells.""" + pin, records = snapshot + _require( + tuple(p for p in ss.reports.coverage._MEMBER_PINS if p[0] == 2024) == (pin,), + "LITERAL_PIN_CHANGED", + ) + people = state.frame.person + channel = people[provenance.support_channel_column("person")] + result = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + result["native_person_id"] = people[ + provenance.spine_source_id_column("person") + ].to_numpy() + result["source"] = channel.to_numpy() + result["social_security_source_total"] = np.nan + result["source_reporting_universe"] = False + result["source_reporting_unit"] = "person_report_record" + components = ss.reports.basis_owner.COMPONENTS + for column in components: + result[column], result["allowed_" + column] = np.nan, True + result["basis_origin"] = "unresolved" + result["allocation_origin"] = "unresolved_allocation_provenance" + asec = people.loc[channel.eq("asec")] + literals = ( + pd.DataFrame(records, columns=ss.reports.READ_COLUMNS) + .set_index("PERIDNUM", drop=False) + .loc[asec.PERIDNUM] + ) + _require( + np.array_equal(literals.A_AGE.astype("int64"), asec.A_AGE) + and np.array_equal( + ss.reports._codes(literals.SS_VAL, range(100000)), + ss.reports._numeric(asec.SS_VAL), + ), + "LITERAL_SOURCE_IDENTITY", + ) + amount, basis, allowed, labels = ss.reports.basis_owner.asec_reporting_basis( + ss.reports._numeric(asec.SS_VAL), + ss.reports._numeric(asec.A_AGE), + ss.reports._codes(literals.SS_YN, {0, 1, 2}), + ss.reports._codes(literals.RESNSS1, range(9)), + ss.reports._codes(literals.RESNSS2, range(9)), + ) + ids = asec.person_id.to_numpy() + result.loc[ids, "social_security_source_total"] = amount + result.loc[ids, "source_reporting_universe"] = ss.reports._numeric(asec.A_AGE) >= 15 + result.loc[ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, column in enumerate(components): + result.loc[ids, column], result.loc[ids, "allowed_" + column] = ( + basis[:, j], + allowed[:, j], + ) + result.loc[ids, "basis_origin"] = labels + result.loc[ids, "allocation_origin"] = ss.reports._allocation_labels( + literals + ).allocation_origin.to_numpy() + acs = people.loc[channel.eq("acs")] + eligible = ss.reports._numeric(acs.AGEP) >= 15 + total = ss.reports._numeric(acs.acs_social_security_income) + result.loc[acs.person_id, "social_security_source_total"] = total + result.loc[acs.person_id, "source_reporting_universe"] = eligible + zero = acs.person_id.to_numpy()[eligible & (total == 0)] + result.loc[zero, list(components)] = 0.0 + result.loc[zero, ["allowed_" + c for c in components]] = False + result.loc[zero, "basis_origin"] = "known_total_zero" + result.loc[acs.person_id.to_numpy()[eligible & (total > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + result.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + return result + + +def _aligned(source_frame, frame, entity): + original, receiving = source_frame.table(entity), frame.table(entity) + identity = entity + "_id" + key, arm = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + for column in ( + original[identity], + receiving[identity], + receiving[key], + receiving[arm], + ): + ss._axis(column) + _require(original[identity].is_unique and receiving[identity].is_unique, "AXIS") + pairs = receiving[[key, arm]] + _require( + receiving[arm].isin((0, 1)).all() + and not pairs.duplicated().any() + and set(receiving[key]) == set(original[identity]) + and pairs.groupby(key, sort=False).size().eq(2).all(), + "CLONE_PAIRS", + ) + aligned = original.set_index(identity).loc[receiving[key]].copy(deep=True) + for column in ( + provenance.spine_source_id_column(entity), + provenance.support_channel_column(entity), + ): + _require( + np.array_equal(aligned[column].to_numpy(), receiving[column].to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + aligned.index = pd.Index(receiving[identity].to_numpy(copy=True), name=identity) + return aligned + + +def _current_money_surface(frame, recipient_mask): + """Aggregate already-qualified financial leaves; issue no source authority. + + The public caller rechecks the actual financial run before this arithmetic. + Its source owner already verifies ACS adjustment and age-universe zeros. + A raw stacked-spine universe gate would incorrectly require ACS columns on + an ASEC-only surface and is not the input boundary for these produced leaves. + """ + person, units = frame.person, frame.table("tax_unit") + plans = {} + for predictor, columns in zip(MONEY_PREDICTORS, MONEY_COLUMNS, strict=True): + plan = support._strict_predictor_source_plan( + predictor, tax_unit=units, person=person + ) + _require( + plan.entity == "person" and plan.columns == columns, + "CURRENT_MONEY_SOURCE_PLAN", + ) + plans[predictor] = plan + support._require_complete_recipient_predictor_sources( + frame, recipient_mask, MONEY_PREDICTORS, source_plans=plans + ) + features = support._tax_unit_feature_frame( + frame, MONEY_PREDICTORS, preserve_nulls=True, source_plans=plans + ) + support._require_complete_recipient_predictors( + features, recipient_mask, MONEY_PREDICTORS + ) + evidence = { + "input_boundary": "current_financial_person_leaves_after_upstream_qualification", + "aggregation": "sum_all_modeled_tax_unit_members", + "predictor_source_mapping": { + predictor: {"entity": plan.entity, "columns": list(plan.columns)} + for predictor, plan in plans.items() + }, + "recipient_tax_unit_rows": int(recipient_mask.sum()), + "recipient_feature_values_sha256": support._predictor_feature_values_sha256( + features.loc[recipient_mask, list(MONEY_PREDICTORS)], + units.loc[recipient_mask, "tax_unit_id"], + ), + "raw_acs_universe_requalified_here": False, + "source_admission_issued": False, + } + evidence["sha256"] = support._receipt_sha256(evidence) + return features, evidence + + +def _project(source_frame, frame, report, measured): + """Pure projection after live ownership checks; never an authority issuer.""" + people, units = frame.person, frame.table("tax_unit") + original_people = _aligned(source_frame, frame, "person") + original_units = _aligned(source_frame, frame, "tax_unit") + unit_source = units.set_index("tax_unit_id")[ + provenance.support_source_id_column("tax_unit") + ] + _require( + np.array_equal( + people.tax_unit_role_input.to_numpy(), + original_people.tax_unit_role_input.to_numpy(), + ) + and np.array_equal( + units.filing_status_input.to_numpy(), + original_units.filing_status_input.to_numpy(), + ) + and np.array_equal( + people.person_tax_unit_id.map(unit_source).to_numpy(), + original_people.person_tax_unit_id.to_numpy(), + ), + "MODELED_ROLE_INHERITANCE", + ) + person_ids = people[provenance.support_source_id_column("person")].to_numpy() + person_report = report.loc[person_ids].copy(deep=True) + person_report.index = pd.Index( + people.person_id.to_numpy(copy=True), name="person_id" + ) + # Recheck every receiving role structure, and independently reproduce the + # source measurement on both arms. Invalid roles never select route eight. + received_measurement, _ = ss._measure(people, units, person_report) + expected = measured.loc[ + units[provenance.support_source_id_column("tax_unit")] + ].copy(deep=True) + expected.index = received_measurement.index + _require( + ss._projection_digest(received_measurement) == ss._projection_digest(expected), + "CLONE_MEASUREMENT", + ) + dependent_count = ( + people.tax_unit_role_input.eq("DEPENDENT") + .groupby(people.person_tax_unit_id) + .sum() + ) + first = puf_source.puf_2015_receiver_size_measurement( + units.filing_status_input.map(_STATUS).to_numpy(dtype=np.int64), + dependent_count.reindex(units.tax_unit_id).to_numpy(dtype=np.int64), + ) + for name, values in first.items(): + received_measurement[name] = values + recipients = ( + units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + ) + features, money_evidence = _current_money_surface(frame, recipients) + for name in MONEY_PREDICTORS: + received_measurement[name] = features[name].to_numpy( + dtype=np.float64, copy=True + ) + matrices, selected = [], [] + for profile in PROFILES: + mask = recipients & received_measurement[ss.ROUTE].eq(profile.value).to_numpy() + ids = units.loc[mask, "tax_unit_id"].to_numpy(copy=True) + selected.extend(ids.tolist()) + if not len(ids): + continue + _require( + len(ids) * (1 + len(profile.predictors)) * 8 <= RAW_BYTES_MAX_BYTES, + "MATRIX_SIZE", + ) + table = received_measurement.loc[ids, list(profile.predictors)].astype( + "float64" + ) + payload = model_input.encode_recipient_matrix( + table, entity="tax_unit", entity_ids=ids + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "MATRIX_SIZE") + matrices.append((profile.value, payload)) + _require( + len(selected) == len(set(selected)) + and set(selected) == set(units.loc[recipients, "tax_unit_id"]), + "RECIPIENT_PARTITION", + ) + return person_report, received_measurement, tuple(matrices), money_evidence + + +@dataclass(frozen=True) +class Puf55SurveyRecipients: + """Detached outputs with their actual upstream run; public copies are values.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + person: pd.DataFrame + tax_unit: pd.DataFrame + matrices: tuple[tuple[str, bytes], ...] + receipt: bytes + + +def _result_stamp(result): + _require( + type(result) is Puf55SurveyRecipients + and type(result.receipt) is bytes + and type(result.matrices) is tuple + and all( + type(row) is tuple + and len(row) == 2 + and type(row[0]) is str + and type(row[1]) is bytes + for row in result.matrices + ), + "RESULT_TYPE", + ) + return ( + _table_digest(result.person), + _table_digest(result.tax_unit), + result.matrices, + result.receipt, + ) + + +def qualify_puf55_survey_recipients(financial_run): + """Require the actual financial run and requalify source observations. + + This admits numerical preparation under retained modeled roles only. It + cannot establish current-money tax-unit reconstruction or release validity. + Source receipt objects and caller-supplied role tables are not accepted. + """ + financial.check_atomic_survey_financial_run(financial_run) + entry = financial._run_entry(financial_run) + state = entry[2] + preparation = state.prefix.preparation + source_frame = state.preparation_entry[2].frame + frame = state.financial_population.frame + snapshot = _literal_snapshot(state.preparation_entry[2]) + report = ss.reports.qualify_current_social_security(preparation) + ss._source_identity(source_frame.person, report) + authoritative_report = _source_report(state.preparation_entry[2], snapshot) + _require( + _table_digest(report.person) == _table_digest(authoritative_report), + "COMPLETE_SOURCE_REPORT", + ) + expected_measurement, measurement_counts = ss._measure( + source_frame.person, source_frame.table("tax_unit"), authoritative_report + ) + person, units, matrices, money_evidence = _project( + source_frame, frame, authoritative_report, expected_measurement + ) + receipt = source._encode( + { + "protocol": PROTOCOL, + "financial_run_sha256": financial.codec.sha(entry[1]), + "preparation_sha256": financial.codec.sha(state.preparation_entry[1]), + "person_projection_sha256": _table_digest(person), + "tax_unit_projection_sha256": _table_digest(units), + "source_report_projection_sha256": _table_digest(authoritative_report), + "asec_current_literal_member_sha256": snapshot[0][3], + "asec_selected_literal_rows": len(snapshot[1]), + "asec_income_year": 2024, + "asec_interview_year": 2025, + "acs_income_window": "rolling prior12 months at2024 interview; ADJINC price basis", + "reporting_grain": "person report; family/child aggregation and repayment/Railroad alignment unresolved", + "ss_measurement": ss.MEASUREMENT, + "native_measurement_counts": measurement_counts, + "measurement_knownness": "all included HEAD/actual JOINT SPOUSE reports available; no dependent, component or partial-sum fallback", + "money_predictor_evidence": money_evidence, + "routes": [ + { + "profile": name, + "matrix_sha256": financial.codec.sha(payload), + "rows": len(model_input.decode_recipient_matrix(payload).features), + "predictors": list(full.PufOutputProfile(name).predictors), + "outputs": list(full.PufOutputProfile(name).targets), + } + for name, payload in matrices + ], + "recipient_tax_units": sum( + len(model_input.decode_recipient_matrix(p).features) + for _, p in matrices + ), + "receiving_tax_units": len(units), + "receiving_persons": len(person), + "partition": "all clone-one tax units exactly once; empty routes omitted in nine/eight order", + "role_boundary": "retained modeled HEAD/actual JOINT SPOUSE roles; not observed filer status", + "money_contract": "existing six features sum all modeled tax-unit members; current financial leaves only", + "current_money_tax_units_reconstructed_here": False, + "person_social_security_changed": False, + "population_changed": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = Puf55SurveyRecipients(financial_run, person, units, matrices, receipt) + stamp = _result_stamp(result) + financial.check_atomic_survey_financial_run(financial_run) + financial._pure_run(financial_run, entry) + fresh = _source_report(state.preparation_entry[2], snapshot) + fresh_measurement, _ = ss._measure( + source_frame.person, source_frame.table("tax_unit"), fresh + ) + final_person, final_units, final_matrices, final_money = _project( + source_frame, frame, fresh, fresh_measurement + ) + _require( + financial._run_entry(financial_run) is entry + and result.financial_run is financial_run + and _result_stamp(result) == stamp + and _table_digest(fresh) == _table_digest(authoritative_report) + and ( + _table_digest(final_person), + _table_digest(final_units), + final_matrices, + receipt, + ) + == stamp + and final_money == money_evidence, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py b/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py new file mode 100644 index 000000000..bd80778bb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py @@ -0,0 +1,258 @@ +"""Actual typed recipient preparation and replay over invented originals.""" + +import sys +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_graph_current_survey_geography import _detached_context +from test_us_puf55_survey_recipients import recipient_financial_run # noqa: F401 + +from microcosm.build.us_runtime import graph_puf55_survey_recipients as graph +from microcosm.graph import StructuralDelta, compile_graph, run_graph +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys + + +@pytest.fixture(scope="module") +def recipient_graph(recipient_financial_run): # noqa: F811 + case = recipient_financial_run + run = case.run + nodes = graph.puf55_survey_recipient_nodes(case.qualified) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *nodes)) + ) + kernels = ( + graph.Puf55SurveyRecipientProjectionKernel(run), + graph.Puf55SurveyRecipientMatrixKernel(run), + ) + for kernel in kernels: + run.kernels.register(kernel) + contexts = {} + + def trace(frame, event, arg): + if event == "call" and frame.f_code is graph._Kernel.run.__code__: + context = frame.f_locals["context"] + contexts[context.node.id] = context + + old = sys.getprofile() + sys.setprofile(trace) + try: + cold = run_graph( + compiled, sources=run.sources, store=run.store, kernels=run.kernels + ) + finally: + sys.setprofile(old) + assert set(contexts) == {graph.PROJECTION_NODE, graph.MATRIX_NODE} + warm = run_graph( + compiled, + sources=run.sources, + store=run.store, + kernels=run.kernels, + resume="require", + ) + yield SimpleNamespace( + case=case, + run=run, + nodes=nodes, + compiled=compiled, + kernels=kernels, + contexts=contexts, + cold=cold, + warm=warm, + ) + graph.financial.check_atomic_survey_financial_run(run) + + +def _verified_artifacts(case, manifest): + """Check actual producer keys and typed closure before reading artifacts.""" + run = case.run + _, sources = _source_paths_and_keys(case.compiled, run.sources, run.store) + keys, implementations = _all_node_keys(case.compiled, run.kernels, sources) + payloads = {} + for node in case.nodes: + record = manifest.node(node.id) + assert record.key == keys[node.id] + assert record.kernel_impl_hash == implementations[node.id] + assert record.typed_artifacts == typed_contracts( + case.compiled, node, keys, run.kernels + ) + assert set(record.opaque_artifacts) == {a.name for a in node.artifact_outputs} + for artifact in node.artifact_outputs: + key = record.opaque_artifacts[artifact.name] + assert key == graph.opaque_artifact_key(keys[node.id], artifact.name) + payloads[node.id, artifact.name] = run.store.load_bytes(key) + matrices = tuple( + (name, payloads[graph.MATRIX_NODE, graph._NAMES[name]]) + for name, _ in case.case.qualified.matrices + ) + return payloads[graph.PROJECTION_NODE, "projection"], matrices + + +def test_actual_twenty_two_node_cold_required_replay_keeps_complete_population( + recipient_graph, +): + case = recipient_graph + assert len(case.compiled.order) == 22 + assert set(case.compiled.order) == { + *case.run.compiled.order, + graph.PROJECTION_NODE, + graph.MATRIX_NODE, + } + order = case.compiled.order + assert ( + order.index(graph.financial.financial.ATTACH_NODE) + < order.index(graph.PROJECTION_NODE) + < order.index(graph.MATRIX_NODE) + ) + assert all( + node.structural is StructuralDelta.NONE + and not node.outputs + and node.weights is None + for node in case.nodes + ) + assert case.cold.key == case.warm.key + assert all(node.hit for node in case.warm.nodes.values()) + assert tuple(o.name for o in case.nodes[1].artifact_outputs) == ( + "matrix_nine", + "matrix_eight", + ) + for manifest in (case.cold, case.warm): + projection, matrices = _verified_artifacts(case, manifest) + got = graph.verify_materialized_puf55_survey_recipients( + case.run, projection=projection, matrices=matrices + ) + assert got.receipt == case.case.qualified.receipt + assert got.matrices == case.case.qualified.matrices + after = manifest.population(case.run.financial_population.version) + graph.financial.survey._same_frame(case.run.financial_population.frame, after) + for entity in after.entities: + pd.testing.assert_frame_equal( + after.table(entity), + case.run.financial_population.frame.table(entity), + check_exact=True, + ) + assert ( + manifest.mass_ledger(case.run.financial_population.version) + == case.run.financial_population.mass_ledger + ) + graph.financial._pure_run(case.run, graph.financial._run_entry(case.run)) + + +@pytest.mark.parametrize( + "defect", + ( + "parameter", + "source_roster", + "artifact_payload", + "artifact_key", + "context_money", + "route_roster", + ), +) +def test_kernel_refuses_unbound_declaration_artifact_or_receiving_values( + recipient_graph, defect +): + case = recipient_graph + context = _detached_context(case.contexts[graph.MATRIX_NODE]) + if defect == "parameter": + context = replace( + context, params={**context.params, "financial_run_sha256": "0" * 64} + ) + elif defect == "source_roster": + context = replace(context, sources={"invented": "/not-read"}) + elif defect == "artifact_payload": + artifacts = dict(context.artifacts) + artifacts["financial_matrix"] = replace( + artifacts["financial_matrix"], payload=b"wrong" + ) + context = replace(context, artifacts=artifacts) + elif defect == "artifact_key": + artifacts = dict(context.artifacts) + artifacts["financial_projection"] = replace( + artifacts["financial_projection"], producer_key="0" * 64 + ) + context = replace(context, artifacts=artifacts) + elif defect == "context_money": + table = context.tables["person"] + table.loc[table.index[0], "employment_income_before_lsr"] += 1 + else: + node = replace(context.node, artifact_outputs=()) + context = replace(context, node=node) + # A real previously qualified context/value pair isolates contract negatives + # from expensive source rebuilds. Actual cold/replay above executes kernels. + with pytest.raises((ValueError, AssertionError)): + case.kernels[1]._context(context, case.case.qualified) + + +@pytest.mark.parametrize( + "target", + ("artifact", "receipt", "source_owner", "baseline_and_result", "mutable_artifact"), +) +def test_last_owner_check_seals_graph_outputs_and_population(recipient_graph, target): + case, fired = recipient_graph, [] + context = _detached_context(case.contexts[graph.MATRIX_NODE]) + original = None + table = case.run.financial_population.frame.person + index = table.index[0] + if target == "source_owner": + original = table.loc[index, "employment_income_before_lsr"] + + def trace(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is graph.financial.check_atomic_survey_financial_run.__code__ + and caller is not None + and caller.f_code is graph._Kernel.run.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + if target == "artifact": + result.artifacts[next(iter(result.artifacts))] = b"wrong" + elif target == "receipt": + result.receipt["release_eligible"] = True + elif target == "baseline_and_result": + key = next(iter(result.artifacts)) + result.artifacts[key] = b"wrong" + caller.f_locals["outputs"][key] = b"wrong" + elif target == "mutable_artifact": + key = next(iter(result.artifacts)) + result.artifacts[key] = bytearray(result.artifacts[key]) + else: + table.loc[index, "employment_income_before_lsr"] += 1 + + old = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + ValueError, match="GRAPH_FINAL_RESULT|FINANCIAL_RUN_.*CHANGED" + ): + case.kernels[1].run(context) + finally: + sys.setprofile(old) + if original is not None: + table.loc[index, "employment_income_before_lsr"] = original + assert fired == [True] + + +@pytest.mark.parametrize("defect", ("projection", "missing_route", "matrix_bytes")) +def test_materialized_replay_refuses_projection_or_route_matrix_substitution( + recipient_graph, defect +): + case = recipient_graph + projection, matrices = _verified_artifacts(case, case.warm) + if defect == "projection": + projection = b"{}" + elif defect == "missing_route": + matrices = () + else: + matrices = ((matrices[0][0], b"wrong"), *matrices[1:]) + with pytest.raises(ValueError, match="MATERIALIZED_ARTIFACTS"): + graph.verify_materialized_puf55_survey_recipients( + case.run, projection=projection, matrices=matrices + ) diff --git a/packages/microcosm-build/tests/test_us_puf55_survey_recipients.py b/packages/microcosm-build/tests/test_us_puf55_survey_recipients.py new file mode 100644 index 000000000..56b44dbb6 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf55_survey_recipients.py @@ -0,0 +1,500 @@ +"""Invented numerical partitions and real original-source financial ownership.""" + +import hashlib +import shutil +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_acs_person_coverage_authentication import ( + Member, + _csv, + _household, + _person, + build_fixture, +) +from test_us_graph_atomic_survey_population import _support_payload +from test_us_puf55_survey_ss_measurement import _values +from test_us_survey_social_security import _social_security_arguments + +from microcosm.build.us_runtime import current_asec_demographics as demographics +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.build.us_runtime import puf55_survey_recipients as values +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _frames(): + """Plain invented Frames for private arithmetic, never source/run issuers.""" + person, units, report = _values() + person["age"] = 55 + person["prior_wages_not_a_predictor"] = 99_999_999.0 + person["unrelated_social_security_cell"] = np.resize([-0.0, np.nan], len(person)) + for i, name in enumerate(values.financial.values.OUTPUTS): + person[name] = (person.person_id + i).to_numpy(dtype=np.float64) + for group in US_SCHEMA.group_entities: + person["person_" + group + "_id"] = person.person_tax_unit_id + tables = { + "person": person, + "tax_unit": units, + **{ + g: pd.DataFrame({g + "_id": units.tax_unit_id.to_numpy()}) + for g in US_SCHEMA.group_entities + if g != "tax_unit" + }, + } + for entity, table in tables.items(): + table[values.provenance.spine_source_id_column(entity)] = table[entity + "_id"] + table[values.provenance.support_channel_column(entity)] = pd.array( + ["asec"] * len(table), dtype="string" + ) + cloned = {} + for entity, table in tables.items(): + arms = [] + for arm in (0, 1): + copy = table.copy(deep=True) + copy[values.provenance.support_source_id_column(entity)] = copy[ + entity + "_id" + ] + copy[values.provenance.support_clone_index_column(entity)] = np.full( + len(copy), arm, dtype=np.int64 + ) + copy[entity + "_id"] += 1000 * arm + if entity == "person": + for group in US_SCHEMA.group_entities: + copy["person_" + group + "_id"] += 1000 * arm + arms.append(copy) + cloned[entity] = pd.concat(arms, ignore_index=True) + + def frame(tables): + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.ones(len(tables["household"])), WeightKind.IMPORTANCE + ) + }, + pd.Series(["invented"] * len(tables["person"]), dtype="string"), + ) + + return frame(tables), frame(cloned), report + + +def _project(native, receiving, report): + measured, _ = values.ss._measure(native.person, native.table("tax_unit"), report) + return values._project(native, receiving, report, measured) + + +def test_disjoint_routes_cover_all_clone_one_and_preserve_both_arms(): + native, receiving, report = _frames() + before = {e: receiving.table(e).copy(deep=True) for e in receiving.entities} + source_report = report.copy(deep=True) + # These are complete current financial leaves on an ASEC-only invented + # surface. Raw ACS WAGP/SEMP columns are neither needed nor manufactured. + assert not {"WAGP", "SEMP"} & set(receiving.person) + person, units, matrices, money_evidence = _project(native, receiving, report) + assert money_evidence["aggregation"] == "sum_all_modeled_tax_unit_members" + assert money_evidence["raw_acs_universe_requalified_here"] is False + assert money_evidence["source_admission_issued"] is False + assert tuple(name for name, _ in matrices) == tuple( + p.value for p in values.PROFILES + ) + nine, eight = [ + values.model_input.decode_recipient_matrix(p).features for _, p in matrices + ] + assert nine.index.tolist() == [1010, 1020, 1030, 1040] + assert eight.index.tolist() == [1050] + assert set(nine.index).isdisjoint(eight.index) + assert set(nine.index) | set(eight.index) == {1010, 1020, 1030, 1040, 1050} + assert tuple(nine) == values.PROFILES[0].predictors + assert tuple(eight) == values.PROFILES[1].predictors + assert [len(p.targets) for p in values.PROFILES] == [55, 55] + np.testing.assert_array_equal(nine[values.ss.TOTAL], [100, 500, 400, 0]) + assert np.isnan(units.loc[1050, values.ss.TOTAL]) + # Money deliberately includes dependent earnings; SS deliberately does not. + assert nine.loc[1010, "puf_predictor_employment_income"] == 1 + 2 + assert nine.loc[1020, "puf_predictor_employment_income"] == 3 + 4 + 5 + assert nine.loc[1030, "puf_2015_filing_status_code"] == 2 + assert nine.loc[1030, "puf_2015_capped_return_size"] == 3 + for arm in (0, 1): + observed = person.loc[report.index + 1000 * arm].copy() + observed.index = report.index + pd.testing.assert_frame_equal(observed, report, check_exact=True) + for entity, table in before.items(): + pd.testing.assert_frame_equal(receiving.table(entity), table, check_exact=True) + pd.testing.assert_frame_equal(report, source_report, check_exact=True) + + +@pytest.mark.parametrize("known", (False, True)) +def test_empty_route_omitted_without_empty_matrix_or_unknown_coercion(known): + native, receiving, report = _frames() + included = native.person.tax_unit_role_input.isin(("HEAD", "SPOUSE")).to_numpy() + report.loc[included, "social_security_source_total"] = 0.0 if known else np.nan + report.loc[included, "source_reporting_universe"] = known + _, units, matrices, _ = _project(native, receiving, report) + assert tuple(name for name, _ in matrices) == ( + values.PROFILES[0 if known else 1].value, + ) + matrix = values.model_input.decode_recipient_matrix(matrices[0][1]).features + assert len(matrix) == 5 and len(matrix.columns) == (9 if known else 8) + assert units[values.ss.KNOWN].eq(known).all() + assert ( + units[values.ss.TOTAL].eq(0).all() + if known + else units[values.ss.TOTAL].isna().all() + ) + + +def test_source_and_receiving_order_are_joined_by_identity(): + native, receiving, report = _frames() + expected = _project(native, receiving, report) + for frame in (native, receiving): + for entity in frame.entities: + table = frame.table(entity) + table.iloc[:] = table.iloc[::-1].to_numpy() + observed = _project(native, receiving, report.iloc[::-1]) + for got, want in zip(observed[:2], expected[:2], strict=True): + pd.testing.assert_frame_equal( + got.sort_index(), want.sort_index(), check_exact=True + ) + for (_, got), (_, want) in zip(observed[2], expected[2], strict=True): + pd.testing.assert_frame_equal( + values.model_input.decode_recipient_matrix(got).features.sort_index(), + values.model_input.decode_recipient_matrix(want).features.sort_index(), + check_exact=True, + ) + + +@pytest.mark.parametrize( + "defect", + ("role", "membership", "clone", "source", "money_missing", "carried_alias"), +) +def test_projection_refuses_invalid_roles_origin_or_money_before_routing(defect): + native, receiving, report = _frames() + person = receiving.person + if defect == "role": + person.loc[person.person_id.eq(1001), "tax_unit_role_input"] = "DEPENDENT" + elif defect == "membership": + person.loc[person.person_id.eq(1001), "person_tax_unit_id"] = 1020 + elif defect == "clone": + person.loc[ + person.person_id.eq(1001), + values.provenance.support_clone_index_column("person"), + ] = 0 + elif defect == "source": + person.loc[ + person.person_id.eq(1001), + values.provenance.spine_source_id_column("person"), + ] = 999 + elif defect == "money_missing": + person.loc[person.person_id.eq(1002), "employment_income_before_lsr"] = np.nan + else: + person["employment_income"] = 999.0 + with pytest.raises( + ValueError, match="PUF55_SURVEY_RECIPIENTS_|missing values before coercion" + ): + _project(native, receiving, report) + + +@pytest.mark.parametrize("defect", ("missing_dividend_component", "nonfinite_income")) +def test_current_financial_money_refuses_missing_components_and_nonfinite_values( + defect, +): + native, receiving, report = _frames() + column, value = ( + ("qualified_dividend_income", np.nan) + if defect == "missing_dividend_component" + else ("self_employment_income_before_lsr", np.inf) + ) + receiving.person.loc[receiving.person.person_id.eq(1002), column] = value + with pytest.raises(ValueError, match="missing values before coercion|nonfinite"): + _project(native, receiving, report) + + +def _recipient_source_arguments(root, patch): + """Extend original SS fixture members before any preparation is issued.""" + args = _social_security_arguments(root, patch, ambiguous=True) + # One genuine source record is outside the age15 question universe. The + # real native unit constructor, not this test, assigns its modeled role. + # Its sole-person GQ tax unit exercises the actual eight-feature route. + households = [ + _household("2024HU0000001", NP=2), + _household("2024HU0000002", NP=1), + _household("2024GQ0000001", NP=1, TYPEHUGQ=2, WGTP=0, TEN=""), + _household("2024GQ0000002", NP=1, TYPEHUGQ=3, WGTP=0, TEN=""), + _household("2024HU0000003", NP=0, TEN=""), + ] + people = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4", SSP=1200), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + _person("2024GQ0000001", 1, 37, AGEP=40, MIL="4", ESR="6", PWGTP=77), + _person( + "2024GQ0000002", + 1, + 38, + AGEP=14, + MAR=5, + MIL="", + ESR="", + PWGTP=78, + WAGP=None, + SEMP=None, + SSP=None, + SSIP=None, + RETP=None, + INTP=None, + ), + ] + acs = build_fixture( + root / "recipient-acs-original", + patch, + household_members=(Member("psam_husa.csv", _csv(households)),), + person_members=(Member("psam_pusa.csv", _csv(people)),), + ) + for name in ("csv_hus.zip", "csv_pus.zip"): + shutil.copyfile(acs.source_dir / name, args["source_dir"] / "acs" / name) + asec = args["source_dir"] / "asec" + # Use the maintained literal-source owner directly, not a ready() substitute. + from microcosm.build.us_runtime import asec_person_income_source as restoration + + members, pins = {}, [] + for year, member, archive, *_ in values.ss.reports.coverage._MEMBER_PINS: + path = asec / f"pppub{year - 1999}.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + lines = table.A_LINENO.map(int) + table["A_SEX"], table["AXSEX"] = ( + lines.map({1: "1", 2: "2"}), + lines.map({1: "0", 2: "4"}), + ) + table["A_EXPRRP"], table["P_SEQ"] = lines.map({1: "1", 2: "5"}), table.A_LINENO + table.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(table), + len(payload), + ) + ) + members[year] = path + for module in (values.ss.reports.coverage, restoration, demographics.demographic): + patch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = root / "recipient-restored-money" + restoration.restore_asec_person_income_source( + asec / "parent.h5", + asec / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, asec / "person-income-attachment.h5" + ) + path = asec / "hhpub25.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + table["GESTFIPS"] = table.H_SEQ.map({"00007": "06", "00008": "36"}) + table.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + patch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(payload).hexdigest(), + size_bytes=len(payload), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return args + + +@pytest.fixture(scope="module") +def recipient_financial_run(tmp_path_factory): + """One real financial execution shared across bounded source/graph checks.""" + root = tmp_path_factory.mktemp("puf55-recipient-financial") + with pytest.MonkeyPatch.context() as patch: + args = _recipient_source_arguments(root, patch) + payload, source_ids = _support_payload() + path = root / "invented-support.npz" + path.write_bytes(payload) + config = values.financial.reconstruction.AtomicSurveyReconstruction( + support_path=str(path), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + run = values.financial.run_atomic_survey_financial( + **args, + geography_config=config, + store_root=root / "store", + demographic_conditioning=True, + n_estimators=2, + return_values=True, + ) + qualified = values.qualify_puf55_survey_recipients(run) + yield SimpleNamespace(run=run, qualified=qualified) + values.financial.check_atomic_survey_financial_run(run) + + +def test_actual_issued_financial_run_retains_source_reports_and_current_money( + recipient_financial_run, +): + case = recipient_financial_run + run, got = case.run, case.qualified + assert got.financial_run is run + evidence = values.financial.codec.decode_json(got.receipt) + assert evidence["financial_run_sha256"] == values.financial.codec.sha( + values.financial._run_entry(run)[1] + ) + assert evidence["current_money_tax_units_reconstructed_here"] is False + assert evidence["population_changed"] is False + assert evidence["release_eligible"] is False + assert tuple(name for name, _ in got.matrices) == tuple( + p.value for p in values.PROFILES + ) + fallback = values.model_input.decode_recipient_matrix(got.matrices[1][1]).features + assert len(fallback) == 1 and len(fallback.columns) == 8 + assert got.tax_unit.loc[fallback.index, values.ss.TOTAL].isna().all() + assert not got.tax_unit.loc[fallback.index, values.ss.KNOWN].any() + person = run.financial_population.frame.person + for _, rows in person.groupby(values.provenance.support_source_id_column("person")): + assert len(rows) == 2 + outputs = got.person.loc[rows.person_id] + left, right = outputs.iloc[0].copy(), outputs.iloc[1].copy() + left.name = right.name + pd.testing.assert_series_equal(left, right, check_exact=True) + assert any(got.person[c].isna().any() for c in values.full.SURVEY_SS_COMPONENTS) + assert not any( + "prior" in p for profile in values.PROFILES for p in profile.predictors + ) + ids = { + int(i) + for _, payload in got.matrices + for i in values.model_input.decode_recipient_matrix(payload).features.index + } + units = run.financial_population.frame.table("tax_unit") + assert ids == set( + units.loc[ + units[values.provenance.support_clone_index_column("tax_unit")].eq(1), + "tax_unit_id", + ] + ) + values.financial._pure_run(run, values.financial._run_entry(run)) + + +@pytest.mark.parametrize("kind", ("receipt", "copied_run")) +def test_descriptive_values_cannot_issue_financial_authority( + recipient_financial_run, kind +): + case = recipient_financial_run + fake = ( + values.financial.codec.decode_json(case.qualified.receipt) + if kind == "receipt" + else replace(case.run) + ) + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + values.qualify_puf55_survey_recipients(fake) + + +@pytest.mark.parametrize("field", ("allowed", "component", "basis")) +def test_every_source_projection_return_is_checked_against_original_literals( + recipient_financial_run, + field, +): + fired = [] + + def profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is values.ss.reports.qualify_current_social_security.__code__ + and caller is not None + and caller.f_code is values.qualify_puf55_survey_recipients.__code__ + ): + fired.append(True) + if field == "allowed": + column = "allowed_" + values.full.SURVEY_SS_COMPONENTS[0] + arg.person.loc[arg.person.index[0], column] = not bool( + arg.person.iloc[0][column] + ) + elif field == "component": + arg.person.loc[ + arg.person.index[0], values.full.SURVEY_SS_COMPONENTS[0] + ] = 999.0 + else: + arg.person.loc[arg.person.index[0], "basis_origin"] = ( + "invented_changed_basis" + ) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises(ValueError, match="COMPLETE_SOURCE_REPORT"): + values.qualify_puf55_survey_recipients(recipient_financial_run.run) + finally: + sys.setprofile(previous) + assert fired == [True] + + +@pytest.mark.parametrize( + "target", ("person", "matrix", "columns_name", "nan_bits", "mutable_matrix") +) +def test_final_run_check_cannot_mutate_detached_recipient_outputs( + recipient_financial_run, target +): + fired = [] + + def profile(frame, event, arg): + caller = frame.f_back + if ( + event == "return" + and frame.f_code + is values.financial.check_atomic_survey_financial_run.__code__ + and caller is not None + and caller.f_code is values.qualify_puf55_survey_recipients.__code__ + and "result" in caller.f_locals + and not fired + ): + fired.append(True) + result = caller.f_locals["result"] + if target == "person": + result.person.iloc[ + 0, result.person.columns.get_loc("social_security_source_total") + ] += 1 + elif target == "matrix": + object.__setattr__(result, "matrices", ()) + elif target == "columns_name": + result.person.columns.name = "changed" + elif target == "nan_bits": + column = values.full.SURVEY_SS_COMPONENTS[0] + position = np.flatnonzero(result.person[column].isna().to_numpy())[0] + result.person.loc[result.person.index[position], column] = np.array( + [0x7FF8000000000042], dtype=np.uint64 + ).view(np.float64)[0] + else: + name, payload = result.matrices[0] + object.__setattr__( + result, + "matrices", + ((name, bytearray(payload)), *result.matrices[1:]), + ) + + previous = sys.getprofile() + sys.setprofile(profile) + try: + with pytest.raises( + ValueError, match="PUF55_SURVEY_RECIPIENTS_(FINAL_PROJECTION|RESULT_TYPE)" + ): + values.qualify_puf55_survey_recipients(recipient_financial_run.run) + finally: + sys.setprofile(previous) + assert fired == [True] From 2ce0a3a99b087d0fc12b3d6347f2d8cd750d6f06 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 04:33:33 -0400 Subject: [PATCH 058/255] Declare bounded system metadata for isolated CI diagnostics --- .github/workflows/test.yml | 1 + ...test_spec_seed_identity_system_metadata.py | 159 ++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 78 +++++++++ 3 files changed, 238 insertions(+) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d0279ae7..c4430bb1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -236,6 +236,7 @@ jobs: packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics timeout-minutes: 15 diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py new file mode 100644 index 000000000..2ea24cad3 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -0,0 +1,159 @@ +"""Invented metadata controls; no main, engine, data, child or real audit hook.""" + +import importlib.util +import os +import platform +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("system_metadata_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def invented_uname(diagnostic, monkeypatch): + observed = SimpleNamespace( + sysname="Linux", + nodename="invented-host", + release="invented-release", + version="invented-version", + machine="invented_arch", + ) + info = platform.uname_result( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + monkeypatch.setattr(diagnostic.sys, "platform", "linux") + monkeypatch.setattr(diagnostic.os, "uname", lambda: observed) + monkeypatch.setattr(diagnostic.platform, "uname", lambda: info) + + def forbidden(*args, **kwargs): + pytest.fail("processor metadata attempted a subprocess") + + monkeypatch.setattr(subprocess, "check_output", forbidden) + return info, observed + + +def test_real_cached_property_uses_machine_without_replacing_stdlib( + diagnostic, invented_uname +): + info, observed = invented_uname + before = ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + assert platform.processor() == observed.machine + assert tuple(info) == ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + observed.machine, + ) + assert bootstrap["system_metadata"]["uname_p_parity_claimed"] is False + assert bootstrap["system_metadata"]["source"] == "os.uname.machine" + assert before == ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + diagnostic.verify_processor_metadata(state, bootstrap) + # An already-cached identical machine value is not silently altered. + again = diagnostic.prime_processor_metadata(bootstrap) + assert again == state + + +def test_conflicting_observed_processor_cache_is_not_overwritten( + diagnostic, invented_uname +): + info, _ = invented_uname + info.processor = "different_observed_processor" + bootstrap = {} + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_CACHE_CONFLICT$"): + diagnostic.prime_processor_metadata(bootstrap) + assert info.processor == "different_observed_processor" + assert bootstrap == {} + + +@pytest.mark.parametrize("change", ("cached_value", "os_machine", "evidence")) +def test_metadata_change_refuses(diagnostic, invented_uname, change): + info, observed = invented_uname + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + if change == "cached_value": + info.processor = "changed" + elif change == "os_machine": + observed.machine = "changed" + else: + bootstrap["system_metadata"]["uname_p_parity_claimed"] = True + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_METADATA_CHANGED$"): + diagnostic.verify_processor_metadata(state, bootstrap) + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks = [] + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(diagnostic.os, "getpid", lambda: 321) + monkeypatch.setattr(diagnostic.os, "readlink", lambda _: "pipe:[invented]") + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + + def resolve(path): + return Path("/proc/321/maps") if path == Path("/proc/self/maps") else path + + monkeypatch.setattr(Path, "resolve", resolve) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), Path("/invented/owned"), Path("/invented/output") + ) + assert len(hooks) == 1 + return hooks[0], refusals + + +@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps")) +def test_only_logical_and_resolved_own_maps_are_admitted(audited, path): + hook, refusals = audited + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [] + + +@pytest.mark.parametrize( + "kind", ("other_process", "other_file", "relative", "descriptor", "child") +) +def test_maps_alias_does_not_admit_other_paths_descriptors_or_children( + diagnostic, audited, kind +): + hook, refusals = audited + path = { + "other_process": "/proc/322/maps", + "other_file": "/proc/321/mem", + "relative": "maps", + "descriptor": 55, + "child": None, + }[kind] + code = "NETWORK_OR_CHILD" if kind == "child" else "READ_SCOPE" + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + if kind == "child": + hook("subprocess.Popen", ("uname", ["uname", "-p"], None, None)) + else: + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [code] diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 9cfb2295f..03c78cc5b 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -9,6 +9,7 @@ import importlib.metadata import json import os +import platform import re import resource import signal @@ -16,6 +17,7 @@ import sys import tempfile import tomllib +from functools import cached_property from pathlib import Path LOCK_SHA256 = "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b" @@ -359,6 +361,8 @@ def install_boundary( Path("/proc/cpuinfo"), Path("/proc/meminfo"), Path("/proc/self/maps"), + # operation_path resolves /proc/self to this process's numeric PID. + Path(f"/proc/{os.getpid()}/maps"), } context_roots = ( @@ -599,6 +603,77 @@ def verify_cpu_only_torch_import(bootstrap: dict) -> None: bootstrap["torch_optional_bindings_fallback_verified"] = True +def prime_processor_metadata(bootstrap: dict) -> tuple: + """Use actual machine architecture for this process's processor metadata. + + This initializes only the documented writable cached_property on the real + uname result. It replaces no platform function, descriptor, class or module. + No equivalence to the external uname -p probe is asserted. + """ + require(sys.platform == "linux", "PROCESSOR_PLATFORM") + observed = os.uname() + info = platform.uname() + fields = ("system", "node", "release", "version", "machine") + actual = ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + require( + type(info) is platform.uname_result + and type(vars(type(info)).get("processor")) is cached_property + and tuple(getattr(info, name) for name in fields) == actual + and type(observed.machine) is str + and bool(re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", observed.machine)), + "PROCESSOR_UNAME_CONTRACT", + ) + prior = vars(info).get("processor") + require( + "processor" not in vars(info) + or (type(prior) is str and prior == observed.machine), + "PROCESSOR_CACHE_CONFLICT", + ) + evidence = { + "policy": "os_uname_machine_as_processor_metadata_v1", + "value": observed.machine, + "source": "os.uname.machine", + "uname_p_parity_claimed": False, + "stdlib_callables_replaced": False, + "lifetime": "diagnostic_process", + "process_maps_permission": "exact_resolved_own_pid_maps_alias", + } + info.processor = observed.machine + bootstrap["system_metadata"] = evidence + return info, actual, encoded(evidence) + + +def verify_processor_metadata(state: tuple, bootstrap: dict) -> None: + info, actual, evidence = state + observed = os.uname() + require( + platform.uname() is info + and tuple( + getattr(info, name) + for name in ("system", "node", "release", "version", "machine") + ) + == actual + and ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + == actual + and type(vars(info).get("processor")) is str + and vars(info)["processor"] == actual[-1] + and encoded(bootstrap.get("system_metadata")) == evidence, + "PROCESSOR_METADATA_CHANGED", + ) + + def thread_controls(torch) -> dict[str, object]: controls = { "environment": {name: os.environ.get(name) for name in THREAD_ENV}, @@ -896,6 +971,7 @@ def main() -> int: lock = tomllib.loads((root / "uv.lock").read_text()) installed = versions(lock) phase = "derive" + processor_state = prime_processor_metadata(bootstrap) with ( open(os.devnull, "w") as quiet, contextlib.redirect_stdout(quiet), @@ -913,6 +989,7 @@ def main() -> int: payloads = derive( root, owned, run, before, installed, torch, controls, bootstrap ) + verify_processor_metadata(processor_state, bootstrap) require(not refusals, "BOUNDARY_REFUSAL") exit_code = 0 except Exception as error: @@ -948,6 +1025,7 @@ def main() -> int: ) try: if exit_code == 0: + verify_processor_metadata(processor_state, bootstrap) require(thread_controls(torch) == controls, "THREADS_CHANGED") require(not refusals, "BOUNDARY_REFUSAL") publish( From 2904f158126702127520683ea48ec3fdded69bed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 04:49:00 -0400 Subject: [PATCH 059/255] Record the verified native survey geography and financial pilot --- docs/us-launch-review.md | 23 +++- ...native-atomic-financial-cold-20260910.json | 104 ++++++++++++++++++ 2 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 experiments/us-native-atomic-financial-cold-20260910.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index fcf199077..aa240992a 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -45,6 +45,17 @@ need reconciliation; this draft does not supersede either review. ## Verification and review scope +The first native twenty-node survey, block-geography and financial cold pilot +now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 +household records and 6,928 person records after cloning. The run assigns one +block before cloning, retains location inheritance and attaches seven financial +fields while preserving the other population fields. It took 110.12 minutes and +11.45 GB peak memory. The final receipt and independent source/resource +postcheck pass. This is the original frozen 491-source implementation; newer +cache and budget changes are separate revisions. Required native cached replay, +PUF augmentation, model quality and calibration remain pending. See the +[scoped native cold result](../experiments/us-native-atomic-financial-cold-20260910.json). + The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` passed 36 integration controls and 57 enrichment/replay controls on invented inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and @@ -146,8 +157,9 @@ from the benchmark's accepted baseline. The following work remains open: -1. Run the accepted twenty-node atomic and financial composition on the genuine - combined survey population. Its cold/required replay, complete retained +1. Complete required cached replay of the accepted native twenty-node atomic + and financial cold pilot, then validate the complete survey scale. Its + invented cold/required replay, complete retained fields and geography, default three-feature versus explicit five-feature conditioning, and both final mutation refusals pass on invented inputs. Two test-code mistakes in the initial four-case run were corrected and @@ -216,9 +228,10 @@ readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is `5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). Postchecks reauthenticated code and compared the 104 native pins from bounded -receipts without reopening source bodies. This clears the national support -normalization check; assigning households, evaluating geographic fit and -calibrating the enriched survey remain separate acceptance steps. +receipts without reopening source bodies. National support normalization and +household assignment in the small native financial pilot now pass. Evaluating +geographic fit and calibrating the enriched survey remain separate acceptance +steps. The financial successor's unchanged positive control now passes with both original fixture teardowns and final code/resource checks: 647.19 CPU seconds, diff --git a/experiments/us-native-atomic-financial-cold-20260910.json b/experiments/us-native-atomic-financial-cold-20260910.json new file mode 100644 index 000000000..6a6d1d586 --- /dev/null +++ b/experiments/us-native-atomic-financial-cold-20260910.json @@ -0,0 +1,104 @@ +{ + "age_calibration_acceptance": false, + "approval_sha256": "837bf6c06fbe186f615080cd844ce0e92eecbad387e800f6d7cc4bfa02b673ae", + "artifact_identity_verification": "pinned manifest metadata and internal producer/key/size relations only; no graph store, export or snapshot reopen", + "collected_tests": 1, + "compiled_nodes": 20, + "composed_prefix_store_hits": 10, + "cpu_seconds": 6591.938158, + "cross_process_source_authority_granted": false, + "current_checkout_certified": false, + "date": "2026-09-10", + "entity_rows": { + "clone": { + "household": 3168, + "person": 6928 + }, + "create": { + "household": 1584, + "person": 3464 + }, + "financial": { + "household": 3168, + "person": 6928 + } + }, + "features": [ + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", + "survey_predictor_is_female", + "survey_predictor_state_fips" + ], + "financial_cold_nodes": 10, + "financial_successor_acceptance": false, + "guard_before_after_identity_count": 509, + "guard_sha256": "5bffe33e4c60055070bb9ab9a51ad2e87bfbf86fcd843e1da0d6407d53debda1", + "head_coordinate": "e0718150d062f0fb34702d1443c701f0e1ac59e9", + "implementation_overlays": { + "acs_fingerprint_sharing": false, + "budget_detached_view_correction": false, + "compilation_cache": false + }, + "limits": { + "automatic_retry": false, + "children": 0, + "cpu_seconds": 7200, + "engine": false, + "network": false, + "numeric_threads": 1, + "peak_rss_bytes": 34359738368, + "wall_seconds": 10800 + }, + "manifest_metadata_sha256": "2168b79ff60b481636b4dacf504ff70f4f1934b92359d86cb95ce1ae78987b7a", + "model_quality_established": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_identity_verification": "config/staging/manifest/guard before-after metadata; no postcheck payload read, hash or stat", + "native_input_bytes": 3735332980, + "native_input_count": 10, + "owned_columns": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response" + ], + "peak_rss_bytes": 11452743680, + "postcheck_script_sha256": "ae9b4dacefbfb06c7571c81a55a87f50c080b9e3c27e4c3b39f8e4f878a95d1e", + "puf_enrichment_acceptance": false, + "receipt_sha256": "aea67b38a7a9af17a87b3bdce9e31c8033bdf2d14f7f01191cc7ec12d0d43eae", + "release_acceptance": false, + "remaining": [ + "Required cached replay on this exact native source and graph", + "Actual two-route PUF55 fitting and single finalization/attachment", + "Model-input coverage and fit-quality checks", + "National and congressional-district calibration/holdouts", + "Full and pruned export verification with exact-file dashboard" + ], + "required_replay_run": false, + "resource_map_sha256": "62a6329a8d84ad96410581f4e5043c4b62fb648a7aa7737a2ee8eed61a80a7bc", + "root_postcheck_sha256": "50eabf02b476f2fdc55e15812e374fd76cf6002feb7fd60f10810e4d2122ac8c", + "runtime_result": 0, + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "one_selected_native_survey_atomic_financial_cold_development_pilot", + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "cold_development_pilot_accepted", + "targets": [ + "survey_current_INT_VAL", + "survey_current_DIV_VAL", + "survey_current_CAP_VAL" + ], + "test_outcomes": [ + "passed" + ], + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_safe_owned": 505 + }, + "wall_seconds": 6607.312281332968 +} From d80593b2a98df9334dd97648dac1200d19c96128 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 08:57:50 -0400 Subject: [PATCH 060/255] Preserve missing merge metadata and record verified calibration controls --- docs/us-launch-review.md | 38 +++- ...alibration-consolidation-149-20260910.json | 50 +++++ ...uf55-canonical-create-failed-20260910.json | 32 +++ ...f55-canonical-cycle39-failed-20260910.json | 34 +++ ...test_spec_seed_identity_system_metadata.py | 194 ++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 6 +- 6 files changed, 346 insertions(+), 8 deletions(-) create mode 100644 experiments/us-calibration-consolidation-149-20260910.json create mode 100644 experiments/us-puf55-canonical-create-failed-20260910.json create mode 100644 experiments/us-puf55-canonical-cycle39-failed-20260910.json diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 62927e734..a6c8b6e85 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -47,8 +47,9 @@ need reconciliation; this draft does not supersede either review. The first native twenty-node survey, block-geography and financial cold pilot now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 -household records and 6,928 person records after cloning. The run assigns one -block before cloning, retains location inheritance and attaches seven financial +household records and 6,928 person records after cloning. Ten financial nodes ran +cold over the stored ten-node prefix, which retains the block assigned before +cloning. The composition retains location inheritance and attaches seven financial fields while preserving the other population fields. It took 110.12 minutes and 11.45 GB peak memory. The final receipt and independent source/resource postcheck pass. This is the original frozen 491-source implementation; newer @@ -197,8 +198,9 @@ The following work remains open: the build must retain one assigned Census block and derive its larger geographies. Prior wages remain excluded. A source operator's existence is not complete cell coverage. -4. Extend the accepted small native replay to the complete build. The genuine - seven-node age-development run now passes at 1/1000: 1,584 source households, +4. Retain the earlier seven-node age-development result as historical evidence; + the twenty-node cold/replay pilot in item 1 is the newer native milestone. + The seven-node run passed at 1/1000: 1,584 source households, 3,168 records after the support clone, final export/readback and owner/target verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. @@ -218,8 +220,27 @@ maintained whole-cohort finalizer for both routes together and either route alone. The tests also refuse changed donor row order, weights and late raw or donor mutations. This verifies the numerical layer; the canonical donor CREATE, typed model ancestry and complete survey attachment remain separate graph checks. +The first canonical donor CREATE attempt failed during kernel construction +because its live-code checker followed a circular function closure. The +[failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) +is preserved. The cycle correction passed its ten new marker controls, but +the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) +still refused before CREATE because the code-state snapshot changed during +initialization. That exact changing member is being diagnosed; neither donor +version is adopted, and the native full PUF host remains pending. See the [numerical evidence](../experiments/us-puf55-two-route-numerical-20260910.json). +All 149 selected calibration controls now pass on the combined US/UK solver, +including grouped bounds, fixed zero support, informed gates, budget search, +ordinary best-iterate behavior and strict diagnostic payloads. The run took +4.57 wall seconds at 394 MB peak memory; independent checks confirm the exact +34 source files, copied fixtures and all 149 unique test results. The preceding +run stopped before tests because two still-denied import probes exceeded their +reporting ceilings; the fresh invocation changed only those reporting ceilings +and run identities. No data access was added. This is invented-data solver +verification, separate from native national/CD calibration; see the +[calibration evidence](../experiments/us-calibration-consolidation-149-20260910.json). + Recipient qualification now covers all 28 distinct original public controls: 27 passed in the original run and the five-case corrected detached-output run passed, with four overlapping controls. The original run remains failed because @@ -265,9 +286,12 @@ checks financial replay and a subsequent weight-only change. The [seven remainin now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, 802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls are accepted on the same source snapshot. A subsequent source review found an -additional detached-view callback gap in the budget and weight-only views; -targeted corrections and regression controls are in progress. Native calibration -remains separate. +additional detached-view callback gap in the budget and weight-only views. +The correction is adopted after four targeted controls passed in 543 wall +seconds, including nine callback branches and independent source/resource checks. +The original eight and corrected four are separate source-version checks; see +the [correction evidence](../experiments/us-budget-detached-view-correction-20260910.json). +The native pilots exclude this overlay, and native calibration remains separate. A separate, bounded profile of the unchanged invented cold/required financial fixture passes its test, original teardown and all final code/resource checks: diff --git a/experiments/us-calibration-consolidation-149-20260910.json b/experiments/us-calibration-consolidation-149-20260910.json new file mode 100644 index 000000000..166e89640 --- /dev/null +++ b/experiments/us-calibration-consolidation-149-20260910.json @@ -0,0 +1,50 @@ +{ + "cases": 149, + "checks": { + "copied_test_helper_fixture_files": 10, + "exact_case_and_junit_rosters": true, + "junit_selector_order": true, + "junit_unique_names": 149, + "owned_before_after_current_match": true, + "owned_files": 20, + "source_before_after_current_match": true, + "source_files": 34, + "terminal_pins_final_match": true + }, + "coverage": [ + "US grouped upper bounds, fixed zero support and unsupported combinations", + "UK informed gates and budget search", + "ordinary best iterate plus same-runtime frozen optimizer controls", + "dense/CSR combined behavior and strict diagnostic payload/rebuild" + ], + "cpu_seconds": 3.9523219999999997, + "current_34_source_subset_matches": true, + "date": "2026-09-10", + "excluded_control": "cross-runtime saved golden requiring18 interop threads; same-runtime frozen optimizer comparisons still executed", + "failed_predecessor_receipt_sha256": "240b17fdb30b8dfa0cddb3f58343fc4a1ba0bc14e44b852cd188cb1bd7a6703b", + "full_checkout_certified": false, + "guard_sha256": "c4218acbd403c1ff4b05f3cbb4d47654e1a1b51cbcf7f6b8b045c9252248aa80", + "junit_sha256": "57ba56887978b71aa86152f36f83484471a216829f4b7cca9b20c15ea1af9bee", + "native_calibration_accepted": false, + "peak_rss_bytes": 394067968, + "postchecker_sha256": "ee5bd078009c7842d7f9c19adf5452bc2c1c8c97a44c4a890c6fd4e7ee924bff", + "predecessor_failure": "collection stopped before tests: two still-denied stdlib ZIP probe counts exceeded their reporting ceilings", + "receipt_sha256": "242fa8cb76d5e6f1fa4589319bf6764c4f63620b3cda2e9e2a15f69549f362c3", + "release_acceptance": false, + "root_postcheck_sha256": "dca84da504acf53c65d68f7076ebe1ecf12a8d64a6479c40255c164717f3d10d", + "schema_version": "microcosm.scoped-calibration-control.v1", + "scope": "exact invented calibration149 source/runtime evidence only; no native, release or unrelated consolidated-checkout acceptance", + "selectors": 68, + "source_files": 34, + "source_map_sha256": "7de4ee1007343d72a3618cd95850700cb28bf87cd89405d956a0f92d37634850", + "status": "accepted_invented_calibration149", + "successor_change": "fresh once-only run; two reporting ceilings changed2 to3; exact same source, selectors, permissions and600CPU/900wall/one-thread limits", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 149 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.565558874979615 +} diff --git a/experiments/us-puf55-canonical-create-failed-20260910.json b/experiments/us-puf55-canonical-create-failed-20260910.json new file mode 100644 index 000000000..e2d8251d6 --- /dev/null +++ b/experiments/us-puf55-canonical-create-failed-20260910.json @@ -0,0 +1,32 @@ +{ + "corrected_source_pending": true, + "cpu_seconds": 7.677668, + "date": "2026-09-10", + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_DEPTH", + "failure_stage": "kernel construction, while traversing cyclic function closure/mapping references", + "guard_sha256": "b50c65325ec901aada10368562ba481f3253f8d4fbdfba7c38fb6a5089de7612", + "independent_physical_postcheck": false, + "junit_sha256": "c9f6146ec4ada2ea277311ff8f9609207585acc7c12233ef6f9437f273a82e4d", + "native_input_access": false, + "passed_cases_reported": 1, + "peak_rss_bytes": 500629504, + "production_adopted": false, + "receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "6fdb0549fe8331db32450f3161a2d8cf1e00096c9245728e84360a08050277da", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 4, + "skipped": 0, + "tests": 29 + }, + "unexpected_refusals": {}, + "wall_seconds": 10.18145758396713 +} diff --git a/experiments/us-puf55-canonical-cycle39-failed-20260910.json b/experiments/us-puf55-canonical-cycle39-failed-20260910.json new file mode 100644 index 000000000..8f4142131 --- /dev/null +++ b/experiments/us-puf55-canonical-cycle39-failed-20260910.json @@ -0,0 +1,34 @@ +{ + "automatic_retry": false, + "cpu_seconds": 8.717074, + "date": "2026-09-10", + "exact_changed_module_or_member_identified": false, + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_STATE_CHANGED", + "failure_stage": "first executor implementation_hash after source fixture initializes, before CREATE execution", + "guard_sha256": "367e4de4c1587082d36dd4ae1f09847e7e8db3ef312ba33f5e867addf1119318", + "independent_physical_postcheck": false, + "junit_sha256": "fc00a99888c91d1d8fb6cf80ad381236a2c25d8b64cd0a95fa3d02a8951645d3", + "native_input_access": false, + "passed_cases_reported": 14, + "peak_rss_bytes": 507789312, + "predecessor_receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "production_adopted": false, + "receipt_sha256": "ec5dfec245cc5cf1a29aebb9afe81f0768dcd79cefa9c28e2610af5c1c79c229", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "027ab7d21adf3f74cac798e759eed27bd59d20a1b769839549e557d8c61ba0a8", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 1, + "skipped": 0, + "tests": 39 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.895262666977942 +} diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py index 2ea24cad3..8b4c89b98 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -157,3 +157,197 @@ def test_maps_alias_does_not_admit_other_paths_descriptors_or_children( else: hook("open", (path, "r", os.O_RDONLY)) assert refusals == [code] + + +@pytest.fixture +def coordinate_environment(diagnostic, monkeypatch): + """Exercise coordinate validation against an isolated invented environment.""" + values = { + "DIAG_CHECKOUT_SHA": "1" * 40, + "GITHUB_SHA": "1" * 40, + "DIAG_WORKFLOW_SHA": "2" * 40, + "DIAG_PR_HEAD_SHA": "3" * 40, + "DIAG_PR_BASE_SHA": "4" * 40, + "DIAG_MERGE_SHA": "5" * 40, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_RUN_ID": "123456", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": "PolicyEngine/microcosm", + } + # coordinates() consumes only environ. Keep the real process environment, + # filesystem, dependency imports and bootstrap paths outside these cases. + monkeypatch.setattr(diagnostic, "os", SimpleNamespace(environ=values)) + return values + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_coordinates_preserve_all_distinct_supplied_identities( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + original = dict(values) + assert diagnostic.coordinates() == { + "checkout_sha": "1" * 40, + "event_sha": "1" * 40, + "workflow_sha": "2" * 40, + "pr_head_sha": "3" * 40, + "pr_base_sha": "4" * 40, + "event_merge_sha": "5" * 40, + "github_run_id": "123456", + "github_run_attempt": "2", + "event": event, + "repository": "PolicyEngine/microcosm", + } + assert values == original + + +@pytest.mark.parametrize("missing", (False, True)) +def test_pull_request_coordinates_keep_absent_merge_metadata_empty( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + if missing: + del values["DIAG_MERGE_SHA"] + else: + values["DIAG_MERGE_SHA"] = "" + result = diagnostic.coordinates() + assert result["event"] == "pull_request" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + assert result["pr_head_sha"] == "3" * 40 + assert result["pr_base_sha"] == "4" * 40 + assert values.get("DIAG_MERGE_SHA", "") == "" + + +@pytest.mark.parametrize("missing", (False, True)) +def test_push_coordinates_allow_absent_pull_request_metadata( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = "push" + for name in ("DIAG_PR_HEAD_SHA", "DIAG_PR_BASE_SHA", "DIAG_MERGE_SHA"): + if missing: + del values[name] + else: + values[name] = "" + result = diagnostic.coordinates() + assert result["event"] == "push" + assert result["pr_head_sha"] == result["pr_base_sha"] == "" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_CHECKOUT_SHA"), + ("push", "GITHUB_SHA"), + ("push", "DIAG_WORKFLOW_SHA"), + ), +) +def test_required_coordinate_hashes_still_refuse_when_absent( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + del values[name] + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("pull_request", "DIAG_MERGE_SHA"), + ("push", "DIAG_PR_HEAD_SHA"), + ("push", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_MERGE_SHA"), + ), +) +def test_nonempty_coordinate_hashes_still_require_valid_format( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values[name] = "not-a-sha" + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + "value", ("null", "A" * 40, "f" * 39, "f" * 41, "f" * 40 + "\n") +) +def test_missing_merge_exception_does_not_accept_malformed_present_hashes( + diagnostic, coordinate_environment, value +): + coordinate_environment["DIAG_MERGE_SHA"] = value + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_checkout_must_match_event_even_without_merge_metadata( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values["DIAG_MERGE_SHA"] = "" + values["DIAG_CHECKOUT_SHA"] = "6" * 40 + with pytest.raises(diagnostic.RefusalError, match="^CHECKOUT$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value"), + ( + ("GITHUB_RUN_ID", None), + ("GITHUB_RUN_ID", ""), + ("GITHUB_RUN_ID", "not-a-run"), + ("GITHUB_RUN_ATTEMPT", None), + ("GITHUB_RUN_ATTEMPT", ""), + ("GITHUB_RUN_ATTEMPT", "not-an-attempt"), + ), +) +def test_coordinate_run_identity_remains_required( + diagnostic, coordinate_environment, name, value +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^RUN_COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value", "code"), + ( + ("GITHUB_EVENT_NAME", None, "EVENT"), + ("GITHUB_EVENT_NAME", "", "EVENT"), + ("GITHUB_EVENT_NAME", "workflow_dispatch", "EVENT"), + ("GITHUB_REPOSITORY", None, "REPOSITORY"), + ("GITHUB_REPOSITORY", "invented/other", "REPOSITORY"), + ), +) +def test_coordinate_event_and_repository_boundaries_remain_required( + diagnostic, coordinate_environment, name, value, code +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + diagnostic.coordinates() diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 355a99b98..291cca3ca 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -211,7 +211,11 @@ def coordinates() -> dict[str, str]: event = os.environ.get("GITHUB_EVENT_NAME", "") require(event in {"push", "pull_request"}, "EVENT") for key, value in result.items(): - optional = event == "push" and key.startswith(("pr_", "event_merge")) + # The event payload may omit merge_commit_sha. Preserve that absence; + # checkout/event SHAs are separate coordinates, not a substitute value. + optional = key == "event_merge_sha" or ( + event == "push" and key.startswith("pr_") + ) require( bool(re.fullmatch(r"[0-9a-f]{40}", value)) or (optional and not value), "COORDINATES", From c5b25a91067bcc3a09c6720d0ef3764e0b0d0a85 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 10:18:13 -0400 Subject: [PATCH 061/255] Add verified PUF donor graph and finalized output seals --- ...-puf55-canonical-and-output-seals.added.md | 4 + docs/us-launch-review.md | 47 +- docs/us-uk-release-path.md | 95 +++ ...canonical-create-47-controls-20260910.json | 52 ++ ...ical-output-seal-46-controls-20260910.json | 115 +++ .../us_runtime/graph_puf55_canonical_donor.py | 635 +++++++++++++++++ .../build/us_runtime/puf55_canonical_donor.py | 27 +- .../us_runtime/puf55_route_finalization.py | 25 +- .../test_us_graph_puf55_canonical_donor.py | 661 ++++++++++++++++++ ...t_us_puf55_route_numerical_finalization.py | 57 ++ .../tests/test_us_puf59_canonical.py | 65 ++ 11 files changed, 1772 insertions(+), 11 deletions(-) create mode 100644 changelog.d/us-puf55-canonical-and-output-seals.added.md create mode 100644 docs/us-uk-release-path.md create mode 100644 experiments/us-puf55-canonical-create-47-controls-20260910.json create mode 100644 experiments/us-puf55-numerical-output-seal-46-controls-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py create mode 100644 packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py diff --git a/changelog.d/us-puf55-canonical-and-output-seals.added.md b/changelog.d/us-puf55-canonical-and-output-seals.added.md new file mode 100644 index 000000000..0e6679803 --- /dev/null +++ b/changelog.d/us-puf55-canonical-and-output-seals.added.md @@ -0,0 +1,4 @@ +Add a source-bound PUF55 canonical donor graph node with cold/replay controls, +stable live-code identity checks, and finalized-candidate seals for the shared +two-route numerical finalizer. Complete native survey attachment and release +validation remain separate build steps. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index a6c8b6e85..e5ec261fe 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -14,6 +14,29 @@ exports. Enrichment clones inherit their household's assigned location. The PUF operator acts on the combined survey frame. A separate ASEC–PUF base is not the intended architecture. +National and local analysis share one fully constructed, enriched Frame. The +analysis branch begins after harmonization, atomic geography assignment, cloning +and imputation. Calibration changes weights; L0 controls sparsity. An export can +select a geographic scope and remove zero-weight households, but must preserve +every retained record's input values, knownness, entity relationships and assigned +geography. There is no separate national or local enrichment operation. A source +or mapping correction produces a new common parent Frame for both views. + +Release verification must bind each full or pruned export to that parent's +identity and its calibration target/weight specification, compare retained +inputs by stable entity ID, and reject changed inputs, dangling entity links or +regenerated geography. Weight and scope changes are explicit exceptions; record +values are not. This is the required contract, not a claim that all of these +checks have already passed on the release candidate. The separate legacy +ACS-local hours omission in [#765](https://github.com/PolicyEngine/microcosm/issues/765) +illustrates the drift this common construction path removes. + +Pre-calibration source quality and post-calibration export integrity are distinct +checks. Joint calibration may change an origin's contribution or set its weights +to zero. A pruned export must not be required to make every survey origin +independently resemble a national population, or to rerun imputation to satisfy +such a requirement. + The optional ten-node survey prefix now assigns a block before cloning, and its thirteen-node age-calibration extension passes cold execution and required replay on invented originals. A twenty-node extension now carries atomic @@ -164,6 +187,10 @@ from the benchmark's accepted baseline. The following work remains open: +The [US and UK release path](us-uk-release-path.md) puts these implementation +items in the complete population-quality, calibration and consumer-release +sequence. Passing component controls is one step in that sequence. + 1. Extend the accepted native twenty-node atomic and financial cold/replay pilot to the complete survey scale. Its invented cold/required replay, complete retained @@ -214,21 +241,29 @@ The following work remains open: These are active workstreams. No release, merge or deployment is implied by this draft, and the source changes do not relax the outstanding acceptance checks. -The two-route numerical finalization layer now passes 45 invented controls, +The two-route numerical finalization layer now passes 46 invented controls, including 110 actual target fits, 220 applications and comparisons against the maintained whole-cohort finalizer for both routes together and either route alone. The tests also refuse changed donor row order, weights and late raw or -donor mutations. This verifies the numerical layer; the canonical donor CREATE, -typed model ancestry and complete survey attachment remain separate graph checks. +donor mutations. The v2 numerical receipt seals the complete finalized candidate +before return and refuses a change during receipt encoding. This verifies the +numerical layer; typed model ancestry and complete survey attachment remain +separate graph checks. The first canonical donor CREATE attempt failed during kernel construction because its live-code checker followed a circular function closure. The [failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) is preserved. The cycle correction passed its ten new marker controls, but the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) still refused before CREATE because the code-state snapshot changed during -initialization. That exact changing member is being diagnosed; neither donor -version is adopted, and the native full PUF host remains pending. -See the [numerical evidence](../experiments/us-puf55-two-route-numerical-20260910.json). +initialization. The changing member was the serialized Python code object; +the corrected marker retains the actual code object and its immutable fields. +After correcting three obsolete test accesses to the public PopulationView API, +all 47 canonical CREATE/converter controls pass, including cold execution, +required replay and teardown. The corrected numerical and canonical sources are +adopted locally. These checks use invented donors; the native full PUF host +remains pending. See the +[46-control numerical evidence](../experiments/us-puf55-numerical-output-seal-46-controls-20260910.json) +and [47-control canonical evidence](../experiments/us-puf55-canonical-create-47-controls-20260910.json). All 149 selected calibration controls now pass on the combined US/UK solver, including grouped bounds, fixed zero support, informed gates, budget search, diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md new file mode 100644 index 000000000..8262cf434 --- /dev/null +++ b/docs/us-uk-release-path.md @@ -0,0 +1,95 @@ +# US and UK release path + +Planning snapshot, 10 September 2026. This is a release work plan, not a +certification of either country. Component tests, historical candidates and +current-candidate acceptance are distinct evidence. + +## Products and common construction + +The US product must support national and congressional-district analysis. The +UK product must support national, constituency and local-authority analysis. +Each country should have one fully constructed population from which full and +smaller analysis files are derived. Record the survey, monetary, policy and +boundary reference periods explicitly; do not infer them from a filename. + +After harmonization, assign one atomic area to each household and derive larger +geographies from versioned mappings. Geographic support expansion, when needed, +precedes assignment and has explicit identities and weights. Subsequent +enrichment clones inherit the assigned location. Once enrichment is complete, +calibration changes household weights; scope selection and pruning retain the +original values, missingness and relationships for every retained household. + +For a shared set of weights and consistent measurement definitions, additive +local totals must reconcile to national totals. Separately refitted compact +views need declared approximation tolerances and their own diagnostics. A +common source population alone does not guarantee identical estimates. + +## Release sequence + +| Stage | US work | UK work | Exit evidence | +| --- | --- | --- | --- | +| Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | +| Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | +| Geographic support | Block assignment before enrichment cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment and derivation for each nation; adequate constituency/LA support | Observed geography respected; stable household draws; clone/export inheritance; support and fit measured at advertised levels | +| Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | +| Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | +| Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | +| Release and consumption | Exact-file dashboard, methods/papers, immutable package and real PolicyEngine loading | Exact-file dashboard and certification; immutable cut, loader/pointer integration and PolicyEngine-UK adoption | Fresh export/readback, reproducible manifest and checksums, consumer calculation smoke test, reviewed release and rollback path | + +## Current position + +The US new graph has passed a small real survey → block assignment → clone → +financial-enrichment build and required replay. The combined survey population +has 1,584 source households and 3,168 records after cloning. This is not the +complete enriched, calibrated national/CD file. The next critical sequence is +complete PUF/remaining input integration, a small real full build, model and +calibration evaluation, then progressive scale and release verification. See +[the US review guide](us-launch-review.md) for scoped acceptance records. + +UK work already includes raw-source enrichment, national/local calibration, +size experiments and release assembly. Its recorded P50/P95b 55,000-household +candidates fail area-support and target-fit checks; the skipped holdout does +not establish an incumbent-comparison pass. Historical dense assembly also +does not satisfy today's release battery. See +[the size evidence](../experiments/355-uk-dataset-size-receipts.md) and +[the current dense release runbook](uk-dense-release-assembly-runbook-762.md). + +The current UK route clones for geographic support and then uses the existing +OA ladder. It has not adopted the new shared, household-keyed atomic geography +graph immediately after harmonization. Maria's +[PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle +household targets and the Northern Ireland lookup; that source work does not +by itself migrate the assignment owner. Anthony's +[PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging +telemetry and synthetic smoke evidence, not a calibrated release. + +UK release coverage must be re-established on the chosen integrated build and +locked model. The declared 145-input contract's older candidate evidence must +not certify the current stage set. The +[migration epic](https://github.com/PolicyEngine/microcosm/issues/665) tracks +source and calibration work; +[exact-count release parity](https://github.com/PolicyEngine/microcosm/issues/898) +and [default promotion](https://github.com/PolicyEngine/microcosm/issues/823) +remain separate consumer-release requirements. Existing open issues are leads +for candidate-specific verification, not proof every historical defect remains. + +## Team meeting decisions + +1. Confirm the release years, advertised geographies and full/compact products. +2. Assign one accountable owner to each country candidate and one owner to each + shared dependency: runtime, calibration, diagnostics and release consumers. +3. Review every row above for the latest actual candidate. Name the missing + artifact and next measurable result, rather than reporting code completion + as population acceptance. +4. Agree calibration, holdout, incumbent and representative-reform criteria, + including which differences need explanation and which block release. +5. Set dates for the next complete small candidate, the full quality review and + the production-sized candidate using measured runs rather than linear timing + extrapolation. + +Country data work, target/source reconciliation, independent validation, and +dashboard/consumer integration can run in parallel. Each final dashboard and +release verdict must refer to the same candidate bytes. Cross-sectional release +does not require retirement dynamics, every possible forecast year, or a +complete Chronicle/Orrery redesign. Keep those workstreams active without +making them prerequisites for the first accepted cross-section. diff --git a/experiments/us-puf55-canonical-create-47-controls-20260910.json b/experiments/us-puf55-canonical-create-47-controls-20260910.json new file mode 100644 index 000000000..0942a6a1b --- /dev/null +++ b/experiments/us-puf55-canonical-create-47-controls-20260910.json @@ -0,0 +1,52 @@ +{ + "cpu_seconds": 11.761269, + "current_checkout_runtime_certification": false, + "failed_predecessor_accepted": false, + "guard_sha256": "3dd78388b0426c92183db97aaa48516903b61ef9f4f646f35b9301c56662aa7a", + "junit_sha256": "0e6a0a3d4f77363c7ccd1705afd75e8082219ca4a24cfc6d36db383b77e2daab", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_access": false, + "observed_outcomes": { + "error": 0, + "failed": 0, + "passed": 47, + "skipped": 0 + }, + "peak_rss_bytes": 501907456, + "postcheck_script_sha256": "f1c8d68d9c4edd92391317683f308f599f6dbc771a4ef9c538acf7d6e4b679e0", + "preceding_stable47_failed_postcheck_sha256": "cb2e3a344b4fa2e7bef7b1e60cf085b8a583567e54c7c2c940345639bc9081ce", + "preserved_failed_predecessors": [ + "us-puf55-canonical-create-failed-20260910.json", + "us-puf55-canonical-cycle39-failed-20260910.json" + ], + "receipt_sha256": "8f7ac625296fbb22dcd976918b4718af14cbaf31288e528583d1fb49ad6e95c9", + "release_acceptance": false, + "resource_map_sha256": "57927f284c647abf03ea1e8370ddeb6035b4e68ff93af1e4f745a5027307e7ef", + "root_postcheck_sha256": "5617f7949e261d42975cb088671f83bda71c37d424e190a209766df5e4ce20ec", + "scope": "47_invented_canonical_donor_and_converter_controls", + "scope_note": "The exact frozen canonical donor CREATE and projection controls pass cold execution and required replay. Source adopted locally. Native donor, complete survey attachment, calibration and release acceptance remain separate.", + "source_adoption": true, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py": "a04c8dd2d89b4d3e5c563707587545299bd218aadf567ee7e772b9bbc5755311", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py": "6a8d07e38f5f0bff2619a7eb53fd0f34a10eed798f79de44c82e7740db923066", + "packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py": "675bd57728a6ef56e79a0a24c79713fbb98e454be385fae607528d57048ce76d", + "packages/microcosm-build/tests/test_us_puf59_canonical.py": "85a6d41e716e0d3f1794851dd6cc6007603834a22982f31058fa5bf6da31df6d" + }, + "source_graph_control_execution": "actual canonical CREATE cold and required replay with all 47 selected assertions and original fixture teardown passed", + "source_map_sha256": "dd16d99e9d0bbc3b1bf3aacddcd1159887758c2a8183bcee0333715270bcd9bb", + "status": "frozen_canonical_create_and_converter_controls_accepted", + "survey_attachment_host_execution": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 47 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 509 + }, + "wall_seconds": 14.356884333014023 +} diff --git a/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json new file mode 100644 index 000000000..17567c038 --- /dev/null +++ b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json @@ -0,0 +1,115 @@ +{ + "actual_model_chain_verified": true, + "approval_sha256": "359a56ef088584a149b04a6a4bfa5e047c97f6e90db368376cbcf67d1564756d", + "bridge_owner_sha256": "ca49788ebd67ea807b072cbfbf4817fd7051f967f77098d0e2f8586576add13f", + "cpu_seconds": 18.093259, + "current_checkout_runtime_certification": false, + "financial_fixture_execution": false, + "fixture_apply_target_calls": 220, + "fixture_donors_per_model": 4, + "fixture_fit_target_calls": 110, + "fixture_trees": 2, + "full65_and_puf55_chain_acceptance": false, + "graph_host_execution": false, + "guard_sha256": "5bc1ac74e18b6379d51cf34e55ada8afb17fbb11d06428ebaefd81b4a92baab6", + "host_receiving_value_seal_still_required": true, + "junit_sha256": "aa2825f0279ed6cdbec8b08cad1eebbc287d2bd6d813a538a68b26885c76f481", + "model_cache_metadata_scans": 1506, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "new_numerical_cases": 28, + "numerical_test_sha256": "7552d83574e000b5e5279e8155711f0d4435125a7e2fe95571ef8b4f9189db26", + "one_whole_cohort_finalizer_per_call": true, + "peak_rss_bytes": 680542208, + "peer_target_review_sha256": "a616d764c903fd90383f7e2b6147bd9f065ff4164e0b98f4ce315f8b646d7b42", + "postcheck_script_sha256": "c0a6e6f5ff02a9086488702b04a5d6c475c903108e6c933c654941141aa0bd59", + "preserved_failed_predecessor": { + "accepted": false, + "failed": 1, + "passed": 45, + "reason": "The new test addressed the source alias employment_income instead of finalized employment_income_before_lsr. It failed before applying its intended mutation.", + "receipt_sha256": "26bf35628a60abacbf765725ebc7cab3a0ed0afd844b1d71e9a153fa97ca6ef7", + "root_failed_postcheck_sha256": "709621c49cd300e83c96fe8100d5257c462b86e3b96b0dd29a4585cbd94c025b", + "tests": 46, + "unexpected_denied_traceback_labels": 22 + }, + "production_adopted": true, + "project_imports_or_runtime_reexecution": false, + "puf_fit_and_numerical_finalization": true, + "receipt_sha256": "b9e6349583a1959e918e491c1267c28535919fc2ef699d1608e9ea52b951f020", + "release_acceptance": false, + "resource_map_sha256": "8add0e99307b3cfe6ba7fb5cc4f68f711d5bf7f82ccde8213bd52ed04ab4ee80", + "root_postcheck_sha256": "e85d557093ac8b53ea4c16eac883b9dae2598889ca35cb56f8db0f9b0f270098", + "root_review_sha256": "73ae7e8ea10b317de40d2a11419aaaf15437b48cd689eb27a1ce6d4473cfc325", + "scope": "46_invented_two_route_numerical_and_raw_merge_controls", + "scope_note": "All 46 invented numerical/raw controls pass on the frozen source. The v2 numerical receipt seals the complete finalized candidate before return. The added test changes an owned finalized value during receipt encoding and requires refusal. Source is adopted locally; native donor/source admission, graph host ancestry, complete enrichment, calibration and release remain separate.", + "selected_test_cases": [ + "test_two_real_chains_finalize_once_on_the_complete_original_cohort", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[0]", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[1]", + "test_donor_parity_refuses_before_any_model_decode[recid_order]", + "test_donor_parity_refuses_before_any_model_decode[target]", + "test_donor_parity_refuses_before_any_model_decode[weight]", + "test_donor_parity_refuses_before_any_model_decode[capacity]", + "test_donor_parity_refuses_before_any_model_decode[shared_money]", + "test_donor_parity_refuses_before_any_model_decode[shared_status]", + "test_route_state_and_actual_model_consumption_are_bound[phase]", + "test_route_state_and_actual_model_consumption_are_bound[model_seed]", + "test_route_state_and_actual_model_consumption_are_bound[matrix]", + "test_route_state_and_actual_model_consumption_are_bound[producer]", + "test_route_state_and_actual_model_consumption_are_bound[raw_history]", + "test_route_state_and_actual_model_consumption_are_bound[last_model]", + "test_route_state_and_actual_model_consumption_are_bound[ninth_donor_value]", + "test_route_state_and_actual_model_consumption_are_bound[both_donor_targets]", + "test_route_state_and_actual_model_consumption_are_bound[both_recid_order]", + "test_route_state_and_actual_model_consumption_are_bound[both_weights]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_routes]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_clone_one]", + "test_last_model_return_cannot_change_a_retained_donor[target]", + "test_last_model_return_cannot_change_a_retained_donor[capacity]", + "test_model_donor_frame_matches_existing_single_profile_preparation", + "test_model_frame_constructor_return_cannot_coerce_a_donor_column", + "test_finalizer_return_cannot_change_retained_raw_or_donor[raw]", + "test_finalizer_return_cannot_change_retained_raw_or_donor[donor]", + "test_receipt_encoding_cannot_change_the_already_sealed_candidate", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[False]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[True]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_refuses_inexact_recipient_partition[overlap]", + "test_refuses_inexact_recipient_partition[missing]", + "test_refuses_inexact_recipient_partition[extra]", + "test_refuses_inexact_recipient_partition[clone_zero]", + "test_refuses_changed_matrix_route_or_storage[matrix]", + "test_refuses_changed_matrix_route_or_storage[order]", + "test_refuses_changed_matrix_route_or_storage[mutable_bytes]", + "test_refuses_changed_matrix_route_or_storage[profile]", + "test_complete_chain_history_is_checked_before_merge[seed]", + "test_complete_chain_history_is_checked_before_merge[raw]", + "test_complete_chain_history_is_checked_before_merge[model_history]", + "test_complete_chain_history_is_checked_before_merge[target_history]", + "test_changed_detached_decoder_values_refuse", + "test_later_decoder_cannot_change_an_earlier_route_table" + ], + "source_adoption": true, + "source_and_typed_producer_authentication_still_required": true, + "source_count": 500, + "source_map_sha256": "67bdcfc92a36563de2b866b6d6b3bca88c127e90854977c681f56d61b6988e20", + "status": "frozen_numerical_output_seal_controls_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 46 + }, + "unchanged_raw_cases": 18, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 508 + }, + "wall_seconds": 21.00948112498736 +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py new file mode 100644 index 000000000..086c0032e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py @@ -0,0 +1,635 @@ +"""One source-backed canonical PUF55 donor CREATE, shared by both routes. + +All ordinary returns reach one nine-predictor model Frame. An eight-predictor +training Slice can omit the transported SS total without constructing another +donor cohort. Technical model persons are not source persons. The emitted +receipts describe construction; the eventual host must verify actual source +keys, producer implementations and typed ancestry before trusting any model. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from collections.abc import Mapping +from dataclasses import fields, is_dataclass +from functools import partial +from pathlib import Path +from types import FunctionType + +import numpy as np + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + StructuralDelta, + codecs, + source_hash, +) +from microcosm.graph import canonical as graph_canonical +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import Population + +from . import full_puf_enrichment as full +from . import graph_full_puf_enrichment as physical +from . import puf55_canonical_donor as projection +from . import puf55_route_finalization as numerical +from . import puf59_canonical as canonical +from . import puf59_canonical_artifact as envelope +from . import puf_full_source as source +from . import puf_full_source_graph as source_graph +from . import puf_interest_components as interest +from . import puf_raw_source as raw + +CANONICAL_DONOR_NODE = "survey_puf55.canonical_donor" +CANONICAL_DONOR_TYPE = ArtifactType("microcosm.us.puf59_canonical_return", 2) +DONOR_PROJECTION_TYPE = ArtifactType("microcosm.us.puf55_donor_projection", 1) +PHASE = "us.survey_puf55.canonical_donor.v1" +MAX_PROJECTION_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_CANONICAL_SOURCE_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _pin(pin): + _require(type(pin) is raw.SourcePin, "PIN_TYPE") + values = tuple(getattr(pin, field.name) for field in fields(raw.SourcePin)) + _require( + all(type(value) is str for value in values[:3]) + and type(pin.bytes) is int + and all(type(value) is str for value in values[4:7]) + and type(pin.delivered_header) is tuple + and all(type(value) is str for value in pin.delivered_header) + and type(pin.data_records) is int, + "PIN_FIELDS", + ) + return values + + +def _definition(definition): + """Pure check against reconstructed literals, never caller-made authority.""" + _require(type(definition) is raw.PufRawSourceDefinition, "DEFINITION_TYPE") + payload, route, digest = definition.canonical, definition.route, definition.sha256 + _require( + type(payload) is bytes + and type(route) is str + and route in ("packaged", "test_fixture") + and type(digest) is str + and _sha(payload) == digest + and canonical_json(raw._thawed(definition.document)) == payload, + "DEFINITION_BYTES", + ) + rebuilt = raw._definition_from_document(json.loads(payload), route=route) + actual = ( + route, + payload, + digest, + _pin(definition.main), + _pin(definition.demographic), + ) + _require( + actual + == ( + rebuilt.route, + rebuilt.canonical, + rebuilt.sha256, + _pin(rebuilt.main), + _pin(rebuilt.demographic), + ), + "DEFINITION_PINS", + ) + return actual + + +def _fresh_definition(fixture_definition): + packaged_bytes = raw._packaged_bytes() + _require(type(packaged_bytes) is bytes, "PACKAGED_BYTES") + packaged = raw._definition_from_document( + json.loads(packaged_bytes), route="packaged" + ) + _definition(packaged) + if fixture_definition is None: + return packaged, _sha(packaged_bytes) + _definition(fixture_definition) + document = json.loads(fixture_definition.canonical) + _require( + fixture_definition.route == "test_fixture" + and document.get("authority") == "invented_fixture_nonauthority" + and not {fixture_definition.main.sha256, fixture_definition.demographic.sha256} + & {packaged.main.sha256, packaged.demographic.sha256}, + "FIXTURE_AUTHORITY", + ) + # Reproduce the maintained fixture contract with the freshly read packaged + # pins, without consulting or accepting its process-global _PACKAGED cache. + return raw._definition_from_document(document, route="test_fixture"), _sha( + packaged_bytes + ) + + +def _recipe(seed, growth_scheme): + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(growth_scheme) is str and growth_scheme in ("family_observed", "cpi_only"), + "GROWTH_SCHEME", + ) + qbi, growth = canonical.qbi, canonical.growth + parameters = qbi._json(qbi.model_parameters()).encode() + _require(_sha(parameters) == qbi.PARAMETERS_SHA256, "QBI_PARAMETERS") + _require( + _sha(growth._RECIPE_JSON.encode()) == growth.RECIPE_SHA256 + and canonical_json(growth.growth_recipe()) == canonical_json(growth._RECIPE), + "GROWTH_RECIPE", + ) + runtime_bands = interest.puf_e19200_agi_bands_runtime_identity() + return { + "phase": PHASE, + "seed": seed, + "growth_scheme": growth_scheme, + "source_statistical_year": 2015, + "money_year": 2024, + "qbi_parameters_sha256": qbi.PARAMETERS_SHA256, + "growth_recipe_sha256": growth.RECIPE_SHA256, + "interest_asset_sha256": canonical.INTEREST_ASSET_SHA256, + "interest_band_facts_sha256": canonical.INTEREST_BAND_FACTS_SHA256, + "interest_runtime_sha256": runtime_bands["sha256"], + "profile": full.PUF55_SURVEY_SS.value, + "ordinary_cohort": "complete; zero design weights retained", + } + + +def _params(definition, seed, growth_scheme, packaged_sha256): + return { + **_recipe(seed, growth_scheme), + "definition": definition.params_text, + "definition_sha256": definition.sha256, + "packaged_definition_bytes_sha256": packaged_sha256, + "source_pins": canonical_json( + (_pin(definition.main), _pin(definition.demographic)) + ).decode("ascii"), + } + + +def _node(definition, params): + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + _require(len(profile.predictors) == 9 and len(profile.targets) == 55, "PROFILE") + return Node( + CANONICAL_DONOR_NODE, + CanonicalPuf55DonorKernel.ref, + structural=StructuralDelta.CREATE, + sources=(definition.main.source_name, definition.demographic.source_name), + outputs=tuple( + Owned("tax_unit", name, "float64") + for name in (*profile.predictors, *profile.targets) + ), + params=params, + artifact_outputs=( + ArtifactOutput("full_return_source", source_graph.FULL_RETURN_SOURCE_TYPE), + ArtifactOutput("canonical_donor", CANONICAL_DONOR_TYPE), + ArtifactOutput("donor_projection", DONOR_PROJECTION_TYPE), + ), + description="Construct one complete source-pinned modeled PUF55 donor; nine/eight training Slices share its cohort.", + ) + + +def canonical_puf55_donor_node( + *, seed=578, growth_scheme="family_observed", fixture_definition=None +): + definition, packaged_sha = _fresh_definition(fixture_definition) + return _node(definition, _params(definition, seed, growth_scheme, packaged_sha)) + + +def _modules(): + # Include actual consumed codecs, numerical source/recipe owners and Frame + # construction/storage helpers. No survey issuer or country engine is used. + modules = ( + sys.modules[__name__], + raw, + source, + source_graph, + canonical, + canonical.qbi, + canonical.growth, + envelope, + projection, + interest, + full, + full.support, + numerical, + full.codec, + full.qrf_target, + physical, + physical.operator_column_contracts, + physical.population_ops, + physical.store_ops, + codecs, + graph_canonical, + sys.modules[full.Frame.__module__], + sys.modules[full.support.EntitySchema.__module__], + sys.modules[full.support.Weights.__module__], + ) + return tuple(dict.fromkeys(modules)) + + +def _marker(value, depth=0): + """Snapshot borrowed code/configuration as an immutable object graph. + + Functions can close over a namespace that contains the same function (for + example generated class annotation machinery). Record an object's local + reference before descending, so a back edge is explicit rather than an + unbounded expansion. Each new call captures all reachable contents again; + the memo is neither a source cache nor an exemption for mutable objects. + """ + seen = {} + + def visit(item, level): + kind = type(item) + if kind in (type(None), bool, int, str, bytes, float): + _require(level <= 16, "LIVE_DEPTH") + return kind, item.hex() if kind is float else item + previous = seen.get(id(item)) + if previous is not None: + _require(previous[0] is item, "LIVE_REFERENCE") + return "reference", previous[1] + _require(level <= 16, "LIVE_DEPTH") + reference = len(seen) + # Keep the object alive while traversing borrowed mappings/dataclasses, + # preventing an identity from being recycled during this snapshot. + seen[id(item)] = (item, reference) + if kind in (tuple, list): + contents = tuple(visit(child, level + 1) for child in item) + elif kind in (set, frozenset): + contents = frozenset(visit(child, level + 1) for child in item) + elif isinstance(item, Mapping): + contents = tuple( + (visit(k, level + 1), visit(v, level + 1)) for k, v in item.items() + ) + elif kind is np.ndarray and not item.dtype.hasobject: + contents = item.dtype.str, item.shape, item.tobytes() + elif is_dataclass(item) and not isinstance(item, type): + contents = tuple( + (field.name, visit(getattr(item, field.name), level + 1)) + for field in fields(item) + ) + elif kind is FunctionType: + # Keep identity as well as immutable code/default/closure contents; + # comparing retained mutable functions to themselves is not a seal. + # Marshal bytes also encode interpreter reference-sharing state; + # retaining a returned literal can change them without a code edit. + # Code objects and their constant graph are immutable. Keep the + # object alive, retain its identity, and snapshot public fields; + # mutable function defaults/closures still use visit below. + code_object = item.__code__ + code = ( + code_object, + id(code_object), + code_object.co_argcount, + code_object.co_posonlyargcount, + code_object.co_kwonlyargcount, + code_object.co_nlocals, + code_object.co_stacksize, + code_object.co_flags, + code_object.co_code, + code_object.co_consts, + code_object.co_names, + code_object.co_varnames, + code_object.co_freevars, + code_object.co_cellvars, + code_object.co_filename, + code_object.co_name, + code_object.co_qualname, + code_object.co_firstlineno, + code_object.co_linetable, + code_object.co_exceptiontable, + ) + defaults = visit(item.__defaults__, level + 1) + kwdefaults = visit(item.__kwdefaults__, level + 1) + closure = [] + for cell in item.__closure__ or (): + try: + cell_value = cell.cell_contents + except ValueError: + closure.append(("empty_cell",)) + else: + closure.append(("cell", visit(cell_value, level + 1))) + contents = ( + item, + code, + defaults, + kwdefaults, + tuple(closure), + ) + else: + contents = id(item) + return "object", reference, kind, contents + + return visit(value, depth) + + +def _live(): + result = [] + for module in _modules(): + result.append(("module", module, module.__name__, module.__file__)) + for name, value in vars(module).items(): + if name.startswith("__") or (module is raw and name == "_PACKAGED"): + continue + if isinstance(value, type) and value.__module__ == module.__name__: + members = [] + for member, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + members.append((member, _marker(function))) + elif not member.startswith("__"): + members.append((member, _marker(function))) + result.append((module.__name__, name, value, tuple(members))) + else: + result.append((module.__name__, name, _marker(value))) + result.append( + ( + "rng", + tuple( + _marker(value) + for value in ( + np.random.default_rng, + np.random.Generator, + np.random.PCG64, + np.random.SeedSequence, + ) + ), + ) + ) + return tuple(result) + + +def _frame_seal(frame): + return physical._population_stamp( + Population.from_frame(frame, CANONICAL_DONOR_NODE) + ) + + +class CanonicalPuf55DonorKernel(KernelBase): + ref = "us.survey_puf55.canonical_donor@1" + capabilities = Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=("numpy", "pandas"), + ) + + def __init__( + self, *, seed=578, growth_scheme="family_observed", fixture_definition=None + ): + definition, packaged_sha = _fresh_definition(fixture_definition) + self._definition = definition + self.source_codecs = raw.puf_raw_source_codecs(definition) + self.source_refs = raw.us_puf_raw_source_refs(definition) + params = canonical_json(_params(definition, seed, growth_scheme, packaged_sha)) + # Retain primitive pins and immutable baselines before any future borrow. + self._state = ( + definition, + _definition(definition), + params, + packaged_sha, + seed, + growth_scheme, + self.implementation_hash(), + _live(), + ) + self._check_state(self._state) + + def implementation_hash(self): + # The executor calls this on required replay too. Resource reads cannot + # live only in run(), which is deliberately skipped on a cache hit. + state = getattr(self, "_state", None) + if state is not None: + self._check_state(state) + packaged = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + code = source_hash(*_modules(), dependencies=self.capabilities.dependencies) + if state is not None: + self._check_state(state) + _require( + type(packaged) is bytes + and asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "IMPLEMENTATION_RESOURCES", + ) + return _sha( + canonical_json( + { + "code": code, + "packaged_definition_bytes_sha256": _sha(packaged), + "interest_asset": asset, + "csv_acceptance": raw.csv_acceptance_profile(), + } + ) + ) + + def _check_state(self, state): + definition, definition_seal, params, packaged_sha, seed, scheme, _, live = state + _require( + self._state is state + and self._definition is definition + and _definition(definition) == definition_seal + and canonical_json(_params(definition, seed, scheme, packaged_sha)) + == params + and self.source_refs == raw.us_puf_raw_source_refs(definition) + and _live() == live, + "LIVE_STATE_CHANGED", + ) + expected = (definition.main, definition.demographic) + _require( + type(self.source_codecs) is codecs.SourceCodecRegistry + and not self.source_codecs.names(), + "CODEC_MODE", + ) + _require( + set(self.source_codecs.bytes_names()) == {p.codec for p in expected}, + "CODEC_ROSTER", + ) + for pin in expected: + loader = self.source_codecs.get(pin.codec) + _require( + type(loader) is partial + and loader.func is raw.load_pinned_puf_bytes + and loader.args == () + and set(loader.keywords) == {"pin"} + and loader.keywords["pin"] is pin + and _pin(loader.keywords["pin"]) == _pin(pin), + "CODEC_BINDING", + ) + + def _context(self, context, state, expected_paths=None): + definition, _, params, *_ = state + _require( + context.node == _node(definition, json.loads(params)) + and canonical_json(dict(context.params)) == params + and not context.tables + and not context.weights + and not context.artifacts + and len(context.strata) == 0 + and set(context.sources) + == {definition.main.source_name, definition.demographic.source_name}, + "DECLARATION_OR_CONTEXT", + ) + paths = tuple( + (p.source_name, context.sources[p.source_name]) + for p in (definition.main, definition.demographic) + ) + _require( + all(type(path) is type(Path()) and path.is_absolute() for _, path in paths), + "SOURCE_PATH", + ) + _require( + expected_paths is None or paths == expected_paths, "SOURCE_PATH_CHANGED" + ) + return paths + + def _read_sources(self, paths, state): + definition = state[0] + self._check_state(state) + return tuple( + codecs.load_source_bytes(pin.codec, path, registry=self.source_codecs) + for pin, (_, path) in zip( + (definition.main, definition.demographic), paths, strict=True + ) + ) + + def _read_resources(self, state): + packaged_bytes = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + implementation = self.implementation_hash() + # All file/metadata reads precede these pure comparisons. + self._check_state(state) + _require( + type(packaged_bytes) is bytes + and _sha(packaged_bytes) == state[3] + and implementation == state[6], + "SOURCE_CODE_OR_RESOURCE_CHANGED", + ) + _require( + asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "INTEREST_RESOURCE", + ) + + def run(self, context): + state = self._state + self._check_state(state) + paths = self._context(context, state) + self._read_resources(state) + buffers = self._read_sources(paths, state) + definition, _, params, _, seed, scheme, *_ = state + decoded = source.decode_full_puf_source(*buffers, definition) + raw_payload = source.encode_full_puf_source(decoded) + constructed = canonical.construct_canonical_puf59( + decoded, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=canonical.INTEREST_ASSET_SHA256, + seed=seed, + growth_scheme=scheme, + ) + canonical_payload = envelope.encode_canonical_puf59( + constructed, expected_growth_scheme=scheme + ) + donor, evidence = projection.canonical_puf55_donor_from_artifact( + canonical_payload, + expected_artifact_sha256=_sha(canonical_payload), + expected_growth_scheme=scheme, + profile=full.PUF55_SURVEY_SS, + ) + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + selected = full._validated_model_donor(donor, profile=profile) + frame = numerical._model_donor_frame(selected) + ids = decoded.status["RECID"][decoded.ordinary] + _require( + np.array_equal(donor.index.to_numpy(), ids) + and np.array_equal(frame.table("tax_unit").index.to_numpy(), ids) + and frame.weights_for("tax_unit").values.tobytes() + == ( + decoded.status["S006"][decoded.ordinary].astype(np.float64) / 100 + ).tobytes() + and all(dtype == np.dtype("float64") for dtype in selected.dtypes), + "ORDINARY_COHORT_OR_CONVERSION", + ) + frame_sha = _frame_seal(frame) + receipt = { + **json.loads(params), + "protocol": PHASE, + "source_route": definition.route, + "source_rows": len(decoded.status["RECID"]), + "ordinary_rows": len(ids), + "zero_weight_rows": int((frame.weights_for("tax_unit").values == 0).sum()), + "full_return_source_sha256": _sha(raw_payload), + "canonical_donor_sha256": _sha(canonical_payload), + "projection": evidence, + "model_frame_sha256": frame_sha, + "source_recid_sha256": _sha(ids.astype("= 64 + recid = 999996 + i - 64 if aggregate else 501000 + i + row = {name: "0" for name in packaged.main.delivered_header} + mars = 0 if aggregate else 1 + i % 4 + secondary = int(mars == 2) + primary = int(not aggregate and i % 13 != 0) + row.update( + RECID=str(recid), + FLPDYR="2014" if i % 7 == 0 else "2015", + FLPDMO="12", + MARS=str(mars), + DSI="0", + S006="0" if i == 0 else str(125 + i), + XFPT=str(primary), + XFST=str(secondary), + XTOT=str(primary + secondary), + ) + if aggregate: + row["E19200"] = "" + else: + row.update( + E00200=str(40000 + i), + E00900=str(-300 if i % 5 == 0 else 2000 + i), + E00600="120", + E00650="30", + E25940="700", + E25980="500", + E25920="100", + E25960="50", + E26110="20", + E26170="200", + E26190="300", + E26160="10", + E26180="20", + E26100="5", + E26270="1495", + E00100=str(500 * i - 10000), + E19200=str(1000 + i), + E02400="50", + E03230="100", + E87530="300", + ) + rows.append(row) + if defect == "duplicate": + rows[0]["RECID"] = rows[1]["RECID"] + elif defect == "reordered": + rows = [*reversed(rows[:64]), *rows[64:]] + header = ",".join(packaged.main.delivered_header) + if defect == "header": + header = header.replace("RECID", "WRONG", 1) + main = ( + header + + "\n" + + "\n".join( + ",".join(row[name] for name in packaged.main.delivered_header) + for row in rows + ) + + "\n" + ).encode() + demo_row = dict( + RECID="501000", + AGEDP1="0", + AGEDP2="0", + AGEDP3="0", + AGERANGE="2", + EARNSPLIT="0", + GENDER="1", + ) + demo = ( + ",".join(packaged.demographic.delivered_header) + + "\n" + + ",".join(demo_row[name] for name in packaged.demographic.delivered_header) + + "\n" + ).encode() + document = owner.raw.definition_document_json(packaged) + document.update(route="test_fixture", authority="invented_fixture_nonauthority") + for name, payload, count in (("main", main, 68), ("demographic", demo, 1)): + document["sources"][name].update( + bytes=len(payload), + data_records=count, + sha256=hashlib.sha256(payload).hexdigest(), + git_blob_sha1=hashlib.sha1( + b"blob " + str(len(payload)).encode() + b"\0" + payload + ).hexdigest(), + ) + definition = owner.raw.fixture_definition(document) + root.mkdir() + paths = {} + for pin, payload in ((definition.main, main), (definition.demographic, demo)): + path = root / (pin.name + ".csv") + path.write_bytes(payload) + paths[pin.source_name] = path + return definition, paths, (main, demo) + + +def _context(kernel, paths): + node = owner._node(kernel._definition, json.loads(kernel._state[2])) + return KernelContext( + node=node, + tables={}, + weights={}, + strata=pd.Series([], dtype=object, name="stratum"), + params=node.params, + rng=np.random.default_rng(1), + sources=dict(paths), + ) + + +@pytest.fixture(scope="module") +def canonical_run(tmp_path_factory): + root = tmp_path_factory.mktemp("puf55_canonical_source") + definition, paths, buffers = _original_sources(root / "originals") + kernel = owner.CanonicalPuf55DonorKernel(fixture_definition=definition) + node = owner.canonical_puf55_donor_node(fixture_definition=definition) + assert node == _context(kernel, paths).node + compiled = compile_graph(Graph("us", kernel.source_refs, (node,))) + store = ContentStore(root / "store", codecs=kernel.source_codecs) + kernels = KernelRegistry() + kernels.register(kernel) + cold = run_graph(compiled, sources=paths, store=store, kernels=kernels) + warm = run_graph( + compiled, sources=paths, store=store, kernels=kernels, resume="require" + ) + yield SimpleNamespace( + definition=definition, + paths=paths, + buffers=buffers, + kernel=kernel, + node=node, + compiled=compiled, + cold=cold, + warm=warm, + store=store, + ) + kernel._check_state(kernel._state) + assert kernel._read_sources(tuple(paths.items()), kernel._state) == buffers + + +def _payloads(run, manifest): + record = manifest.node(run.node.id) + result = {} + for output in run.node.artifact_outputs: + assert record.typed_artifacts["outputs"][output.name] == descriptor( + producer=run.node.id, + artifact=output.name, + type_=output.type, + producer_key=record.key, + capabilities=record.capabilities, + ) + key = opaque_artifact_key(record.key, output.name) + assert record.opaque_artifacts[output.name] == key + result[output.name] = run.store.load_bytes(key) + return result + + +def test_actual_create_and_required_replay_preserve_all_ordinary_sources(canonical_run): + run = canonical_run + assert tuple(run.compiled.order) == (run.node.id,) + assert not run.cold.node(run.node.id).store_hit + assert run.warm.node(run.node.id).store_hit + assert run.cold.content_addressed == run.warm.content_addressed + cold, warm = _payloads(run, run.cold), _payloads(run, run.warm) + assert cold == warm + original = owner.source.decode_full_puf_source(*run.buffers, run.definition) + assert cold["full_return_source"] == owner.source.encode_full_puf_source(original) + arrays, receipt = owner.envelope.decode_canonical_puf59(cold["canonical_donor"]) + np.testing.assert_array_equal(arrays["RECID"], np.arange(501000, 501064)) + assert receipt["rows"] == 64 and receipt["source_statistical_year"] == 2015 + assert arrays["weight"][0] == 0 + np.testing.assert_array_equal( + arrays["weight"], original.status["S006"][original.ordinary] / 100 + ) + frame = run.cold.population(run.node.id) + assert owner._frame_seal(frame) == owner._frame_seal( + run.warm.population(run.node.id) + ) + np.testing.assert_array_equal(frame.table("tax_unit").tax_unit_id, np.arange(1, 65)) + np.testing.assert_array_equal(frame.table("tax_unit").index, arrays["RECID"]) + report = json.loads(cold["donor_projection"]) + assert report["source_rows"] == 68 and report["ordinary_rows"] == 64 + assert report["zero_weight_rows"] == 1 and report["source_route"] == "test_fixture" + for name in ( + "technical_person_rows_are_source_persons", + "source_admission_issued", + "population_admission_issued", + "current_money_tax_unit_reconstruction", + "release_eligible", + ): + assert report[name] is False + + +def test_both_predictor_slices_share_one_donor_cohort(canonical_run): + run = canonical_run + payload = _payloads(run, run.cold)["canonical_donor"] + nine, _ = owner.projection.canonical_puf55_donor_from_artifact( + payload, expected_artifact_sha256=owner._sha(payload) + ) + eight, _ = owner.projection.canonical_puf55_donor_from_artifact( + payload, + expected_artifact_sha256=owner._sha(payload), + profile=owner.full.PUF55_SURVEY_SS_NO_TOTAL, + ) + pd.testing.assert_frame_equal( + eight, nine.drop(columns=owner.full.SURVEY_SS_TOTAL_PREDICTOR) + ) + frame = run.cold.population(run.node.id) + for profile in (owner.full.PUF55_SURVEY_SS, owner.full.PUF55_SURVEY_SS_NO_TOTAL): + declaration = Slice("tax_unit", profile.predictors) + selected = frame.table(declaration.entity).loc[:, list(declaration.columns)] + assert selected.shape == (64, len(profile.predictors)) + pd.testing.assert_frame_equal(selected, nine.loc[:, list(profile.predictors)]) + assert not any("prior" in name for name in selected) + assert len(run.node.outputs) == 64 + assert len({ref.name for ref in run.kernel.source_refs}) == 2 + + +@pytest.mark.parametrize("seed", [True, -1, 2**64, 1.5]) +def test_seed_is_an_exact_uint64(canonical_run, seed): + with pytest.raises(ValueError, match="PUF55_CANONICAL_SOURCE_SEED"): + owner.canonical_puf55_donor_node( + seed=seed, fixture_definition=canonical_run.definition + ) + + +@pytest.mark.parametrize("kind", ["packaged", "detached_pin"]) +def test_caller_definition_cannot_issue_production_authority(canonical_run, kind): + if kind == "packaged": + definition = owner.raw.packaged_definition() + else: + original = canonical_run.definition + definition = replace( + original, main=replace(original.main, bytes=original.main.bytes + 1) + ) + with pytest.raises( + ValueError, match="PUF55_CANONICAL_SOURCE_(FIXTURE_AUTHORITY|DEFINITION_PINS)" + ): + owner.CanonicalPuf55DonorKernel(fixture_definition=definition) + + +@pytest.mark.parametrize("kind", ["params", "columns", "artifact", "sources"]) +def test_exact_declaration_and_artifact_roster(canonical_run, kind): + run = canonical_run + context = _context(run.kernel, run.paths) + if kind == "params": + context = replace(context, params={**dict(context.params), "seed": 999}) + elif kind == "columns": + context = replace( + context, + node=replace( + context.node, outputs=(Owned("tax_unit", "wrong", "float64"),) + ), + ) + elif kind == "artifact": + context = replace( + context, + node=replace( + context.node, artifact_outputs=context.node.artifact_outputs[:-1] + ), + ) + else: + context = replace(context, sources={}) + with pytest.raises( + ValueError, match="PUF55_CANONICAL_SOURCE_DECLARATION_OR_CONTEXT" + ): + run.kernel.run(context) + + +def test_wrong_source_codec_cannot_replace_pinned_loader(canonical_run, monkeypatch): + kernel = canonical_run.kernel + codec = kernel._definition.main.codec + with monkeypatch.context() as patch: + patch.setitem( + kernel.source_codecs._byte_loaders, + codec, + lambda *a, **k: canonical_run.buffers[0], + ) + with pytest.raises(ValueError, match="PUF55_CANONICAL_SOURCE_CODEC_BINDING"): + kernel.run(_context(kernel, canonical_run.paths)) + + +def test_changed_source_bytes_refuse_before_construction(tmp_path): + definition, paths, _ = _original_sources(tmp_path / "source_changed") + kernel = owner.CanonicalPuf55DonorKernel(fixture_definition=definition) + path = paths[definition.main.source_name] + payload = path.read_bytes() + path.write_bytes(payload.replace(b"40000", b"49999", 1)) + with pytest.raises(ValueError): + kernel.run(_context(kernel, paths)) + + +@pytest.mark.parametrize("defect", ["header", "duplicate"]) +def test_pinned_bytes_still_require_actual_source_structure(tmp_path, defect): + definition, paths, _ = _original_sources(tmp_path / defect, defect=defect) + kernel = owner.CanonicalPuf55DonorKernel(fixture_definition=definition) + with pytest.raises(ValueError): + kernel.run(_context(kernel, paths)) + + +def test_source_order_is_not_replaced_by_technical_identifiers(tmp_path): + definition, paths, _ = _original_sources(tmp_path / "reordered", defect="reordered") + kernel = owner.CanonicalPuf55DonorKernel(fixture_definition=definition) + result = kernel.run(_context(kernel, paths)) + np.testing.assert_array_equal( + result.frame.table("tax_unit").index, np.arange(501063, 500999, -1) + ) + np.testing.assert_array_equal( + result.frame.table("tax_unit").tax_unit_id, np.arange(1, 65) + ) + assert result.frame.weights_for("tax_unit").values[-1] == 0 + + +def test_implementation_identity_rechecks_interest_on_replay_path(canonical_run): + kernel = canonical_run.kernel + fired = False + previous = sys.getprofile() + target = owner.interest.puf_e19200_interest_components_asset_identity.__code__ + + def profile(frame, event, arg): + nonlocal fired + if ( + event == "return" + and frame.f_code is target + and frame.f_back is not None + and frame.f_back.f_code + is owner.CanonicalPuf55DonorKernel.implementation_hash.__code__ + ): + fired = True + arg["asset_sha256"] = "0" * 64 + + sys.setprofile(profile) + try: + with pytest.raises( + ValueError, match="PUF55_CANONICAL_SOURCE_IMPLEMENTATION_RESOURCES" + ): + # run_graph invokes this even when run() will be a cache hit. + kernel.implementation_hash() + finally: + sys.setprofile(previous) + assert fired + + +def _at_last_resource_return(kernel, context, mutate): + """A profiler exercises the intended last borrowed-I/O return boundary.""" + fired = False + previous = sys.getprofile() + target = owner.CanonicalPuf55DonorKernel._read_resources.__code__ + + def profile(frame, event, arg): + nonlocal fired + if event == "return" and frame.f_code is target and frame.f_back is not None: + caller = frame.f_back + if ( + caller.f_code is owner.CanonicalPuf55DonorKernel.run.__code__ + and "result" in caller.f_locals + and not fired + ): + fired = True + mutate(caller.f_locals) + + sys.setprofile(profile) + try: + with pytest.raises(ValueError, match="PUF55_CANONICAL_SOURCE_"): + kernel.run(context) + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize("kind", ["qbi", "growth", "bands"]) +def test_late_recipe_or_band_mutation_refuses(canonical_run, monkeypatch, kind): + run = canonical_run + with monkeypatch.context() as patch: + + def mutate(_): + if kind == "qbi": + patch.setattr(owner.canonical.qbi, "PARAMETERS_SHA256", "0" * 64) + elif kind == "growth": + patch.setitem( + owner.canonical.growth._RECIPE, "sensitivity_cpi_factor", "9" + ) + else: + bands = owner.interest.US_PUF_E19200_AGI_BANDS + patch.setattr( + owner.interest, + "US_PUF_E19200_AGI_BANDS", + ( + replace( + bands[0], + investment_interest_amount=bands[ + 0 + ].investment_interest_amount + + 1, + ), + *bands[1:], + ), + ) + + _at_last_resource_return(run.kernel, _context(run.kernel, run.paths), mutate) + + +@pytest.mark.parametrize( + "kind", ["frame", "bytes", "mutable_bytes", "receipt", "paired_baseline"] +) +def test_late_detached_outputs_have_immutable_seals(canonical_run, kind): + run = canonical_run + + def mutate(local): + result = local["result"] + if kind == "frame": + result.frame.table("tax_unit").columns.name = "changed" + elif kind == "bytes": + result.artifacts["canonical_donor"] = b"wrong" + elif kind == "mutable_bytes": + result.artifacts["canonical_donor"] = bytearray( + result.artifacts["canonical_donor"] + ) + elif kind == "receipt": + result.receipt["ordinary_rows"] = 1 + else: + result.artifacts["canonical_donor"] = b"wrong" + result.receipt["canonical_donor_sha256"] = owner._sha(b"wrong") + + _at_last_resource_return(run.kernel, _context(run.kernel, run.paths), mutate) + + +def test_late_source_path_substitution_refuses(canonical_run): + run = canonical_run + context = _context(run.kernel, run.paths) + + def mutate(_): + context.sources[run.kernel._definition.main.source_name] = next( + iter(run.paths.values()) + ).with_name("other.csv") + + _at_last_resource_return(run.kernel, context, mutate) + + +def test_late_function_replacement_refuses(canonical_run, monkeypatch): + run = canonical_run + with monkeypatch.context() as patch: + + def mutate(_): + patch.setattr( + owner.canonical.growth, "grow_puf_2015_to_2024", lambda *a, **k: None + ) + + _at_last_resource_return(run.kernel, _context(run.kernel, run.paths), mutate) + + +@pytest.mark.parametrize( + "mutation", + [ + "config", + "defaults", + "kwdefaults", + "code", + "closure", + "alias", + "dataclass", + "array", + ], +) +def test_live_marker_tracks_mutations_through_cyclic_class_closure( + monkeypatch, mutation +): + """A real class method closes over a mapping containing itself and aliases.""" + shared = [7] + bindings = {"config": 11, "left": shared, "right": shared} + + @dataclass + class Cyclic: + value: int + + def method(self, value=3, *, factor=5): + return bindings, value * factor + + def changed(self, value=3, *, factor=5): + return bindings, value + factor + + method = Cyclic.method + bindings.update( + method=method, + self=bindings, + record=Cyclic(13), + array=np.array([17.0], dtype=np.float64), + ) + # _live walks real class methods, including dataclass-generated methods and + # Python 3.14 class annotation closures, as it does in the production owners. + module = SimpleNamespace( + __name__=Cyclic.__module__, __file__=__file__, Cyclic=Cyclic + ) + monkeypatch.setattr(owner, "_modules", lambda: (module,)) + baseline = owner._live() + assert owner._live() == baseline + if mutation == "config": + bindings["config"] += 1 + elif mutation == "defaults": + method.__defaults__ = (4,) + elif mutation == "kwdefaults": + method.__kwdefaults__["factor"] += 1 + elif mutation == "code": + method.__code__ = changed.__code__ + elif mutation == "closure": + replacement = dict(bindings, config=19) + cell = method.__closure__[method.__code__.co_freevars.index("bindings")] + cell.cell_contents = replacement + elif mutation == "alias": + # Equal leaf values in a new container must not hide changed aliasing. + bindings["right"] = list(shared) + elif mutation == "dataclass": + bindings["record"].value += 1 + else: + bindings["array"][0] += 1 + assert owner._live() != baseline + + +def test_live_marker_keeps_first_visit_depth_limit(): + cyclic = [] + current = cyclic + for _ in range(16): + child = [] + current.append(child) + current = child + current.append(cyclic) + # The depth-17 back edge names its existing object; it does not expand it. + assert owner._marker(cyclic) == owner._marker(cyclic) + deep = 0 + for _ in range(17): + deep = [deep] + with pytest.raises(ValueError, match="PUF55_CANONICAL_SOURCE_LIVE_DEPTH"): + owner._marker(deep) + + +def test_live_marker_distinguishes_empty_closure_cells(): + state = {} + + def cyclic(): + return state + + state["function"] = cyclic + cell = cyclic.__closure__[0] + populated = owner._marker(cyclic) + del cell.cell_contents + empty = owner._marker(cyclic) + assert owner._marker(cyclic) == empty + assert empty != populated + cell.cell_contents = {} + assert owner._marker(cyclic) != empty + cell.cell_contents["function"] = cyclic + assert owner._marker(cyclic) == populated + + +def test_live_marker_stable_when_normal_calls_retain_code_literals(): + def ordinary(): + def nested(): + return "nested ordinary result; retained outside its code object" + + return "ordinary result; retained outside its code object", nested + + code = ordinary.__code__ + baseline = owner._marker(ordinary) + retained = [ordinary() for _ in range(16)] + nested_results = [nested() for _, nested in retained] + assert len(retained) == len(nested_results) == 16 + assert ordinary.__code__ is code + assert owner._marker(ordinary) == baseline + retained.clear() + nested_results.clear() + assert owner._marker(ordinary) == baseline + + +@pytest.mark.parametrize("growth_scheme", ("family_observed", "cpi_only")) +def test_live_marker_stable_after_retaining_actual_recipe(growth_scheme): + # Exercise the exact function located by the startup diagnostic, using its + # actual packaged recipe and band identity; no recipe or resource is mocked. + recipe = owner._recipe + code = recipe.__code__ + baseline = owner._marker(recipe) + retained = [recipe(578, growth_scheme) for _ in range(8)] + assert all(row == retained[0] for row in retained) + assert all(row["growth_scheme"] == growth_scheme for row in retained) + assert recipe.__code__ is code + assert owner._marker(recipe) == baseline + retained.clear() + assert owner._marker(recipe) == baseline + + +@pytest.mark.parametrize( + "mutation", + ("equal_code_replacement", "different_code", "constant", "defaults", "kwdefaults"), +) +def test_live_marker_tracks_code_replacement_and_nested_mutable_defaults(mutation): + def subject(values=None, *, options=None): + return values, options, "original constant; independent of public bytecode" + + def replacement(values=None, *, options=None): + return options, values, "different executable body" + + subject.__defaults__ = ([7],) + subject.__kwdefaults__ = {"options": {"scale": 3}} + baseline = owner._marker(subject) + code = subject.__code__ + if mutation == "equal_code_replacement": + subject.__code__ = code.replace() + assert subject.__code__ is not code + assert subject.__code__ == code + elif mutation == "different_code": + subject.__code__ = replacement.__code__ + elif mutation == "constant": + subject.__code__ = code.replace( + co_consts=tuple( + "changed literal; bytecode remains identical" + if type(value) is str + else value + for value in code.co_consts + ) + ) + assert subject.__code__.co_code == code.co_code + assert subject.__code__.co_consts != code.co_consts + elif mutation == "defaults": + subject.__defaults__[0].append(11) + else: + subject.__kwdefaults__["options"]["scale"] += 1 + assert owner._marker(subject) != baseline diff --git a/packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py b/packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py index e9ba6640d..d36d2e589 100644 --- a/packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py +++ b/packages/microcosm-build/tests/test_us_puf55_route_numerical_finalization.py @@ -234,6 +234,10 @@ def observe(call, event, arg): ) _assert_preserved(case.frame, candidate) receipt = full.codec.decode_json(payload) + assert receipt["protocol"] == "microcosm.us.puf55-route-finalization.v2" + assert receipt["candidate_frame_sha256"] == bridge._candidate_frame_sha256( + candidate + ) assert receipt["recipient_rows"] == 4 and receipt["finalizer_calls"] == 1 assert receipt["model_consumed_donor_values_checked"] is True assert [row["profile"] for row in receipt["routes"]] == [ @@ -266,6 +270,10 @@ def test_one_nonempty_route_uses_same_whole_cohort_finalizer( route = _apply(case.fitted[route_index], case.ids[::-1]) candidate, payload = _call(frame, (route,)) receipt = full.codec.decode_json(payload) + assert receipt["protocol"] == "microcosm.us.puf55-route-finalization.v2" + assert receipt["candidate_frame_sha256"] == bridge._candidate_frame_sha256( + candidate + ) assert receipt["recipient_rows"] == 4 and len(receipt["routes"]) == 1 assert receipt["routes"][0]["profile"] == bridge.PROFILES[route_index].value raw, _ = bridge.merge_puf55_route_draws( @@ -562,3 +570,52 @@ def observe(call, event, arg): finally: sys.setprofile(old) assert fired + + +def test_receipt_encoding_cannot_change_the_already_sealed_candidate(numerical_routes): + case = numerical_routes + fired, calls = [], [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is full.support.finalize_us_puf_tax_detail_predictions.__code__ + ): + calls.append(1) + caller = frame.f_back + if ( + event == "return" + and frame.f_code is full.codec.encode_json.__code__ + and caller is not None + and caller.f_code is bridge._finalize_puf55_routes.__code__ + and "candidate_sha256" in caller.f_locals + and not fired + ): + candidate = caller.f_locals["candidate"] + table = candidate.person + mask = full.support.puf_tax_detail_clone_mask(table, entity="person") + row = np.flatnonzero(mask)[0] + target = "employment_income_before_lsr" + assert target in bridge.PROFILES[0].person_outputs + assert table[target].dtype == np.dtype("float64") + column = table.columns.get_loc(target) + before = table.iloc[row, column] + assert np.isfinite(before) + table.iloc[row, column] = before + 123.0 + assert ( + np.isfinite(table.iloc[row, column]) + and table.iloc[row, column] != before + ) + fired.append(1) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + ValueError, match="^PUF55_ROUTE_FINALIZATION_CANDIDATE_CHANGED$" + ): + _call(case.frame, case.routes) + finally: + sys.setprofile(previous) + assert fired == calls == [1] diff --git a/packages/microcosm-build/tests/test_us_puf59_canonical.py b/packages/microcosm-build/tests/test_us_puf59_canonical.py index 652c39c4a..261e8b0dc 100644 --- a/packages/microcosm-build/tests/test_us_puf59_canonical.py +++ b/packages/microcosm-build/tests/test_us_puf59_canonical.py @@ -416,6 +416,43 @@ def build(source, **kwargs): "puf55_source_bytes_unchanged", artifact.encode_canonical_puf59(candidate) == packed, ) + explicit, explicit_metadata = donor55.canonical_puf55_donor_from_artifact( + packed, + expected_artifact_sha256=digest, + profile=enrichment.PUF55_SURVEY_SS, + ) + check( + "puf55_default_nine_profile_unchanged", + explicit.equals(projected) and explicit_metadata == metadata, + ) + eight, eight_metadata = donor55.canonical_puf55_donor_from_artifact( + packed, + expected_artifact_sha256=digest, + profile=enrichment.PUF55_SURVEY_SS_NO_TOTAL, + ) + check( + "puf55_eight_exact_projection_without_source_or_target_change", + eight.equals(projected.drop(columns=[enrichment.SURVEY_SS_TOTAL_PREDICTOR])) + and tuple(eight.columns) + == ( + *enrichment.PUF55_SURVEY_SS_NO_TOTAL.predictors, + *profile.targets, + "weight", + "puf_person_incidence_capacity", + ) + and artifact.encode_canonical_puf59(candidate) == packed, + ) + check( + "puf55_eight_omission_is_explicit_and_grants_no_authority", + eight_metadata["profile"] == enrichment.PUF55_SURVEY_SS_NO_TOTAL.value + and eight_metadata["ordered_columns"] == list(eight.columns) + and eight_metadata["social_security_total"]["predictor"] is None + and eight_metadata["social_security_total"]["raw_field"] == "E02400" + and eight_metadata["artifact_sha256"] == digest + and eight_metadata["source_receipt_sha256"] == candidate.receipt["sha256"] + and eight_metadata["source_authority_granted"] is False + and eight_metadata["release_eligible"] is False, + ) refuses( "puf55_expected_artifact_identity_required", lambda packed=packed: donor55.canonical_puf55_donor_from_artifact( @@ -424,6 +461,24 @@ def build(source, **kwargs): "PUF55_DONOR_ARTIFACT_IDENTITY", ) + for invalid_profile in ( + enrichment.PUF59, + enrichment.FULL65, + enrichment.PUF55_SURVEY_SS_NO_TOTAL.value, + None, + ): + refuses( + "puf55_explicit_closed_profile_required", + lambda invalid_profile=invalid_profile: ( + donor55.canonical_puf55_donor_from_artifact( + packed, + expected_artifact_sha256=digest, + profile=invalid_profile, + ) + ), + "PUF55_DONOR_PROFILE", + ) + # Correctly rehashed, internally consistent artifacts still cannot change # the upstream SS carrier semantics accepted by this projection. def repacked_ss_change(*, column=None, method=None): @@ -456,6 +511,16 @@ def repacked_ss_change(*, column=None, method=None): "PUF55_DONOR_SS_CARRIER", ) + refuses( + "puf55_eight_cannot_bypass_carrier_contract", + lambda altered=altered: donor55.canonical_puf55_donor_from_artifact( + altered, + expected_artifact_sha256=hashlib.sha256(altered).hexdigest(), + profile=enrichment.PUF55_SURVEY_SS_NO_TOTAL, + ), + "PUF55_DONOR_SS_CARRIER", + ) + # Integrity successor: domain-valid mutations cannot inherit old provenance. import struct From 9d20d6e8cc643cdf318b025ac75afe0d31578e8e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 10:42:50 -0400 Subject: [PATCH 062/255] Clarify post-clone geography and parallel release review --- docs/geography-assignment.md | 26 +++++++++++++------- docs/us-launch-review.md | 46 ++++++++++++++++++++++++++++++------ docs/us-uk-release-path.md | 27 +++++++++++++++------ 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index e7a3cec67..cca500c02 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -1,10 +1,16 @@ # Geography assignment in the population graph -Design decision, 9 September 2026: assign one small census area to each household -after constructing and harmonizing the survey spine. Derive larger geographies -from that area using identified, versioned mappings. PUF and other enrichment -clones inherit the location. Geographic support expansion, if needed, is a -separate declared population operation with explicit identities and weights. +Design decision, corrected 10 September 2026: finish constructing the full survey +multispine, including its initial support and PUF clones, then assign one small +census area to each resulting household. Derive larger geographies from that +area using identified, versioned mappings. Distinct clones may receive different +areas subject to their observed source constraints. The assigned location then +remains fixed through subsequent enrichment, calibration and analysis views. + +The earlier block-before-clone implementation and its passing controls remain +historical evidence. They do not demonstrate this corrected ordering. Geographic +assignment must use a stable post-clone identity, including a clone discriminator +where original-source identity alone is shared by multiple households. This document records the intended contract and the implementation gap. It does not certify a new geographic source, assignment run or population file. @@ -37,12 +43,12 @@ paths while the shared operators and country adapters receive their own checks. ```mermaid flowchart LR - S[Combined survey spine] --> A[Assign one small area per household] + S[Complete survey multispine including clones] --> A[Assign one small area per resulting household] C[Observed survey geography constraints] --> A L[Versioned small-area support] --> A A --> D[Derive larger geographies] M[Versioned geographic mappings] --> D - D --> E[Enrichment clones inherit location] + D --> E[Subsequent enrichment retains location] E --> R[Rules and calibration] R --> P[Pruned analysis file retains location] ``` @@ -87,8 +93,10 @@ the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/gui - Bind each draw to a stable household identity, seed and assignment definition. Reordering rows or selecting an existing household must not redraw its location. Changed source support or boundary mappings produce a new identified revision. -- Assign household members and enrichment clones consistently. Materialized - county, district and other fields must agree with the selected anchor's mapping. +- Assign household members consistently with their resulting household. Initial + clones have distinct draw identities and may receive different anchors. + Materialized county, district and other fields must agree with that household's + selected anchor. Source geography remains a separate observed constraint. - Show assignment and derivation as separate operations in the graph. Expose the source constraints, support, sampling weights, mapping conventions and judgment annotations in the inspector. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index e5ec261fe..13492fd64 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -7,15 +7,16 @@ release verification have not passed together. ## Architecture and review order -The intended construction order is ACS + CPS ASEC → combined survey multispine → -household Census block assignment → derived larger geographies → PUF clone and -donor enrichment → rules evaluation and calibration → verified full and pruned -exports. Enrichment clones inherit their household's assigned location. The PUF +The intended construction order is ACS + CPS ASEC → complete survey multispine +including its initial support/PUF clones → household Census block assignment → +derived larger geographies → donor enrichment → rules evaluation and calibration +→ verified full and pruned exports. Each resulting household retains its assigned +location through the later stages. The PUF operator acts on the combined survey frame. A separate ASEC–PUF base is not the intended architecture. National and local analysis share one fully constructed, enriched Frame. The -analysis branch begins after harmonization, atomic geography assignment, cloning +analysis branch begins after harmonization, cloning, atomic geography assignment and imputation. Calibration changes weights; L0 controls sparsity. An export can select a geographic scope and remove zero-weight households, but must preserve every retained record's input values, knownness, entity relationships and assigned @@ -37,7 +38,7 @@ to zero. A pruned export must not be required to make every survey origin independently resemble a national population, or to rerun imputation to satisfy such a requirement. -The optional ten-node survey prefix now assigns a block before cloning, and its +The earlier ten-node survey prefix assigns a block before cloning, and its thirteen-node age-calibration extension passes cold execution and required replay on invented originals. A twenty-node extension now carries atomic geography through current financial imputation and required replay. Native @@ -47,6 +48,11 @@ operator is a separate path. See [Geography assignment](geography-assignment.md) for the country contracts, implementation and source boundaries. +The 10 September sequencing correction requires the full initial multispine and +its clones before block assignment, allowing each resulting household its own +constrained draw. The accepted ten/twenty-node runs retain their original order; +the corrected order needs its own implementation and replay checks. + | Area | Main source entry points | What to review | | --- | --- | --- | | Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | @@ -54,7 +60,7 @@ implementation and source boundaries. | Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | | Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | | Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | -| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Block-first assignment, observed-source constraints, versioned mappings and clone inheritance; the older joint-cell operator remains separate | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment, observed-source constraints, versioned mappings and stable clone identities; moving assignment after cloning remains in progress | | Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | | Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | @@ -66,6 +72,32 @@ before deciding how to land it. Shared runtime changes overlap with [#885](https://github.com/PolicyEngine/microcosm/pull/885). Their final contracts need reconciliation; this draft does not supersede either review. +## Parallel review and dependencies + +Completion of this integration draft is not a prerequisite for all UK work. +Depend on identified shared changes where necessary, with independently reviewed +PRs and compatible interfaces. Keep the US native build, UK candidate repair, +source/target reconciliation and shared runtime work moving in parallel. + +| Reviewer | Review focus in this draft | Work that can continue independently | +| --- | --- | --- | +| Anthony | Shared graph/Frame contracts, artifact and replay behavior, calibration changes, and overlap with #885/#873; agree the implementation to retain and the smallest shared prerequisites to land | Runtime standardization and UK staging telemetry; a complete US PUF build is not their prerequisite | +| Maria | UK-facing shared calibration behavior, source/input contracts and proposed atomic-geography adapter; identify compatibility requirements for the current UK candidate | #900, Chronicle/lookup reconciliation, UK support/fit diagnosis and candidate validation; these do not depend on native US PUF integration | +| US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | + +Review the shared contracts first and record each dependency as a named change +with a pinned revision. Country work can remain based on current main; do not +rebase it onto this entire moving integration branch merely to participate in +review. Extract an agreed shared prerequisite into a smaller PR when a consumer +actually needs it, with compatibility checks for both countries. + +The shared post-clone geography interface is under revision following the +10 September ordering correction. Consumers adopting that new path should agree +on its interface before wiring it. Existing target/source repairs can proceed on +their reviewed base. Passing this draft's invented controls does not certify +Maria's UK candidate, and the final US release need not gate her independent +source and calibration work. No whole-PR merge is implied by this review map. + ## Verification and review scope The first native twenty-node survey, block-geography and financial cold pilot diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 8262cf434..77f8ce674 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -12,10 +12,12 @@ Each country should have one fully constructed population from which full and smaller analysis files are derived. Record the survey, monetary, policy and boundary reference periods explicitly; do not infer them from a filename. -After harmonization, assign one atomic area to each household and derive larger -geographies from versioned mappings. Geographic support expansion, when needed, -precedes assignment and has explicit identities and weights. Subsequent -enrichment clones inherit the assigned location. Once enrichment is complete, +After constructing and harmonizing the complete multispine, including its +initial clones, assign one atomic area to each resulting household and derive +larger geographies from versioned mappings. Distinct clones may receive different +areas, constrained by observed source geography and keyed by stable clone +identity. Subsequent enrichment retains the assigned location. Once construction +and enrichment are complete, calibration changes household weights; scope selection and pruning retain the original values, missingness and relationships for every retained household. @@ -30,7 +32,7 @@ common source population alone does not guarantee identical estimates. | --- | --- | --- | --- | | Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | | Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | -| Geographic support | Block assignment before enrichment cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment and derivation for each nation; adequate constituency/LA support | Observed geography respected; stable household draws; clone/export inheritance; support and fit measured at advertised levels | +| Geographic support | Block assignment after initial multispine cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment after full spine cloning, and derivation for each nation; adequate constituency/LA support | Observed geography respected; distinct stable clone draws; subsequent location invariance; support and fit measured at advertised levels | | Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | | Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | | Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | @@ -38,6 +40,15 @@ common source population alone does not guarantee identical estimates. ## Current position +| Stage | US demonstrated result | UK demonstrated result | +| --- | --- | --- | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; complete PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Geography | Earlier pre-clone block assignment and inheritance pass in the small real build; corrected post-clone order and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared stable post-clone owner not integrated | +| Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | +| Full/compact files | Small prefix export and required replay pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | + The US new graph has passed a small real survey → block assignment → clone → financial-enrichment build and required replay. The combined survey population has 1,584 source households and 3,168 records after cloning. This is not the @@ -55,8 +66,10 @@ does not satisfy today's release battery. See [the current dense release runbook](uk-dense-release-assembly-runbook-762.md). The current UK route clones for geographic support and then uses the existing -OA ladder. It has not adopted the new shared, household-keyed atomic geography -graph immediately after harmonization. Maria's +OA ladder. It has not adopted the new shared atomic geography graph keyed to +stable post-clone household identity. Its existing placement after cloning is +consistent with the user's 10 September ordering correction; the source, +identity and shared-operator migration remain separate work. Maria's [PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle household targets and the Northern Ireland lookup; that source work does not by itself migrate the assignment owner. Anthony's From 752158197a5df898413bc32dfc104ba0f66733cb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 10:47:39 -0400 Subject: [PATCH 063/255] Record agreed UK graph and dashboard sequencing --- docs/us-launch-review.md | 33 ++++++++++++++++++++++----------- docs/us-uk-release-path.md | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 13492fd64..a522fb1d6 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -67,10 +67,12 @@ the corrected order needs its own implementation and replay checks. Paths beginning `us_runtime/` are relative to `packages/microcosm-build/src/microcosm/build/`. The branch contains a substantial consolidation; review each area and its tests -before deciding how to land it. Shared runtime changes overlap with -[#873](https://github.com/PolicyEngine/microcosm/pull/873) and Anthony's -[#885](https://github.com/PolicyEngine/microcosm/pull/885). Their final contracts -need reconciliation; this draft does not supersede either review. +before deciding how to land it. Shared model work overlaps with +[#873](https://github.com/PolicyEngine/microcosm/pull/873). Anthony considers his +runtime proposal [#885](https://github.com/PolicyEngine/microcosm/pull/885) +superseded by this direction; it is reference material, not a prerequisite to +retain or merge separately. This does not imply that its GitHub PR is closed or +that this draft has received his approval. ## Parallel review and dependencies @@ -81,15 +83,24 @@ source/target reconciliation and shared runtime work moving in parallel. | Reviewer | Review focus in this draft | Work that can continue independently | | --- | --- | --- | -| Anthony | Shared graph/Frame contracts, artifact and replay behavior, calibration changes, and overlap with #885/#873; agree the implementation to retain and the smallest shared prerequisites to land | Runtime standardization and UK staging telemetry; a complete US PUF build is not their prerequisite | -| Maria | UK-facing shared calibration behavior, source/input contracts and proposed atomic-geography adapter; identify compatibility requirements for the current UK candidate | #900, Chronicle/lookup reconciliation, UK support/fit diagnosis and candidate validation; these do not depend on native US PUF integration | +| Anthony | Shared graph/Frame and calibration compatibility; target hierarchy and metadata flow into diagnostics/dashboard, including #855 | Target/dashboard changes follow #900; UK staging #896 proceeds independently of full UK graph conversion and native US PUF integration | +| Maria | Register each existing UK pipeline stage as a graph node using this branch's shared machinery; verify UK behavior and coverage | #900 lands before graph conversion; UK imputation, support/fit diagnosis and candidate validation continue; graph-node registration stacks on #893, with stage reordering in a later separate PR | | US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | -Review the shared contracts first and record each dependency as a named change -with a pinned revision. Country work can remain based on current main; do not -rebase it onto this entire moving integration branch merely to participate in -review. Extract an agreed shared prerequisite into a smaller PR when a consumer -actually needs it, with compatibility checks for both countries. +Coordination updated 10 September: Maria is deliberately stacking her UK +graph-node registration on #893. That specific dependent work should pin the +parent revision and coordinate shared-interface changes here. Her source/target +repairs and Anthony's staging work do not have to wait for the whole draft. +Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) +and its dashboard consumer follow the target reconciliation in +[#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph +nodes and changing their order are separate review steps. The US integration +work must not build a competing full UK graph; its shared atomic-area adapter +is an input to Maria's owned integration. + +Record shared dependencies as named changes with pinned revisions. Extract a +smaller shared prerequisite where useful, while preserving compatibility with +Maria's intentional stack. Proposed sequencing does not certify or merge any PR. The shared post-clone geography interface is under revision following the 10 September ordering correction. Consumers adopting that new path should agree diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 77f8ce674..e714b9d4e 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -88,6 +88,28 @@ for candidate-specific verification, not proof every historical defect remains. ## Team meeting decisions +This document is the shared US/UK release-path reference. The country review +guides and run receipts provide supporting detail; update the sequence and +ownership here when they change. + +The 10 September coordination sequence is: + +- Maria's #900 source/target reconciliation goes first; Anthony's #855 target + hierarchy/diagnostics and corresponding dashboard consumer follow it. +- Maria owns registering every existing UK stage as a graph node, stacking on + #893. Changes to UK stage order follow in a separate PR. The shared atomic-area + adapter work supports that integration and does not create a second UK graph. +- Anthony's UK staging #896 can proceed independently. His earlier runtime + proposal #885 is superseded in intent, not another required implementation. +- US construction and verification continue alongside these changes. Complete + graph coverage in both countries is required for the broader advertised + launch; passing tests for individual stages does not establish that coverage. + +The latest UK full rebuild and evaluation were reported as underway at the +meeting. Its new report must replace historical quality evidence only after the +actual candidate and results are available; the approximate four-hour runtime +reported for that build is not a US or release ETA. + 1. Confirm the release years, advertised geographies and full/compact products. 2. Assign one accountable owner to each country candidate and one owner to each shared dependency: runtime, calibration, diagnostics and release consumers. From f4e5ec4eb761e26f65667f693ef37d3ea68adfc8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 11:12:02 -0400 Subject: [PATCH 064/255] Record merged UK target dependency in release plan --- docs/us-launch-review.md | 3 +++ docs/us-uk-release-path.md | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index a522fb1d6..de1f7a64b 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -91,6 +91,9 @@ Coordination updated 10 September: Maria is deliberately stacking her UK graph-node registration on #893. That specific dependent work should pin the parent revision and coordinate shared-interface changes here. Her source/target repairs and Anthony's staging work do not have to wait for the whole draft. +PR #900 merged at 14:54 UTC and is incorporated here from main at +`6f7571e1ab8c516154289bd8ebf75cb2646f03b5`. The combination preserves both parent +histories and does not rerun or recertify the frozen US controls. Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) and its dashboard consumer follow the target reconciliation in [#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index e714b9d4e..40cf8aeb9 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -94,8 +94,9 @@ ownership here when they change. The 10 September coordination sequence is: -- Maria's #900 source/target reconciliation goes first; Anthony's #855 target - hierarchy/diagnostics and corresponding dashboard consumer follow it. +- Maria's #900 source/target reconciliation merged on 10 September at 14:54 UTC; + Anthony's #855 target hierarchy/diagnostics and corresponding dashboard + consumer follow it. Its merge clears that dependency, not all release gates. - Maria owns registering every existing UK stage as a graph node, stacking on #893. Changes to UK stage order follow in a separate PR. The shared atomic-area adapter work supports that integration and does not create a second UK graph. From b1626e11ff52d0ee6bd7f2dc4084922b35fd6860 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 13:10:01 -0400 Subject: [PATCH 065/255] Add shared UK atomic-area adapter and lazy runtime exports --- ...3-uk-atomic-area-and-lazy-exports.added.md | 1 + docs/us-uk-release-path.md | 13 +- .../uk-atomic-area-lazy52-20260910.json | 37 + .../microcosm/build/uk_runtime/__init__.py | 2898 +++++++++++++---- .../build/uk_runtime/atomic_area_support.py | 388 +++ .../uk_runtime/atomic_household_identity.py | 85 + .../tests/test_uk_atomic_area_support.py | 482 +++ .../tests/test_uk_runtime_lazy_exports.py | 138 + 8 files changed, 3493 insertions(+), 549 deletions(-) create mode 100644 changelog.d/893-uk-atomic-area-and-lazy-exports.added.md create mode 100644 experiments/uk-atomic-area-lazy52-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py create mode 100644 packages/microcosm-build/tests/test_uk_atomic_area_support.py create mode 100644 packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py diff --git a/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md new file mode 100644 index 000000000..9614ce4df --- /dev/null +++ b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md @@ -0,0 +1 @@ +Add supplied-source UK atomic-area assignment keyed to stable post-clone household identity, and load UK runtime exports only when requested. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 40cf8aeb9..6420b4786 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -43,7 +43,7 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | | Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; complete PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | -| Geography | Earlier pre-clone block assignment and inheritance pass in the small real build; corrected post-clone order and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared stable post-clone owner not integrated | +| Geography | Earlier pre-clone block assignment and inheritance pass in the small real build; corrected post-clone order and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix export and required replay pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | @@ -76,6 +76,17 @@ by itself migrate the assignment owner. Anthony's [PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging telemetry and synthetic smoke evidence, not a calibrated release. +The shared UK adapter now has accepted supplied-array tests for England/Wales +Output Areas, Scottish Output Areas and Northern Ireland Data Zones. These +checks cover source conventions, observed-region constraints, stable post-clone +identity and row-order/subset invariance. Ordinary UK imports also preserve the +public export contract without eagerly loading unrelated stages or country +engines. All 52 checks passed with unchanged source bytes and no unexpected +access attempts; see the [scoped acceptance record](../experiments/uk-atomic-area-lazy52-20260910.json). +No native UK area source, whole-spine build or calibration result is certified +by those checks. Maria can use these shared primitives in the existing UK graph +integration rather than starting a second country runtime. + UK release coverage must be re-established on the chosen integrated build and locked model. The declared 145-input contract's older candidate evidence must not certify the current stage set. The diff --git a/experiments/uk-atomic-area-lazy52-20260910.json b/experiments/uk-atomic-area-lazy52-20260910.json new file mode 100644 index 000000000..076f3a32f --- /dev/null +++ b/experiments/uk-atomic-area-lazy52-20260910.json @@ -0,0 +1,37 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "country_distribution_paths_dereferenced_by_postchecker": false, + "cpu_seconds": 2.335378, + "guard_policy_passed": true, + "model_resources": 0, + "model_source_files": 0, + "native_payloads_read": 0, + "physical_postcheck": "pass", + "physical_postcheck_sha256": "dc9ca4ad0c4cd77f40a4a0affb12487975d88a6e9d0e7ff2a8e41011e4430071", + "receipt_sha256": "b6abf03c647e79415f593a7f8a2cfc933b6ac1a47cf92ba552f2e0cf38d3a838", + "release_acceptance": false, + "resources": 0, + "reviewer": "root", + "scope": "Invented three-system UK atomic-area adapter and ordinary lazy-package behavior; no native UK population or calibration acceptance", + "source_count": 64, + "source_files": { + "packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py": "accb5208bfc22f0d2905dad12ad390b416f424506e61c417d169215fe4bad78d", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py": "b2006451f4afcbc2c5192eb421dd7c94d11262889c0a10aa2d1040f1fa28ec8e", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py": "8d2164ae90eb521ee838936c6ca17e1ce08f43650889ea14afc9bcb82301a325", + "packages/microcosm-build/src/microcosm/build/uk_runtime/content_identity.py": "589002ba67e1c669f40b13968f16ee439937bfbd7e36d6ad4ac9ee195af2b3e5", + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py": "5b7287cd228fa7f5385cc19e6ce73184749b3c8b655953440d7273f86f3eabb3", + "packages/microcosm-build/tests/test_uk_atomic_area_support.py": "2109e447bf9446b6c4047bc742f0d9c1ed93f89b926fb9be335ae9567292759a", + "packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py": "207f5ba4b21737902e42ed8f72f3251425f1aa68959309d3f2ff0345f0f4296b" + }, + "source_map_sha256": "e7c59f91af92643378cefffb2a59be700d1940d3b3dc5aadfb4a4bf0df8e9eff", + "source_plus_owned": 73, + "suite_accepted": true, + "supplied_geography_conventions": [ + "EW OA 2021", + "Scotland OA 2022", + "Northern Ireland Data Zone 2021" + ], + "tests_passed": 52, + "wall_seconds": 3.199442916957196, + "xml_sha256": "34a5afffba8a3f7cc8933e306da19e76c8b597c9d31cffbadfeda902a1edbf8d" +} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 187310e40..05a835787 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -1,552 +1,564 @@ -"""UK build helpers for Microcosm-owned local-geography artifacts.""" +"""UK build helpers, resolved from their defining modules on explicit access. -from microcosm.build.uk_runtime.age_tail import ( - UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, - UK_AGE_TAIL_BANDS, - UK_AGE_TAIL_DECLARED_SEEDS, - UK_AGE_TOP_CODE, - UKAgeTailStageTransform, - disaggregate_uk_age_top_code, - load_uk_age_tail_band_populations, -) -from microcosm.build.uk_runtime.battery_bindings import ( - UK_GATE_REGISTRY, - UKGateBinding, -) -from microcosm.build.uk_runtime.calibration_run import ( - load_bound_spine_sidecar, - runtime_provenance, - spine_provenance_from_sidecar, -) -from microcosm.build.uk_runtime.cgt_calibration import ( - UK_CGT_ANNUAL_EXEMPT_AMOUNTS, - UK_CGT_GAINS_AMOUNT_COLUMN, - UK_CGT_SOURCE_COLUMN, - UK_CGT_TAXPAYER_COUNT_COLUMN, - UKCGTTargetMaterialization, - materialize_uk_cgt_calibration_frame, - uk_cgt_annual_exempt_amount, -) -from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, -) -from microcosm.build.uk_runtime.diagnostics import ( - UK_DIAGNOSTICS_SCHEMA_VERSION, - UK_TARGET_GEOGRAPHY_LEVELS, - uk_calibration_diagnostics_payload, - uk_fit_by_family, - uk_support_limited_misses, - uk_weakest_areas_by_fit, - uk_weakest_families, - uk_weight_summary, - uk_zero_weight_strata, - write_uk_calibration_diagnostics, -) -from microcosm.build.uk_runtime.firm_generation import ( - EMPLOYMENT_BANDS, - HMRC_BAND_COLUMNS, - INPUT_FILES, - VAT_LIABILITY_BANDS, - VINTAGES, - AxiomVATRuleEvaluator, - UKFirmCalibrationResult, - UKFirmGenerationConfig, - UKFirmGenerationResult, - UKFirmSourceData, - UKFirmTargetLayout, - UKFirmValidationReport, - UKFirmVATRuleEvaluator, - assign_employment, - assign_vat_flags, - build_firm_target_matrix, - employment_band_name, - generate_base_firms, - generate_input_values, - generate_uk_firm_population, - hmrc_band_name, - map_to_hmrc_band_indices, - optimize_firm_weights, - read_uk_firm_source_data, - solve_firm_weights, - target_diagnostics, - uk_firm_source_data_from_frames, - uk_firm_source_data_from_ledger_facts, - validate_uk_firm_population, - write_uk_firm_population, -) -from microcosm.build.uk_runtime.fiscal_targets import ( - UK_CGT_REQUIRED_COLUMNS, - UK_CGT_TARGET_COVERAGE_REQUIREMENTS, - UK_CGT_TARGET_SPECS, - UK_FISCAL_TARGET_REGISTRY, -) -from microcosm.build.uk_runtime.frs_council_tax import ( - FRS_COUNCIL_TAX_OUTPUT_COLUMNS, - UKFRSCouncilTaxStageTransform, - add_frs_council_tax, - derive_council_tax, -) -from microcosm.build.uk_runtime.frs_disability import ( - FRS_DISABILITY_OUTPUT_COLUMNS, - UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, - UKDWPDisabilityCategoryRates, - UKDWPDisabilityFlagRates, - UKFRSDisabilityStageTransform, - add_frs_disability, - derive_frs_disability, - uk_dwp_disability_category_rates, - uk_dwp_disability_flag_rates, -) -from microcosm.build.uk_runtime.frs_education import ( - EDUCQUAL_MAP, - FRS_EDUCATION_OUTPUT_COLUMNS, - UKFRSEducationStageTransform, - add_frs_education, - derive_current_education, - derive_frs_education, -) -from microcosm.build.uk_runtime.frs_education_grants import ( - FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, - FRS_EDUCATION_GRANT_REWRITES, - UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, - UKDSAPolicy, - UKFRSEducationGrantSplitStageTransform, - add_frs_education_grant_split, - allocate_reported_education_grants, - disabled_students_allowance_capacity, - uk_dsa_policy, -) -from microcosm.build.uk_runtime.frs_employment import ( - FRS_EMPLOYMENT_OUTPUT_COLUMNS, - UKFRSEmploymentStageTransform, - add_frs_employment, - derive_frs_employment, -) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - UKFRSHMRCRetainedLeavesResult, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, -) -from microcosm.build.uk_runtime.frs_legacy_proxies import ( - FRS_LEGACY_PROXY_OUTPUT_COLUMNS, - UK_LEGACY_PROXY_PREDICTORS, - UKFRSLegacyProxiesStageTransform, - UKLegacyJSAPolicy, - derive_frs_legacy_proxies, - uk_legacy_jsa_policy, -) -from microcosm.build.uk_runtime.frs_release import ( - UK_YEAR_RULES, - UKFRSRelease, - load_uk_frs_release, - resolve_uk_year_rule, -) -from microcosm.build.uk_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - UK_ENGLAND_WALES_REGION_CODES, - UK_GEOGRAPHY_LADDER_COLUMNS, - UK_LONDON_REGION_CODE, - UK_OA_LADDER_DERIVED_LAYERS, - UK_OA_LADDER_KIND, - UK_OA_LADDER_SCHEMA_VERSION, - UkOaLadder, - assign_uk_geography_ladder, - expected_uk_ladder_area_support, - load_uk_oa_ladder, - uk_geography_ladder_assignment_summary, - uk_geography_ladder_gate, - uk_region_mix, -) -from microcosm.build.uk_runtime.geography_sources import ( - ENGLAND_LAD_REGION_URL, - ENGLAND_WALES_OA2021_COUNT, - EW_OA_CONSTITUENCY_URL, - EW_OA_HIERARCHY_URL, - EW_OA_HOUSEHOLDS_URL, - EW_OA_LAD23_URL, - EW_OA_POPULATION_URL, - EW_OA_WARD_URL, - LAD23_ITL_URL, - NI_DZ2021_COUNT, - NI_DZ_GEOJSON_ZIP_URL, - NI_DZ_HOUSEHOLDS_CSV_URL, - NI_DZ_LOOKUP_SHEET, - NI_DZ_PARLCON24_LOOKUP_XLSX_URL, - NI_DZ_POPULATION_CSV_URL, - NI_PARLCON24_COUNT, - SCOTLAND_CENSUS_INDEX_ZIP_URL, - SCOTLAND_OA2022_COUNT, - SCOTLAND_OA_CONSTITUENCY_URL, - SCOTLAND_OA_DZ_IZ_URL, - SCOTLAND_OA_LAU_ITL_URL, - SCOTLAND_OA_POPULATION_URL, - build_complete_uk_geography_crosswalk, - build_england_wales_crosswalk, - build_great_britain_crosswalk, - build_northern_ireland_crosswalk, - build_official_uk_geography_crosswalk, - build_scotland_crosswalk, - load_england_lad_region_lookup, - load_england_wales_oa_constituencies, - load_england_wales_oa_hierarchy, - load_england_wales_oa_households, - load_england_wales_oa_population, - load_england_wales_oa_ward_lookup, - load_ew_oa_lad23_lookup, - load_lad_itl_lookup, - load_ni_dz_hierarchy, - load_ni_dz_households, - load_ni_dz_parlcon24_lookup, - load_ni_dz_population, - load_ni_dz_ward_lookup, - load_scotland_oa_constituencies, - load_scotland_oa_dz_iz_lookup, - load_scotland_oa_households, - load_scotland_oa_lau_lookup, - load_scotland_oa_population, - load_scotland_oa_ward_lookup, - update_england_wales_lad_codes, - write_geography_crosswalk, -) -from microcosm.build.uk_runtime.hmrc_calibration import ( - DEFAULT_HMRC_CALIBRATION_EPOCHS, - DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, - DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, - DEFAULT_HMRC_MAX_WEIGHT_RATIO, - HMRC_ASSESSABLE_INCOME_COLUMN, - HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, - HMRC_TAXPAYER_COLUMN, - UKHMRCIncomeCalibration, - UKHMRCTargetMaterialization, - calibrate_uk_hmrc_income, - materialize_uk_hmrc_calibration_frame, -) -from microcosm.build.uk_runtime.hmrc_income import ( - HMRC_SPI_BUILD_PERIOD, - HMRC_SPI_COLLATED_ODS_URL, - HMRC_SPI_INCOME_COMPONENTS, - HMRC_SPI_PUBLICATION_URL, - HMRC_SPI_SOURCE_VINTAGE, - HMRC_SPI_TARGET_RECORD_COUNT, - HMRCIncomeBandTargetRecord, - HMRCIncomeSourceProvenance, - HMRCIncomeTargetSet, - materialize_hmrc_spi_income_band_targets, - verify_hmrc_spi_collated_ods, -) -from microcosm.build.uk_runtime.hmrc_replay import ( - CANONICAL_HMRC_FACT_FENCES, - FULL_FRS_TI_BAND_FENCE_ID, - HMRCFactFence, - HMRCReplayDiagnosticAggregate, - HMRCReplayFact, - HMRCReplayReport, - build_conservative_hmrc_replay_report, - classify_hmrc_replay_targets, - write_hmrc_replay_report, -) -from microcosm.build.uk_runtime.hmrc_source_contract import ( - HMRC_DISTRIBUTIONAL_INPUTS, - UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, - assert_uk_hmrc_income_source_contract_current, -) -from microcosm.build.uk_runtime.ladder_targets import ( - constituency_household_targets, - ladder_target_provenance, - ladder_vs_chronicle_household_dispersion, - local_authority_household_targets, -) -from microcosm.build.uk_runtime.ledger_targets import ( - UK_CENSUS_HOUSEHOLDS_TARGET_ID, - UK_CROSS_GRAIN_BRIDGES, - UK_CROSS_GRAIN_GRAIN_PRECEDENCE, - UK_CROSS_GRAIN_RULE, - UKFrameTargetAdapter, - UKLedgerTargetCompilation, - apply_uk_cross_grain_reconciliation, - compile_uk_local_target_registry, - compile_uk_target_registry, - load_uk_local_area_crosswalk, - materialize_uk_ledger_targets, - uk_census_household_uprating, - uk_ledger_households_total, - uk_local_target_surface, -) -from microcosm.build.uk_runtime.local_doctrine import ( - UK_LOCAL_CLONE_COUNT, - UK_LOCAL_MAX_WEIGHT_RATIO, - UK_LOCAL_SOLVE_DOCTRINE, - UK_LOCAL_SOLVE_EPOCHS, - UK_LOCAL_TARGET_LOSS_CAP, - UK_LOCAL_TARGET_WEIGHT_RULE, - UKLocalSolveDoctrine, - uk_local_doctrine_with_overrides, - uk_local_target_loss_weights, -) -from microcosm.build.uk_runtime.local_geography import ( - align_area_targets, -) -from microcosm.build.uk_runtime.local_rowwise import ( - UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, - UK_LOCAL_HOLDOUT_FOLDS, - UK_LOCAL_HOLDOUT_SEED, - UKRowwiseDoctrineSolve, - UKRowwiseLocalMatrix, - UKRowwiseNationalRows, - build_uk_rowwise_local_matrix, - build_uk_rowwise_local_surface_matrix, - past_cap_census, - require_adjudicated_uk_local_binding, - rotated_uk_local_holdout, - rowwise_area_support_summary, - rowwise_calibration_mass_reason, - solve_uk_rowwise_weights_under_doctrine, - uk_area_support_summary, - uk_ladder_area_support_summary, -) -from microcosm.build.uk_runtime.local_target_census import ( - CENSUS_KIND, - CENSUS_RESOURCE, - CENSUS_SCHEMA_VERSION, - METRIC_STATUS_BOUND_IN_CODE, - SOURCE_STATUS_DOCUMENTED_UNPINNED, - assert_uk_local_target_census_current, - build_uk_local_target_census, - committed_uk_local_target_census_path, - load_uk_local_target_census, - write_uk_local_target_census, -) -from microcosm.build.uk_runtime.local_targets import ( - AGE_BANDS, - AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, - AREA_TYPES, - COUNTRY_TO_REGION, - INCOME_VARIABLES, - LA_EXTRA_METRICS, - area_groups_from_codes, - compute_household_metrics, - metric_names, - metric_names_from_target_profile, - metric_tables_by_area_group, -) -from microcosm.build.uk_runtime.national_calibration import ( - CalibrationFrameAdapter, - drop_injected_measure_inputs, - inject_measure_inputs, - prepare_uk_target_frame, -) -from microcosm.build.uk_runtime.national_doctrine import ( - UK_NATIONAL_L0_LAMBDA, - UK_NATIONAL_LEARNING_RATE, - UK_NATIONAL_MASS_RULE, - UK_NATIONAL_MAX_WEIGHT_RATIO, - UK_NATIONAL_SEED, - UK_NATIONAL_SOLVE_DOCTRINE, - UK_NATIONAL_SOLVE_EPOCHS, - UK_NATIONAL_TARGET_LOSS_CAP, - UK_NATIONAL_TARGET_WEIGHT_RULE, - UKNationalSolveDoctrine, - uk_doctrine_with_overrides, - uk_national_target_loss_weights, -) -from microcosm.build.uk_runtime.national_frame import ( - UK_NATIONAL_SCHEMA, - UKNationalStage, - UKStagingProvenance, - load_uk_national_frame, - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_spine_frame, - uk_spine_source_family_units, -) -from microcosm.build.uk_runtime.oa_ladder_sources import ( - LADDER_OA_COLUMNS, - assemble_uk_oa_ladder, - concat_uk_ladder_frames, - join_uk_oa_ladder_layers, -) -from microcosm.build.uk_runtime.parity_reference import ( - EFRS_PARITY_KNOWN_GAPS_RESOURCE, - EFRS_PARITY_REFERENCE_RESOURCE, - EfrsParityKnownGap, - EfrsParityReference, - EfrsParitySource, - load_efrs_parity_known_gaps, - load_efrs_parity_reference, -) -from microcosm.build.uk_runtime.release_identity import ( - UK_DENSE_RELEASE_ID, - UK_RELEASE_TIER_CPS_TRANSFER, - UK_RELEASE_TIER_FRS, - UK_RELEASE_TIERS, - UKReleaseIdentity, - apply_uk_release_identity, - format_uk_release_id, - validate_uk_release_tier, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, - UK_LOADER_INPUT_ALIASES, - UK_RELEASE_INPUT_COVERAGE_RESOURCE, - PolicyEngineUKCoverageEngine, - UKEffectiveMassCoveragePolicy, - UKReleaseInputColumn, - UKReleaseInputCoverageManifest, - assert_uk_release_input_coverage_build_stages, - assert_uk_release_input_coverage_manifest_current, - load_uk_release_input_coverage_manifest, - uk_release_input_coverage_gate, - uk_release_input_coverage_required_columns, - uk_release_input_coverage_reviewed_exclusions, -) -from microcosm.build.uk_runtime.rowwise_dataset import ( - ARTIFACT_CLONE_INDEX_COLUMN, - BENUNIT_ID_COLUMNS, - HOUSEHOLD_ID_COLUMNS, - MASS_CONSERVATION_RELATIVE_TOLERANCE, - PERSON_ID_COLUMNS, - POOL_SOURCE_LINEAGE_COLUMN, - UK_SINGLE_YEAR_TABLES, - UKLadderRowwiseDatasetResult, - UKRowwiseDatasetResult, - apply_uk_source_lineage_modulus, - clone_uk_dataset_tables_with_ladder_geography, - clone_uk_dataset_tables_with_rowwise_geography, - clone_uk_dataset_with_ladder_geography, - clone_uk_dataset_with_rowwise_geography, - ladder_clone_index_column, - load_uk_rowwise_dataset, - read_uk_single_year_weight_metadata, - validate_uk_ladder_rowwise_dataset_tables, - validate_uk_rowwise_dataset_tables, - write_uk_rowwise_dataset, -) -from microcosm.build.uk_runtime.rowwise_geography import ( - AREA_TYPE_TO_CROSSWALK_COLUMN, - CROSSWALK_COLUMNS, - FRS_REGION_TO_COUNTRY, - FRS_REGION_TO_REGION_CODE, - ROWWISE_GEOGRAPHY_COLUMNS, - RowwiseGeographyAssignment, - assign_household_geography, - clone_entity_frame, - expected_uk_rowwise_area_support, - geography_coverage_summary, - id_multiplier_for_values, - prepare_geography_crosswalk, - validate_geography_coverage, -) -from microcosm.build.uk_runtime.size_evaluation import ( - PRE_REGISTERED_OUTCOMES_V1, - RowwiseRun, - area_support_tables, - dense_reference_deltas, - fit_tables, - footprint, - frozen_vs_recomputed, - gate_table, - load_run, - paired_targets, - run_acceptance, - summarize, - weight_tables, -) -from microcosm.build.uk_runtime.spi_income import ( - SPI_DONOR_DOI, - SPI_DONOR_FILENAME, - SPI_DONOR_SHA256, - SPI_DONOR_SIZE_BYTES, - SPI_DONOR_UKDS_STUDY, - SPI_DONOR_VINTAGE, - SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, - UKSPIIncomeImputationResult, - assert_frs_hmrc_auxiliary_crosswalk_available, - derive_hmrc_income_auxiliaries, - impute_uk_spi_income_support, - verify_spi_donor_identity, -) -from microcosm.build.uk_runtime.spi_spine import ( - EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, - UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, - UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, - UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, - UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, - UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, - UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, - UKFRSHMRCSpineLeavesResult, - UKFRSHMRCSpineLeavesStageTransform, - UKSPIIncomeSpineResult, - UKSPIIncomeSpineStageTransform, - UKSPISupportChannelStageTransform, -) -from microcosm.build.uk_runtime.spi_support import ( - BASE_FRS_SUPPORT_CHANNEL, - DEFAULT_SPI_PRIOR_MASS_SHARE, - DEFAULT_SPI_SUPPORT_HOUSEHOLDS, - FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, - FRS_ONLY_SPI_FILL_PERSON_COLUMNS, - FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_INCOME_COMPONENT_COLUMNS, - SPI_INCOME_IMPUTATION_COLUMNS, - SPI_PRIOR_MASS_CHANGE_REASON, - SPI_REPLACEMENT_STRATA_COLUMNS, - SPI_SYNTHETIC_SUPPORT_CHANNEL, - UK_SPI_SUPPORT_STAGE_NAME, - UKSPISupportResult, - build_uk_spi_support_channel, - create_uk_spi_support_tables, - fill_support_channel_from_source, - replace_uk_spi_support_tables, - support_channel_column, - support_clone_index_column, - support_source_id_column, -) -from microcosm.build.uk_runtime.stage_checkpoints import ( - UK_FRAME_METADATA_KEY, - load_uk_stage_checkpoint, - load_uk_stage_predecessor, - uk_stage_metadata, -) -from microcosm.build.uk_runtime.terminal_gates import ( - UK_DEFAULT_ZERO_WEIGHT_STRATA, - UK_MAX_TARGET_ABS_RELATIVE_ERROR, - UKZeroWeightStratumDeclaration, - uk_degenerate_release_surface_gate, - uk_export_surface_gate, - uk_target_fit_gate, - uk_target_surface_gate, - uk_weight_ess_gate, - uk_weight_ratio_gate, - uk_zero_weight_strata_gate, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - UKInputMassParityPolicy, - UKInputMassReference, - UKQRFTailConcentrationPolicy, - load_uk_input_mass_reference, - load_uk_local_area_support_exclusion_register, - load_uk_reviewed_exclusion_register, - uk_input_mass_parity_gate, - uk_input_mass_totals, - uk_qrf_tail_concentration_columns, - uk_qrf_tail_concentration_gate, -) +Direct helper imports do not initialize unrelated calibration or source loaders. +The public export roster is unchanged; requesting every export with ``import *`` +still imports every defining module. Use ``dir`` or ``__all__`` for discovery: +``vars`` contains only exports already resolved in this process. +""" + +from importlib import import_module as _import_module +from types import MappingProxyType as _MappingProxyType +from typing import TYPE_CHECKING as _TYPE_CHECKING +from typing import Any as _Any + +if _TYPE_CHECKING: + from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, + UK_AGE_TAIL_BANDS, + UK_AGE_TAIL_DECLARED_SEEDS, + UK_AGE_TOP_CODE, + UKAgeTailStageTransform, + disaggregate_uk_age_top_code, + load_uk_age_tail_band_populations, + ) + from microcosm.build.uk_runtime.battery_bindings import ( + UK_GATE_REGISTRY, + UKGateBinding, + ) + from microcosm.build.uk_runtime.calibration_run import ( + load_bound_spine_sidecar, + runtime_provenance, + spine_provenance_from_sidecar, + ) + from microcosm.build.uk_runtime.cgt_calibration import ( + UK_CGT_ANNUAL_EXEMPT_AMOUNTS, + UK_CGT_GAINS_AMOUNT_COLUMN, + UK_CGT_SOURCE_COLUMN, + UK_CGT_TAXPAYER_COUNT_COLUMN, + UKCGTTargetMaterialization, + materialize_uk_cgt_calibration_frame, + uk_cgt_annual_exempt_amount, + ) + from microcosm.build.uk_runtime.content_identity import ( + uk_frame_content_identity, + ) + from microcosm.build.uk_runtime.diagnostics import ( + UK_DIAGNOSTICS_SCHEMA_VERSION, + UK_TARGET_GEOGRAPHY_LEVELS, + uk_calibration_diagnostics_payload, + uk_fit_by_family, + uk_support_limited_misses, + uk_weakest_areas_by_fit, + uk_weakest_families, + uk_weight_summary, + uk_zero_weight_strata, + write_uk_calibration_diagnostics, + ) + from microcosm.build.uk_runtime.firm_generation import ( + EMPLOYMENT_BANDS, + HMRC_BAND_COLUMNS, + INPUT_FILES, + VAT_LIABILITY_BANDS, + VINTAGES, + AxiomVATRuleEvaluator, + UKFirmCalibrationResult, + UKFirmGenerationConfig, + UKFirmGenerationResult, + UKFirmSourceData, + UKFirmTargetLayout, + UKFirmValidationReport, + UKFirmVATRuleEvaluator, + assign_employment, + assign_vat_flags, + build_firm_target_matrix, + employment_band_name, + generate_base_firms, + generate_input_values, + generate_uk_firm_population, + hmrc_band_name, + map_to_hmrc_band_indices, + optimize_firm_weights, + read_uk_firm_source_data, + solve_firm_weights, + target_diagnostics, + uk_firm_source_data_from_frames, + uk_firm_source_data_from_ledger_facts, + validate_uk_firm_population, + write_uk_firm_population, + ) + from microcosm.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_COVERAGE_REQUIREMENTS, + UK_CGT_TARGET_SPECS, + UK_FISCAL_TARGET_REGISTRY, + ) + from microcosm.build.uk_runtime.frs_council_tax import ( + FRS_COUNCIL_TAX_OUTPUT_COLUMNS, + UKFRSCouncilTaxStageTransform, + add_frs_council_tax, + derive_council_tax, + ) + from microcosm.build.uk_runtime.frs_disability import ( + FRS_DISABILITY_OUTPUT_COLUMNS, + UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, + UKDWPDisabilityCategoryRates, + UKDWPDisabilityFlagRates, + UKFRSDisabilityStageTransform, + add_frs_disability, + derive_frs_disability, + uk_dwp_disability_category_rates, + uk_dwp_disability_flag_rates, + ) + from microcosm.build.uk_runtime.frs_education import ( + EDUCQUAL_MAP, + FRS_EDUCATION_OUTPUT_COLUMNS, + UKFRSEducationStageTransform, + add_frs_education, + derive_current_education, + derive_frs_education, + ) + from microcosm.build.uk_runtime.frs_education_grants import ( + FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, + FRS_EDUCATION_GRANT_REWRITES, + UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, + UKDSAPolicy, + UKFRSEducationGrantSplitStageTransform, + add_frs_education_grant_split, + allocate_reported_education_grants, + disabled_students_allowance_capacity, + uk_dsa_policy, + ) + from microcosm.build.uk_runtime.frs_employment import ( + FRS_EMPLOYMENT_OUTPUT_COLUMNS, + UKFRSEmploymentStageTransform, + add_frs_employment, + derive_frs_employment, + ) + from microcosm.build.uk_runtime.frs_hmrc_leaves import ( + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_RETAINED_LEAF_COLUMNS, + FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + UKFRSHMRCRetainedLeavesResult, + UKFRSHMRCRetainedLeavesStageTransform, + retain_uk_frs_hmrc_leaves, + ) + from microcosm.build.uk_runtime.frs_legacy_proxies import ( + FRS_LEGACY_PROXY_OUTPUT_COLUMNS, + UK_LEGACY_PROXY_PREDICTORS, + UKFRSLegacyProxiesStageTransform, + UKLegacyJSAPolicy, + derive_frs_legacy_proxies, + uk_legacy_jsa_policy, + ) + from microcosm.build.uk_runtime.frs_release import ( + UK_YEAR_RULES, + UKFRSRelease, + load_uk_frs_release, + resolve_uk_year_rule, + ) + from microcosm.build.uk_runtime.geography_ladder import ( + GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, + GEOGRAPHY_LADDER_VINTAGES_ATTR, + UK_ENGLAND_WALES_REGION_CODES, + UK_GEOGRAPHY_LADDER_COLUMNS, + UK_LONDON_REGION_CODE, + UK_OA_LADDER_DERIVED_LAYERS, + UK_OA_LADDER_KIND, + UK_OA_LADDER_SCHEMA_VERSION, + UkOaLadder, + assign_uk_geography_ladder, + expected_uk_ladder_area_support, + load_uk_oa_ladder, + uk_geography_ladder_assignment_summary, + uk_geography_ladder_gate, + uk_region_mix, + ) + from microcosm.build.uk_runtime.geography_sources import ( + ENGLAND_LAD_REGION_URL, + ENGLAND_WALES_OA2021_COUNT, + EW_OA_CONSTITUENCY_URL, + EW_OA_HIERARCHY_URL, + EW_OA_HOUSEHOLDS_URL, + EW_OA_LAD23_URL, + EW_OA_POPULATION_URL, + EW_OA_WARD_URL, + LAD23_ITL_URL, + NI_DZ2021_COUNT, + NI_DZ_GEOJSON_ZIP_URL, + NI_DZ_HOUSEHOLDS_CSV_URL, + NI_DZ_LOOKUP_SHEET, + NI_DZ_PARLCON24_LOOKUP_XLSX_URL, + NI_DZ_POPULATION_CSV_URL, + NI_PARLCON24_COUNT, + SCOTLAND_CENSUS_INDEX_ZIP_URL, + SCOTLAND_OA2022_COUNT, + SCOTLAND_OA_CONSTITUENCY_URL, + SCOTLAND_OA_DZ_IZ_URL, + SCOTLAND_OA_LAU_ITL_URL, + SCOTLAND_OA_POPULATION_URL, + build_complete_uk_geography_crosswalk, + build_england_wales_crosswalk, + build_great_britain_crosswalk, + build_northern_ireland_crosswalk, + build_official_uk_geography_crosswalk, + build_scotland_crosswalk, + load_england_lad_region_lookup, + load_england_wales_oa_constituencies, + load_england_wales_oa_hierarchy, + load_england_wales_oa_households, + load_england_wales_oa_population, + load_england_wales_oa_ward_lookup, + load_ew_oa_lad23_lookup, + load_lad_itl_lookup, + load_ni_dz_hierarchy, + load_ni_dz_households, + load_ni_dz_parlcon24_lookup, + load_ni_dz_population, + load_ni_dz_ward_lookup, + load_scotland_oa_constituencies, + load_scotland_oa_dz_iz_lookup, + load_scotland_oa_households, + load_scotland_oa_lau_lookup, + load_scotland_oa_population, + load_scotland_oa_ward_lookup, + update_england_wales_lad_codes, + write_geography_crosswalk, + ) + from microcosm.build.uk_runtime.hmrc_calibration import ( + DEFAULT_HMRC_CALIBRATION_EPOCHS, + DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, + DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, + DEFAULT_HMRC_MAX_WEIGHT_RATIO, + HMRC_ASSESSABLE_INCOME_COLUMN, + HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, + HMRC_TAXPAYER_COLUMN, + UKHMRCIncomeCalibration, + UKHMRCTargetMaterialization, + calibrate_uk_hmrc_income, + materialize_uk_hmrc_calibration_frame, + ) + from microcosm.build.uk_runtime.hmrc_income import ( + HMRC_SPI_BUILD_PERIOD, + HMRC_SPI_COLLATED_ODS_URL, + HMRC_SPI_INCOME_COMPONENTS, + HMRC_SPI_PUBLICATION_URL, + HMRC_SPI_SOURCE_VINTAGE, + HMRC_SPI_TARGET_RECORD_COUNT, + HMRCIncomeBandTargetRecord, + HMRCIncomeSourceProvenance, + HMRCIncomeTargetSet, + materialize_hmrc_spi_income_band_targets, + verify_hmrc_spi_collated_ods, + ) + from microcosm.build.uk_runtime.hmrc_replay import ( + CANONICAL_HMRC_FACT_FENCES, + FULL_FRS_TI_BAND_FENCE_ID, + HMRCFactFence, + HMRCReplayDiagnosticAggregate, + HMRCReplayFact, + HMRCReplayReport, + build_conservative_hmrc_replay_report, + classify_hmrc_replay_targets, + write_hmrc_replay_report, + ) + from microcosm.build.uk_runtime.hmrc_source_contract import ( + HMRC_DISTRIBUTIONAL_INPUTS, + UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, + assert_uk_hmrc_income_source_contract_current, + ) + from microcosm.build.uk_runtime.ladder_targets import ( + constituency_household_targets, + ladder_target_provenance, + ladder_vs_chronicle_household_dispersion, + local_authority_household_targets, + ) + from microcosm.build.uk_runtime.ledger_targets import ( + UK_CENSUS_HOUSEHOLDS_TARGET_ID, + UK_CROSS_GRAIN_BRIDGES, + UK_CROSS_GRAIN_GRAIN_PRECEDENCE, + UK_CROSS_GRAIN_RULE, + UKFrameTargetAdapter, + UKLedgerTargetCompilation, + apply_uk_cross_grain_reconciliation, + compile_uk_local_target_registry, + compile_uk_target_registry, + load_uk_local_area_crosswalk, + materialize_uk_ledger_targets, + uk_census_household_uprating, + uk_ledger_households_total, + uk_local_target_surface, + ) + from microcosm.build.uk_runtime.local_doctrine import ( + UK_LOCAL_CLONE_COUNT, + UK_LOCAL_MAX_WEIGHT_RATIO, + UK_LOCAL_SOLVE_DOCTRINE, + UK_LOCAL_SOLVE_EPOCHS, + UK_LOCAL_TARGET_LOSS_CAP, + UK_LOCAL_TARGET_WEIGHT_RULE, + UKLocalSolveDoctrine, + uk_local_doctrine_with_overrides, + uk_local_target_loss_weights, + ) + from microcosm.build.uk_runtime.local_geography import ( + align_area_targets, + ) + from microcosm.build.uk_runtime.local_rowwise import ( + UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + UK_LOCAL_HOLDOUT_FOLDS, + UK_LOCAL_HOLDOUT_SEED, + UKRowwiseDoctrineSolve, + UKRowwiseLocalMatrix, + UKRowwiseNationalRows, + build_uk_rowwise_local_matrix, + build_uk_rowwise_local_surface_matrix, + past_cap_census, + require_adjudicated_uk_local_binding, + rotated_uk_local_holdout, + rowwise_area_support_summary, + rowwise_calibration_mass_reason, + solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, + uk_ladder_area_support_summary, + ) + from microcosm.build.uk_runtime.local_target_census import ( + CENSUS_KIND, + CENSUS_RESOURCE, + CENSUS_SCHEMA_VERSION, + METRIC_STATUS_BOUND_IN_CODE, + SOURCE_STATUS_DOCUMENTED_UNPINNED, + assert_uk_local_target_census_current, + build_uk_local_target_census, + committed_uk_local_target_census_path, + load_uk_local_target_census, + write_uk_local_target_census, + ) + from microcosm.build.uk_runtime.local_targets import ( + AGE_BANDS, + AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, + AREA_TYPES, + COUNTRY_TO_REGION, + INCOME_VARIABLES, + LA_EXTRA_METRICS, + area_groups_from_codes, + compute_household_metrics, + metric_names, + metric_names_from_target_profile, + metric_tables_by_area_group, + ) + from microcosm.build.uk_runtime.national_calibration import ( + CalibrationFrameAdapter, + drop_injected_measure_inputs, + inject_measure_inputs, + prepare_uk_target_frame, + ) + from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, + UKNationalSolveDoctrine, + uk_doctrine_with_overrides, + uk_national_target_loss_weights, + ) + from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + UKNationalStage, + UKStagingProvenance, + load_uk_national_frame, + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, + write_uk_national_frame, + ) + from microcosm.build.uk_runtime.national_sampling import ( + sample_uk_spine_frame, + uk_spine_source_family_units, + ) + from microcosm.build.uk_runtime.oa_ladder_sources import ( + LADDER_OA_COLUMNS, + assemble_uk_oa_ladder, + concat_uk_ladder_frames, + join_uk_oa_ladder_layers, + ) + from microcosm.build.uk_runtime.parity_reference import ( + EFRS_PARITY_KNOWN_GAPS_RESOURCE, + EFRS_PARITY_REFERENCE_RESOURCE, + EfrsParityKnownGap, + EfrsParityReference, + EfrsParitySource, + load_efrs_parity_known_gaps, + load_efrs_parity_reference, + ) + from microcosm.build.uk_runtime.release_identity import ( + UK_DENSE_RELEASE_ID, + UK_RELEASE_TIER_CPS_TRANSFER, + UK_RELEASE_TIER_FRS, + UK_RELEASE_TIERS, + UKReleaseIdentity, + apply_uk_release_identity, + format_uk_release_id, + validate_uk_release_tier, + ) + from microcosm.build.uk_runtime.release_input_coverage import ( + RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, + UK_LOADER_INPUT_ALIASES, + UK_RELEASE_INPUT_COVERAGE_RESOURCE, + PolicyEngineUKCoverageEngine, + UKEffectiveMassCoveragePolicy, + UKReleaseInputColumn, + UKReleaseInputCoverageManifest, + assert_uk_release_input_coverage_build_stages, + assert_uk_release_input_coverage_manifest_current, + load_uk_release_input_coverage_manifest, + uk_release_input_coverage_gate, + uk_release_input_coverage_required_columns, + uk_release_input_coverage_reviewed_exclusions, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ( + ARTIFACT_CLONE_INDEX_COLUMN, + BENUNIT_ID_COLUMNS, + HOUSEHOLD_ID_COLUMNS, + MASS_CONSERVATION_RELATIVE_TOLERANCE, + PERSON_ID_COLUMNS, + POOL_SOURCE_LINEAGE_COLUMN, + UK_SINGLE_YEAR_TABLES, + UKLadderRowwiseDatasetResult, + UKRowwiseDatasetResult, + apply_uk_source_lineage_modulus, + clone_uk_dataset_tables_with_ladder_geography, + clone_uk_dataset_tables_with_rowwise_geography, + clone_uk_dataset_with_ladder_geography, + clone_uk_dataset_with_rowwise_geography, + ladder_clone_index_column, + load_uk_rowwise_dataset, + read_uk_single_year_weight_metadata, + validate_uk_ladder_rowwise_dataset_tables, + validate_uk_rowwise_dataset_tables, + write_uk_rowwise_dataset, + ) + from microcosm.build.uk_runtime.rowwise_geography import ( + AREA_TYPE_TO_CROSSWALK_COLUMN, + CROSSWALK_COLUMNS, + FRS_REGION_TO_COUNTRY, + FRS_REGION_TO_REGION_CODE, + ROWWISE_GEOGRAPHY_COLUMNS, + RowwiseGeographyAssignment, + assign_household_geography, + clone_entity_frame, + expected_uk_rowwise_area_support, + geography_coverage_summary, + id_multiplier_for_values, + prepare_geography_crosswalk, + validate_geography_coverage, + ) + from microcosm.build.uk_runtime.size_evaluation import ( + PRE_REGISTERED_OUTCOMES_V1, + RowwiseRun, + area_support_tables, + dense_reference_deltas, + fit_tables, + footprint, + frozen_vs_recomputed, + gate_table, + load_run, + paired_targets, + run_acceptance, + summarize, + weight_tables, + ) + from microcosm.build.uk_runtime.spi_income import ( + SPI_DONOR_DOI, + SPI_DONOR_FILENAME, + SPI_DONOR_SHA256, + SPI_DONOR_SIZE_BYTES, + SPI_DONOR_UKDS_STUDY, + SPI_DONOR_VINTAGE, + SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, + UKSPIIncomeImputationResult, + assert_frs_hmrc_auxiliary_crosswalk_available, + derive_hmrc_income_auxiliaries, + impute_uk_spi_income_support, + verify_spi_donor_identity, + ) + from microcosm.build.uk_runtime.spi_spine import ( + EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, + UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, + UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, + UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, + UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, + UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, + UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, + UKFRSHMRCSpineLeavesResult, + UKFRSHMRCSpineLeavesStageTransform, + UKSPIIncomeSpineResult, + UKSPIIncomeSpineStageTransform, + UKSPISupportChannelStageTransform, + ) + from microcosm.build.uk_runtime.spi_support import ( + BASE_FRS_SUPPORT_CHANNEL, + DEFAULT_SPI_PRIOR_MASS_SHARE, + DEFAULT_SPI_SUPPORT_HOUSEHOLDS, + FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, + FRS_ONLY_SPI_FILL_PERSON_COLUMNS, + FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, + HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, + SPI_INCOME_COMPONENT_COLUMNS, + SPI_INCOME_IMPUTATION_COLUMNS, + SPI_PRIOR_MASS_CHANGE_REASON, + SPI_REPLACEMENT_STRATA_COLUMNS, + SPI_SYNTHETIC_SUPPORT_CHANNEL, + UK_SPI_SUPPORT_STAGE_NAME, + UKSPISupportResult, + build_uk_spi_support_channel, + create_uk_spi_support_tables, + fill_support_channel_from_source, + replace_uk_spi_support_tables, + support_channel_column, + support_clone_index_column, + support_source_id_column, + ) + from microcosm.build.uk_runtime.stage_checkpoints import ( + UK_FRAME_METADATA_KEY, + load_uk_stage_checkpoint, + load_uk_stage_predecessor, + uk_stage_metadata, + ) + from microcosm.build.uk_runtime.terminal_gates import ( + UK_DEFAULT_ZERO_WEIGHT_STRATA, + UK_MAX_TARGET_ABS_RELATIVE_ERROR, + UKZeroWeightStratumDeclaration, + uk_degenerate_release_surface_gate, + uk_export_surface_gate, + uk_target_fit_gate, + uk_target_surface_gate, + uk_weight_ess_gate, + uk_weight_ratio_gate, + uk_zero_weight_strata_gate, + ) + from microcosm.build.uk_runtime.weighted_integrity import ( + UKInputMassParityPolicy, + UKInputMassReference, + UKQRFTailConcentrationPolicy, + load_uk_input_mass_reference, + load_uk_local_area_support_exclusion_register, + load_uk_reviewed_exclusion_register, + uk_input_mass_parity_gate, + uk_input_mass_totals, + uk_qrf_tail_concentration_columns, + uk_qrf_tail_concentration_gate, + ) __all__ = [ "PRE_REGISTERED_OUTCOMES_V1", @@ -1006,3 +1018,1793 @@ "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", ] + + +_EXPORTS = _MappingProxyType( + { + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE", + ), + "UK_AGE_TAIL_BANDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BANDS", + ), + "UK_AGE_TAIL_DECLARED_SEEDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_DECLARED_SEEDS", + ), + "UK_AGE_TOP_CODE": ("microcosm.build.uk_runtime.age_tail", "UK_AGE_TOP_CODE"), + "UKAgeTailStageTransform": ( + "microcosm.build.uk_runtime.age_tail", + "UKAgeTailStageTransform", + ), + "disaggregate_uk_age_top_code": ( + "microcosm.build.uk_runtime.age_tail", + "disaggregate_uk_age_top_code", + ), + "load_uk_age_tail_band_populations": ( + "microcosm.build.uk_runtime.age_tail", + "load_uk_age_tail_band_populations", + ), + "UK_GATE_REGISTRY": ( + "microcosm.build.uk_runtime.battery_bindings", + "UK_GATE_REGISTRY", + ), + "UKGateBinding": ( + "microcosm.build.uk_runtime.battery_bindings", + "UKGateBinding", + ), + "load_bound_spine_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "load_bound_spine_sidecar", + ), + "runtime_provenance": ( + "microcosm.build.uk_runtime.calibration_run", + "runtime_provenance", + ), + "spine_provenance_from_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "spine_provenance_from_sidecar", + ), + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS", + ), + "UK_CGT_GAINS_AMOUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_GAINS_AMOUNT_COLUMN", + ), + "UK_CGT_SOURCE_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_SOURCE_COLUMN", + ), + "UK_CGT_TAXPAYER_COUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_TAXPAYER_COUNT_COLUMN", + ), + "UKCGTTargetMaterialization": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UKCGTTargetMaterialization", + ), + "materialize_uk_cgt_calibration_frame": ( + "microcosm.build.uk_runtime.cgt_calibration", + "materialize_uk_cgt_calibration_frame", + ), + "uk_cgt_annual_exempt_amount": ( + "microcosm.build.uk_runtime.cgt_calibration", + "uk_cgt_annual_exempt_amount", + ), + "uk_frame_content_identity": ( + "microcosm.build.uk_runtime.content_identity", + "uk_frame_content_identity", + ), + "UK_DIAGNOSTICS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_DIAGNOSTICS_SCHEMA_VERSION", + ), + "UK_TARGET_GEOGRAPHY_LEVELS": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_TARGET_GEOGRAPHY_LEVELS", + ), + "uk_calibration_diagnostics_payload": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_calibration_diagnostics_payload", + ), + "uk_fit_by_family": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_fit_by_family", + ), + "uk_support_limited_misses": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_support_limited_misses", + ), + "uk_weakest_areas_by_fit": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_areas_by_fit", + ), + "uk_weakest_families": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_families", + ), + "uk_weight_summary": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weight_summary", + ), + "uk_zero_weight_strata": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_zero_weight_strata", + ), + "write_uk_calibration_diagnostics": ( + "microcosm.build.uk_runtime.diagnostics", + "write_uk_calibration_diagnostics", + ), + "EMPLOYMENT_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "EMPLOYMENT_BANDS", + ), + "HMRC_BAND_COLUMNS": ( + "microcosm.build.uk_runtime.firm_generation", + "HMRC_BAND_COLUMNS", + ), + "INPUT_FILES": ("microcosm.build.uk_runtime.firm_generation", "INPUT_FILES"), + "VAT_LIABILITY_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "VAT_LIABILITY_BANDS", + ), + "VINTAGES": ("microcosm.build.uk_runtime.firm_generation", "VINTAGES"), + "AxiomVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "AxiomVATRuleEvaluator", + ), + "UKFirmCalibrationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmCalibrationResult", + ), + "UKFirmGenerationConfig": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationConfig", + ), + "UKFirmGenerationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationResult", + ), + "UKFirmSourceData": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmSourceData", + ), + "UKFirmTargetLayout": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmTargetLayout", + ), + "UKFirmValidationReport": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmValidationReport", + ), + "UKFirmVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmVATRuleEvaluator", + ), + "assign_employment": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_employment", + ), + "assign_vat_flags": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_vat_flags", + ), + "build_firm_target_matrix": ( + "microcosm.build.uk_runtime.firm_generation", + "build_firm_target_matrix", + ), + "employment_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "employment_band_name", + ), + "generate_base_firms": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_base_firms", + ), + "generate_input_values": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_input_values", + ), + "generate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_uk_firm_population", + ), + "hmrc_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "hmrc_band_name", + ), + "map_to_hmrc_band_indices": ( + "microcosm.build.uk_runtime.firm_generation", + "map_to_hmrc_band_indices", + ), + "optimize_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "optimize_firm_weights", + ), + "read_uk_firm_source_data": ( + "microcosm.build.uk_runtime.firm_generation", + "read_uk_firm_source_data", + ), + "solve_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "solve_firm_weights", + ), + "target_diagnostics": ( + "microcosm.build.uk_runtime.firm_generation", + "target_diagnostics", + ), + "uk_firm_source_data_from_frames": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_frames", + ), + "uk_firm_source_data_from_ledger_facts": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_ledger_facts", + ), + "validate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "validate_uk_firm_population", + ), + "write_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "write_uk_firm_population", + ), + "UK_CGT_REQUIRED_COLUMNS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_REQUIRED_COLUMNS", + ), + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", + ), + "UK_CGT_TARGET_SPECS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_SPECS", + ), + "UK_FISCAL_TARGET_REGISTRY": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_FISCAL_TARGET_REGISTRY", + ), + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_council_tax", + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS", + ), + "UKFRSCouncilTaxStageTransform": ( + "microcosm.build.uk_runtime.frs_council_tax", + "UKFRSCouncilTaxStageTransform", + ), + "add_frs_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "add_frs_council_tax", + ), + "derive_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "derive_council_tax", + ), + "FRS_DISABILITY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "FRS_DISABILITY_OUTPUT_COLUMNS", + ), + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS", + ), + "UKDWPDisabilityCategoryRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityCategoryRates", + ), + "UKDWPDisabilityFlagRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityFlagRates", + ), + "UKFRSDisabilityStageTransform": ( + "microcosm.build.uk_runtime.frs_disability", + "UKFRSDisabilityStageTransform", + ), + "add_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "add_frs_disability", + ), + "derive_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "derive_frs_disability", + ), + "uk_dwp_disability_category_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_category_rates", + ), + "uk_dwp_disability_flag_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_flag_rates", + ), + "EDUCQUAL_MAP": ("microcosm.build.uk_runtime.frs_education", "EDUCQUAL_MAP"), + "FRS_EDUCATION_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education", + "FRS_EDUCATION_OUTPUT_COLUMNS", + ), + "UKFRSEducationStageTransform": ( + "microcosm.build.uk_runtime.frs_education", + "UKFRSEducationStageTransform", + ), + "add_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "add_frs_education", + ), + "derive_current_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_current_education", + ), + "derive_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_frs_education", + ), + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS", + ), + "FRS_EDUCATION_GRANT_REWRITES": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_REWRITES", + ), + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS", + ), + "UKDSAPolicy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKDSAPolicy", + ), + "UKFRSEducationGrantSplitStageTransform": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKFRSEducationGrantSplitStageTransform", + ), + "add_frs_education_grant_split": ( + "microcosm.build.uk_runtime.frs_education_grants", + "add_frs_education_grant_split", + ), + "allocate_reported_education_grants": ( + "microcosm.build.uk_runtime.frs_education_grants", + "allocate_reported_education_grants", + ), + "disabled_students_allowance_capacity": ( + "microcosm.build.uk_runtime.frs_education_grants", + "disabled_students_allowance_capacity", + ), + "uk_dsa_policy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "uk_dsa_policy", + ), + "FRS_EMPLOYMENT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_employment", + "FRS_EMPLOYMENT_OUTPUT_COLUMNS", + ), + "UKFRSEmploymentStageTransform": ( + "microcosm.build.uk_runtime.frs_employment", + "UKFRSEmploymentStageTransform", + ), + "add_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "add_frs_employment", + ), + "derive_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "derive_frs_employment", + ), + "FRS_HMRC_INCPBEN_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_INCPBEN_COLUMN", + ), + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", + ), + "FRS_HMRC_PAY_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_PAY_COLUMN", + ), + "FRS_HMRC_RETAINED_LEAF_COLUMNS": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAF_COLUMNS", + ), + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", + ), + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", + ), + "FRS_HMRC_UBISJA_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_UBISJA_COLUMN", + ), + "UKFRSHMRCRetainedLeavesResult": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesResult", + ), + "UKFRSHMRCRetainedLeavesStageTransform": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesStageTransform", + ), + "retain_uk_frs_hmrc_leaves": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "retain_uk_frs_hmrc_leaves", + ), + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS", + ), + "UK_LEGACY_PROXY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UK_LEGACY_PROXY_PREDICTORS", + ), + "UKFRSLegacyProxiesStageTransform": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKFRSLegacyProxiesStageTransform", + ), + "UKLegacyJSAPolicy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKLegacyJSAPolicy", + ), + "derive_frs_legacy_proxies": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "derive_frs_legacy_proxies", + ), + "uk_legacy_jsa_policy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "uk_legacy_jsa_policy", + ), + "UK_YEAR_RULES": ("microcosm.build.uk_runtime.frs_release", "UK_YEAR_RULES"), + "UKFRSRelease": ("microcosm.build.uk_runtime.frs_release", "UKFRSRelease"), + "load_uk_frs_release": ( + "microcosm.build.uk_runtime.frs_release", + "load_uk_frs_release", + ), + "resolve_uk_year_rule": ( + "microcosm.build.uk_runtime.frs_release", + "resolve_uk_year_rule", + ), + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + ), + "GEOGRAPHY_LADDER_VINTAGES_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + ), + "UK_ENGLAND_WALES_REGION_CODES": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_ENGLAND_WALES_REGION_CODES", + ), + "UK_GEOGRAPHY_LADDER_COLUMNS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_GEOGRAPHY_LADDER_COLUMNS", + ), + "UK_LONDON_REGION_CODE": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_LONDON_REGION_CODE", + ), + "UK_OA_LADDER_DERIVED_LAYERS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_DERIVED_LAYERS", + ), + "UK_OA_LADDER_KIND": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_KIND", + ), + "UK_OA_LADDER_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_SCHEMA_VERSION", + ), + "UkOaLadder": ("microcosm.build.uk_runtime.geography_ladder", "UkOaLadder"), + "assign_uk_geography_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "assign_uk_geography_ladder", + ), + "expected_uk_ladder_area_support": ( + "microcosm.build.uk_runtime.geography_ladder", + "expected_uk_ladder_area_support", + ), + "load_uk_oa_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "load_uk_oa_ladder", + ), + "uk_geography_ladder_assignment_summary": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_assignment_summary", + ), + "uk_geography_ladder_gate": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_gate", + ), + "uk_region_mix": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_region_mix", + ), + "ENGLAND_LAD_REGION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_LAD_REGION_URL", + ), + "ENGLAND_WALES_OA2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_WALES_OA2021_COUNT", + ), + "EW_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_CONSTITUENCY_URL", + ), + "EW_OA_HIERARCHY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HIERARCHY_URL", + ), + "EW_OA_HOUSEHOLDS_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HOUSEHOLDS_URL", + ), + "EW_OA_LAD23_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_LAD23_URL", + ), + "EW_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_POPULATION_URL", + ), + "EW_OA_WARD_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_WARD_URL", + ), + "LAD23_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "LAD23_ITL_URL", + ), + "NI_DZ2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ2021_COUNT", + ), + "NI_DZ_GEOJSON_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_GEOJSON_ZIP_URL", + ), + "NI_DZ_HOUSEHOLDS_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_HOUSEHOLDS_CSV_URL", + ), + "NI_DZ_LOOKUP_SHEET": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_LOOKUP_SHEET", + ), + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL", + ), + "NI_DZ_POPULATION_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_POPULATION_CSV_URL", + ), + "NI_PARLCON24_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_PARLCON24_COUNT", + ), + "SCOTLAND_CENSUS_INDEX_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_CENSUS_INDEX_ZIP_URL", + ), + "SCOTLAND_OA2022_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA2022_COUNT", + ), + "SCOTLAND_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_CONSTITUENCY_URL", + ), + "SCOTLAND_OA_DZ_IZ_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_DZ_IZ_URL", + ), + "SCOTLAND_OA_LAU_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_LAU_ITL_URL", + ), + "SCOTLAND_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_POPULATION_URL", + ), + "build_complete_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_complete_uk_geography_crosswalk", + ), + "build_england_wales_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_england_wales_crosswalk", + ), + "build_great_britain_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_great_britain_crosswalk", + ), + "build_northern_ireland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_northern_ireland_crosswalk", + ), + "build_official_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_official_uk_geography_crosswalk", + ), + "build_scotland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_scotland_crosswalk", + ), + "load_england_lad_region_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_lad_region_lookup", + ), + "load_england_wales_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_constituencies", + ), + "load_england_wales_oa_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_hierarchy", + ), + "load_england_wales_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_households", + ), + "load_england_wales_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_population", + ), + "load_england_wales_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_ward_lookup", + ), + "load_ew_oa_lad23_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ew_oa_lad23_lookup", + ), + "load_lad_itl_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_lad_itl_lookup", + ), + "load_ni_dz_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_hierarchy", + ), + "load_ni_dz_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_households", + ), + "load_ni_dz_parlcon24_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_parlcon24_lookup", + ), + "load_ni_dz_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_population", + ), + "load_ni_dz_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_ward_lookup", + ), + "load_scotland_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_constituencies", + ), + "load_scotland_oa_dz_iz_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_dz_iz_lookup", + ), + "load_scotland_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_households", + ), + "load_scotland_oa_lau_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_lau_lookup", + ), + "load_scotland_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_population", + ), + "load_scotland_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_ward_lookup", + ), + "update_england_wales_lad_codes": ( + "microcosm.build.uk_runtime.geography_sources", + "update_england_wales_lad_codes", + ), + "write_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "write_geography_crosswalk", + ), + "DEFAULT_HMRC_CALIBRATION_EPOCHS": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_EPOCHS", + ), + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE", + ), + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR", + ), + "DEFAULT_HMRC_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_WEIGHT_RATIO", + ), + "HMRC_ASSESSABLE_INCOME_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_ASSESSABLE_INCOME_COLUMN", + ), + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN", + ), + "HMRC_TAXPAYER_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXPAYER_COLUMN", + ), + "UKHMRCIncomeCalibration": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCIncomeCalibration", + ), + "UKHMRCTargetMaterialization": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCTargetMaterialization", + ), + "calibrate_uk_hmrc_income": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "calibrate_uk_hmrc_income", + ), + "materialize_uk_hmrc_calibration_frame": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "materialize_uk_hmrc_calibration_frame", + ), + "HMRC_SPI_BUILD_PERIOD": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_BUILD_PERIOD", + ), + "HMRC_SPI_COLLATED_ODS_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_COLLATED_ODS_URL", + ), + "HMRC_SPI_INCOME_COMPONENTS": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_INCOME_COMPONENTS", + ), + "HMRC_SPI_PUBLICATION_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_PUBLICATION_URL", + ), + "HMRC_SPI_SOURCE_VINTAGE": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_SOURCE_VINTAGE", + ), + "HMRC_SPI_TARGET_RECORD_COUNT": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_TARGET_RECORD_COUNT", + ), + "HMRCIncomeBandTargetRecord": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeBandTargetRecord", + ), + "HMRCIncomeSourceProvenance": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeSourceProvenance", + ), + "HMRCIncomeTargetSet": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeTargetSet", + ), + "materialize_hmrc_spi_income_band_targets": ( + "microcosm.build.uk_runtime.hmrc_income", + "materialize_hmrc_spi_income_band_targets", + ), + "verify_hmrc_spi_collated_ods": ( + "microcosm.build.uk_runtime.hmrc_income", + "verify_hmrc_spi_collated_ods", + ), + "CANONICAL_HMRC_FACT_FENCES": ( + "microcosm.build.uk_runtime.hmrc_replay", + "CANONICAL_HMRC_FACT_FENCES", + ), + "FULL_FRS_TI_BAND_FENCE_ID": ( + "microcosm.build.uk_runtime.hmrc_replay", + "FULL_FRS_TI_BAND_FENCE_ID", + ), + "HMRCFactFence": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCFactFence"), + "HMRCReplayDiagnosticAggregate": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayDiagnosticAggregate", + ), + "HMRCReplayFact": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCReplayFact"), + "HMRCReplayReport": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayReport", + ), + "build_conservative_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "build_conservative_hmrc_replay_report", + ), + "classify_hmrc_replay_targets": ( + "microcosm.build.uk_runtime.hmrc_replay", + "classify_hmrc_replay_targets", + ), + "write_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "write_hmrc_replay_report", + ), + "HMRC_DISTRIBUTIONAL_INPUTS": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "HMRC_DISTRIBUTIONAL_INPUTS", + ), + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", + ), + "assert_uk_hmrc_income_source_contract_current": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "assert_uk_hmrc_income_source_contract_current", + ), + "constituency_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "constituency_household_targets", + ), + "ladder_target_provenance": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_target_provenance", + ), + "ladder_vs_chronicle_household_dispersion": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_vs_chronicle_household_dispersion", + ), + "local_authority_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "local_authority_household_targets", + ), + "UK_CENSUS_HOUSEHOLDS_TARGET_ID": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CENSUS_HOUSEHOLDS_TARGET_ID", + ), + "UK_CROSS_GRAIN_BRIDGES": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_BRIDGES", + ), + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE", + ), + "UK_CROSS_GRAIN_RULE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_RULE", + ), + "UKFrameTargetAdapter": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKFrameTargetAdapter", + ), + "UKLedgerTargetCompilation": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKLedgerTargetCompilation", + ), + "apply_uk_cross_grain_reconciliation": ( + "microcosm.build.uk_runtime.ledger_targets", + "apply_uk_cross_grain_reconciliation", + ), + "compile_uk_local_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_local_target_registry", + ), + "compile_uk_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_target_registry", + ), + "load_uk_local_area_crosswalk": ( + "microcosm.build.uk_runtime.ledger_targets", + "load_uk_local_area_crosswalk", + ), + "materialize_uk_ledger_targets": ( + "microcosm.build.uk_runtime.ledger_targets", + "materialize_uk_ledger_targets", + ), + "uk_census_household_uprating": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_census_household_uprating", + ), + "uk_ledger_households_total": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_ledger_households_total", + ), + "uk_local_target_surface": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_local_target_surface", + ), + "UK_LOCAL_CLONE_COUNT": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_CLONE_COUNT", + ), + "UK_LOCAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_MAX_WEIGHT_RATIO", + ), + "UK_LOCAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_DOCTRINE", + ), + "UK_LOCAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_EPOCHS", + ), + "UK_LOCAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_LOSS_CAP", + ), + "UK_LOCAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_WEIGHT_RULE", + ), + "UKLocalSolveDoctrine": ( + "microcosm.build.uk_runtime.local_doctrine", + "UKLocalSolveDoctrine", + ), + "uk_local_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_doctrine_with_overrides", + ), + "uk_local_target_loss_weights": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_target_loss_weights", + ), + "align_area_targets": ( + "microcosm.build.uk_runtime.local_geography", + "align_area_targets", + ), + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE", + ), + "UK_LOCAL_HOLDOUT_FOLDS": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_FOLDS", + ), + "UK_LOCAL_HOLDOUT_SEED": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_SEED", + ), + "UKRowwiseDoctrineSolve": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseDoctrineSolve", + ), + "UKRowwiseLocalMatrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseLocalMatrix", + ), + "UKRowwiseNationalRows": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseNationalRows", + ), + "build_uk_rowwise_local_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_matrix", + ), + "build_uk_rowwise_local_surface_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_surface_matrix", + ), + "past_cap_census": ( + "microcosm.build.uk_runtime.local_rowwise", + "past_cap_census", + ), + "require_adjudicated_uk_local_binding": ( + "microcosm.build.uk_runtime.local_rowwise", + "require_adjudicated_uk_local_binding", + ), + "rotated_uk_local_holdout": ( + "microcosm.build.uk_runtime.local_rowwise", + "rotated_uk_local_holdout", + ), + "rowwise_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_area_support_summary", + ), + "rowwise_calibration_mass_reason": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_calibration_mass_reason", + ), + "solve_uk_rowwise_weights_under_doctrine": ( + "microcosm.build.uk_runtime.local_rowwise", + "solve_uk_rowwise_weights_under_doctrine", + ), + "uk_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_area_support_summary", + ), + "uk_ladder_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_ladder_area_support_summary", + ), + "CENSUS_KIND": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_KIND", + ), + "CENSUS_RESOURCE": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_RESOURCE", + ), + "CENSUS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_SCHEMA_VERSION", + ), + "METRIC_STATUS_BOUND_IN_CODE": ( + "microcosm.build.uk_runtime.local_target_census", + "METRIC_STATUS_BOUND_IN_CODE", + ), + "SOURCE_STATUS_DOCUMENTED_UNPINNED": ( + "microcosm.build.uk_runtime.local_target_census", + "SOURCE_STATUS_DOCUMENTED_UNPINNED", + ), + "assert_uk_local_target_census_current": ( + "microcosm.build.uk_runtime.local_target_census", + "assert_uk_local_target_census_current", + ), + "build_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "build_uk_local_target_census", + ), + "committed_uk_local_target_census_path": ( + "microcosm.build.uk_runtime.local_target_census", + "committed_uk_local_target_census_path", + ), + "load_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "load_uk_local_target_census", + ), + "write_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "write_uk_local_target_census", + ), + "AGE_BANDS": ("microcosm.build.uk_runtime.local_targets", "AGE_BANDS"), + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL": ( + "microcosm.build.uk_runtime.local_targets", + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL", + ), + "AREA_TYPES": ("microcosm.build.uk_runtime.local_targets", "AREA_TYPES"), + "COUNTRY_TO_REGION": ( + "microcosm.build.uk_runtime.local_targets", + "COUNTRY_TO_REGION", + ), + "INCOME_VARIABLES": ( + "microcosm.build.uk_runtime.local_targets", + "INCOME_VARIABLES", + ), + "LA_EXTRA_METRICS": ( + "microcosm.build.uk_runtime.local_targets", + "LA_EXTRA_METRICS", + ), + "area_groups_from_codes": ( + "microcosm.build.uk_runtime.local_targets", + "area_groups_from_codes", + ), + "compute_household_metrics": ( + "microcosm.build.uk_runtime.local_targets", + "compute_household_metrics", + ), + "metric_names": ("microcosm.build.uk_runtime.local_targets", "metric_names"), + "metric_names_from_target_profile": ( + "microcosm.build.uk_runtime.local_targets", + "metric_names_from_target_profile", + ), + "metric_tables_by_area_group": ( + "microcosm.build.uk_runtime.local_targets", + "metric_tables_by_area_group", + ), + "CalibrationFrameAdapter": ( + "microcosm.build.uk_runtime.national_calibration", + "CalibrationFrameAdapter", + ), + "drop_injected_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "drop_injected_measure_inputs", + ), + "inject_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "inject_measure_inputs", + ), + "prepare_uk_target_frame": ( + "microcosm.build.uk_runtime.national_calibration", + "prepare_uk_target_frame", + ), + "UK_NATIONAL_L0_LAMBDA": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_L0_LAMBDA", + ), + "UK_NATIONAL_LEARNING_RATE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_LEARNING_RATE", + ), + "UK_NATIONAL_MASS_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MASS_RULE", + ), + "UK_NATIONAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MAX_WEIGHT_RATIO", + ), + "UK_NATIONAL_SEED": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SEED", + ), + "UK_NATIONAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_DOCTRINE", + ), + "UK_NATIONAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_EPOCHS", + ), + "UK_NATIONAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_LOSS_CAP", + ), + "UK_NATIONAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + ), + "UKNationalSolveDoctrine": ( + "microcosm.build.uk_runtime.national_doctrine", + "UKNationalSolveDoctrine", + ), + "uk_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_doctrine_with_overrides", + ), + "uk_national_target_loss_weights": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_national_target_loss_weights", + ), + "UK_NATIONAL_SCHEMA": ( + "microcosm.build.uk_runtime.national_frame", + "UK_NATIONAL_SCHEMA", + ), + "UKNationalStage": ( + "microcosm.build.uk_runtime.national_frame", + "UKNationalStage", + ), + "UKStagingProvenance": ( + "microcosm.build.uk_runtime.national_frame", + "UKStagingProvenance", + ), + "load_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "load_uk_national_frame", + ), + "uk_household_weight_kind": ( + "microcosm.build.uk_runtime.national_frame", + "uk_household_weight_kind", + ), + "uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "uk_national_frame", + ), + "uk_time_period": ( + "microcosm.build.uk_runtime.national_frame", + "uk_time_period", + ), + "validate_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "validate_uk_national_frame", + ), + "write_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "write_uk_national_frame", + ), + "sample_uk_spine_frame": ( + "microcosm.build.uk_runtime.national_sampling", + "sample_uk_spine_frame", + ), + "uk_spine_source_family_units": ( + "microcosm.build.uk_runtime.national_sampling", + "uk_spine_source_family_units", + ), + "LADDER_OA_COLUMNS": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "LADDER_OA_COLUMNS", + ), + "assemble_uk_oa_ladder": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "assemble_uk_oa_ladder", + ), + "concat_uk_ladder_frames": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "concat_uk_ladder_frames", + ), + "join_uk_oa_ladder_layers": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "join_uk_oa_ladder_layers", + ), + "EFRS_PARITY_KNOWN_GAPS_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_KNOWN_GAPS_RESOURCE", + ), + "EFRS_PARITY_REFERENCE_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_REFERENCE_RESOURCE", + ), + "EfrsParityKnownGap": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityKnownGap", + ), + "EfrsParityReference": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityReference", + ), + "EfrsParitySource": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParitySource", + ), + "load_efrs_parity_known_gaps": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_known_gaps", + ), + "load_efrs_parity_reference": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_reference", + ), + "UK_DENSE_RELEASE_ID": ( + "microcosm.build.uk_runtime.release_identity", + "UK_DENSE_RELEASE_ID", + ), + "UK_RELEASE_TIER_CPS_TRANSFER": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_CPS_TRANSFER", + ), + "UK_RELEASE_TIER_FRS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_FRS", + ), + "UK_RELEASE_TIERS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIERS", + ), + "UKReleaseIdentity": ( + "microcosm.build.uk_runtime.release_identity", + "UKReleaseIdentity", + ), + "apply_uk_release_identity": ( + "microcosm.build.uk_runtime.release_identity", + "apply_uk_release_identity", + ), + "format_uk_release_id": ( + "microcosm.build.uk_runtime.release_identity", + "format_uk_release_id", + ), + "validate_uk_release_tier": ( + "microcosm.build.uk_runtime.release_identity", + "validate_uk_release_tier", + ), + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS": ( + "microcosm.build.uk_runtime.release_input_coverage", + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS", + ), + "UK_LOADER_INPUT_ALIASES": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_LOADER_INPUT_ALIASES", + ), + "UK_RELEASE_INPUT_COVERAGE_RESOURCE": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_RELEASE_INPUT_COVERAGE_RESOURCE", + ), + "PolicyEngineUKCoverageEngine": ( + "microcosm.build.uk_runtime.release_input_coverage", + "PolicyEngineUKCoverageEngine", + ), + "UKEffectiveMassCoveragePolicy": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKEffectiveMassCoveragePolicy", + ), + "UKReleaseInputColumn": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputColumn", + ), + "UKReleaseInputCoverageManifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputCoverageManifest", + ), + "assert_uk_release_input_coverage_build_stages": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_build_stages", + ), + "assert_uk_release_input_coverage_manifest_current": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_manifest_current", + ), + "load_uk_release_input_coverage_manifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "load_uk_release_input_coverage_manifest", + ), + "uk_release_input_coverage_gate": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_gate", + ), + "uk_release_input_coverage_required_columns": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_required_columns", + ), + "uk_release_input_coverage_reviewed_exclusions": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_reviewed_exclusions", + ), + "ARTIFACT_CLONE_INDEX_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ARTIFACT_CLONE_INDEX_COLUMN", + ), + "BENUNIT_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "BENUNIT_ID_COLUMNS", + ), + "HOUSEHOLD_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "HOUSEHOLD_ID_COLUMNS", + ), + "MASS_CONSERVATION_RELATIVE_TOLERANCE": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "MASS_CONSERVATION_RELATIVE_TOLERANCE", + ), + "PERSON_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "PERSON_ID_COLUMNS", + ), + "POOL_SOURCE_LINEAGE_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "POOL_SOURCE_LINEAGE_COLUMN", + ), + "UK_SINGLE_YEAR_TABLES": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UK_SINGLE_YEAR_TABLES", + ), + "UKLadderRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKLadderRowwiseDatasetResult", + ), + "UKRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKRowwiseDatasetResult", + ), + "apply_uk_source_lineage_modulus": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "apply_uk_source_lineage_modulus", + ), + "clone_uk_dataset_tables_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_ladder_geography", + ), + "clone_uk_dataset_tables_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_rowwise_geography", + ), + "clone_uk_dataset_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_ladder_geography", + ), + "clone_uk_dataset_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_rowwise_geography", + ), + "ladder_clone_index_column": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ladder_clone_index_column", + ), + "load_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "load_uk_rowwise_dataset", + ), + "read_uk_single_year_weight_metadata": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "read_uk_single_year_weight_metadata", + ), + "validate_uk_ladder_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_ladder_rowwise_dataset_tables", + ), + "validate_uk_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_rowwise_dataset_tables", + ), + "write_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "write_uk_rowwise_dataset", + ), + "AREA_TYPE_TO_CROSSWALK_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_geography", + "AREA_TYPE_TO_CROSSWALK_COLUMN", + ), + "CROSSWALK_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "CROSSWALK_COLUMNS", + ), + "FRS_REGION_TO_COUNTRY": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_COUNTRY", + ), + "FRS_REGION_TO_REGION_CODE": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_REGION_CODE", + ), + "ROWWISE_GEOGRAPHY_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "ROWWISE_GEOGRAPHY_COLUMNS", + ), + "RowwiseGeographyAssignment": ( + "microcosm.build.uk_runtime.rowwise_geography", + "RowwiseGeographyAssignment", + ), + "assign_household_geography": ( + "microcosm.build.uk_runtime.rowwise_geography", + "assign_household_geography", + ), + "clone_entity_frame": ( + "microcosm.build.uk_runtime.rowwise_geography", + "clone_entity_frame", + ), + "expected_uk_rowwise_area_support": ( + "microcosm.build.uk_runtime.rowwise_geography", + "expected_uk_rowwise_area_support", + ), + "geography_coverage_summary": ( + "microcosm.build.uk_runtime.rowwise_geography", + "geography_coverage_summary", + ), + "id_multiplier_for_values": ( + "microcosm.build.uk_runtime.rowwise_geography", + "id_multiplier_for_values", + ), + "prepare_geography_crosswalk": ( + "microcosm.build.uk_runtime.rowwise_geography", + "prepare_geography_crosswalk", + ), + "validate_geography_coverage": ( + "microcosm.build.uk_runtime.rowwise_geography", + "validate_geography_coverage", + ), + "PRE_REGISTERED_OUTCOMES_V1": ( + "microcosm.build.uk_runtime.size_evaluation", + "PRE_REGISTERED_OUTCOMES_V1", + ), + "RowwiseRun": ("microcosm.build.uk_runtime.size_evaluation", "RowwiseRun"), + "area_support_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "area_support_tables", + ), + "dense_reference_deltas": ( + "microcosm.build.uk_runtime.size_evaluation", + "dense_reference_deltas", + ), + "fit_tables": ("microcosm.build.uk_runtime.size_evaluation", "fit_tables"), + "footprint": ("microcosm.build.uk_runtime.size_evaluation", "footprint"), + "frozen_vs_recomputed": ( + "microcosm.build.uk_runtime.size_evaluation", + "frozen_vs_recomputed", + ), + "gate_table": ("microcosm.build.uk_runtime.size_evaluation", "gate_table"), + "load_run": ("microcosm.build.uk_runtime.size_evaluation", "load_run"), + "paired_targets": ( + "microcosm.build.uk_runtime.size_evaluation", + "paired_targets", + ), + "run_acceptance": ( + "microcosm.build.uk_runtime.size_evaluation", + "run_acceptance", + ), + "summarize": ("microcosm.build.uk_runtime.size_evaluation", "summarize"), + "weight_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "weight_tables", + ), + "SPI_DONOR_DOI": ("microcosm.build.uk_runtime.spi_income", "SPI_DONOR_DOI"), + "SPI_DONOR_FILENAME": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_FILENAME", + ), + "SPI_DONOR_SHA256": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SHA256", + ), + "SPI_DONOR_SIZE_BYTES": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SIZE_BYTES", + ), + "SPI_DONOR_UKDS_STUDY": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_UKDS_STUDY", + ), + "SPI_DONOR_VINTAGE": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_VINTAGE", + ), + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS", + ), + "UKSPIIncomeImputationResult": ( + "microcosm.build.uk_runtime.spi_income", + "UKSPIIncomeImputationResult", + ), + "assert_frs_hmrc_auxiliary_crosswalk_available": ( + "microcosm.build.uk_runtime.spi_income", + "assert_frs_hmrc_auxiliary_crosswalk_available", + ), + "derive_hmrc_income_auxiliaries": ( + "microcosm.build.uk_runtime.spi_income", + "derive_hmrc_income_auxiliaries", + ), + "impute_uk_spi_income_support": ( + "microcosm.build.uk_runtime.spi_income", + "impute_uk_spi_income_support", + ), + "verify_spi_donor_identity": ( + "microcosm.build.uk_runtime.spi_income", + "verify_spi_donor_identity", + ), + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN": ( + "microcosm.build.uk_runtime.spi_spine", + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN", + ), + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS", + ), + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME", + ), + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME", + ), + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND", + ), + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS", + ), + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS", + ), + "UKFRSHMRCSpineLeavesResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesResult", + ), + "UKFRSHMRCSpineLeavesStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesStageTransform", + ), + "UKSPIIncomeSpineResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineResult", + ), + "UKSPIIncomeSpineStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineStageTransform", + ), + "UKSPISupportChannelStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPISupportChannelStageTransform", + ), + "BASE_FRS_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "BASE_FRS_SUPPORT_CHANNEL", + ), + "DEFAULT_SPI_PRIOR_MASS_SHARE": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_PRIOR_MASS_SHARE", + ), + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS", + ), + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS", + ), + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN": ( + "microcosm.build.uk_runtime.spi_support", + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN", + ), + "SPI_INCOME_COMPONENT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_COMPONENT_COLUMNS", + ), + "SPI_INCOME_IMPUTATION_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_IMPUTATION_COLUMNS", + ), + "SPI_PRIOR_MASS_CHANGE_REASON": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_PRIOR_MASS_CHANGE_REASON", + ), + "SPI_REPLACEMENT_STRATA_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_REPLACEMENT_STRATA_COLUMNS", + ), + "SPI_SYNTHETIC_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_SYNTHETIC_SUPPORT_CHANNEL", + ), + "UK_SPI_SUPPORT_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_support", + "UK_SPI_SUPPORT_STAGE_NAME", + ), + "UKSPISupportResult": ( + "microcosm.build.uk_runtime.spi_support", + "UKSPISupportResult", + ), + "build_uk_spi_support_channel": ( + "microcosm.build.uk_runtime.spi_support", + "build_uk_spi_support_channel", + ), + "create_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "create_uk_spi_support_tables", + ), + "fill_support_channel_from_source": ( + "microcosm.build.uk_runtime.spi_support", + "fill_support_channel_from_source", + ), + "replace_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "replace_uk_spi_support_tables", + ), + "support_channel_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_channel_column", + ), + "support_clone_index_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_clone_index_column", + ), + "support_source_id_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_source_id_column", + ), + "UK_FRAME_METADATA_KEY": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "UK_FRAME_METADATA_KEY", + ), + "load_uk_stage_checkpoint": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_checkpoint", + ), + "load_uk_stage_predecessor": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_predecessor", + ), + "uk_stage_metadata": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "uk_stage_metadata", + ), + "UK_DEFAULT_ZERO_WEIGHT_STRATA": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_DEFAULT_ZERO_WEIGHT_STRATA", + ), + "UK_MAX_TARGET_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_MAX_TARGET_ABS_RELATIVE_ERROR", + ), + "UKZeroWeightStratumDeclaration": ( + "microcosm.build.uk_runtime.terminal_gates", + "UKZeroWeightStratumDeclaration", + ), + "uk_degenerate_release_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_degenerate_release_surface_gate", + ), + "uk_export_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_export_surface_gate", + ), + "uk_target_fit_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_fit_gate", + ), + "uk_target_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_surface_gate", + ), + "uk_weight_ess_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ess_gate", + ), + "uk_weight_ratio_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ratio_gate", + ), + "uk_zero_weight_strata_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_zero_weight_strata_gate", + ), + "UKInputMassParityPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassParityPolicy", + ), + "UKInputMassReference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassReference", + ), + "UKQRFTailConcentrationPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKQRFTailConcentrationPolicy", + ), + "load_uk_input_mass_reference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_input_mass_reference", + ), + "load_uk_local_area_support_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_local_area_support_exclusion_register", + ), + "load_uk_reviewed_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_reviewed_exclusion_register", + ), + "uk_input_mass_parity_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_parity_gate", + ), + "uk_input_mass_totals": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_totals", + ), + "uk_qrf_tail_concentration_columns": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_columns", + ), + "uk_qrf_tail_concentration_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_gate", + ), + } +) + +# Reload clears public aliases, retaining the real defining modules and submodules. +for _export_name in __all__: + globals().pop(_export_name, None) +del _export_name + + +def __getattr__(name: str) -> _Any: + """Resolve and cache the exact public object, preserving import failures.""" + try: + module_name, attribute = _EXPORTS[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(_import_module(module_name), attribute) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """Discover declared exports without importing their defining modules.""" + return sorted(set(globals()) | set(__all__)) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py new file mode 100644 index 000000000..a6fedc9d6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py @@ -0,0 +1,388 @@ +"""Convert supplied UK ladder arrays for shared atomic geography operators. + +This module does not acquire sources, admit native data, calibrate, or assign a +location. Source identities and mapping classifications are explicit caller +claims; publisher authentication belongs to the host. Initial spine/support +clones must exist before the resulting declaration is executed. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + RELATIONS, + decode_atomic_support, + encode_atomic_support, + validate_assignment_spec, +) +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE +from microcosm.graph.canonical import canonical_json + +SYSTEMS = ( + "uk_ew_output_area_2021", + "uk_scotland_output_area_2022", + "uk_ni_data_zone_2021", +) +SOURCES = {system: system + "_support" for system in SYSTEMS} +IDENTITY_COLUMN = "geography_household_key" +_INPUT_COLUMNS = ( + "oa_code", + "population", + "households", + "constituency_code", + "region_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "itl3_code", +) +_COUNT_COLUMNS = frozenset({"population", "households"}) +_CODE = re.compile(r"[EWSN][0-9]{8}") +_WARD = re.compile(r"[EWSN][0-9]{2}[0-9RS][0-9]{5}") +_ITL = re.compile(r"TL[C-N][0-9A-Z]{2}") +_LIMIT = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK atomic support: " + message) + + +def _text(value: object) -> bool: + return type(value) is str and bool(value) and value.strip() == value + + +def _profile(system: str) -> tuple[str, str, tuple[str, ...], str]: + _require(system in SYSTEMS, "unknown area system") + if system == SYSTEMS[0]: + regions = tuple( + name + for name in FRS_REGION_TO_REGION_CODE + if name not in {"SCOTLAND", "NORTHERN_IRELAND"} + ) + return "output_area", "2021_census", regions, "EW" + if system == SYSTEMS[1]: + return "output_area", "2022_census", ("SCOTLAND",), "S" + return "data_zone", "2021_census", ("NORTHERN_IRELAND",), "N" + + +def _metadata(descriptions: Mapping, *, vintage: str, system: str) -> dict: + _require(type(descriptions) is dict, "column metadata must be a plain mapping") + _require(set(descriptions) == set(_INPUT_COLUMNS), "exact column metadata required") + result = {} + for column in _INPUT_COLUMNS: + item = descriptions[column] + _require(type(item) is dict, "plain column description required") + expected = ( + {"kind", "source", "basis"} + if column in _COUNT_COLUMNS + else {"kind", "source", "vintage", "relation"} + ) + _require(set(item) == expected, "column description keys differ") + _require( + all(_text(value) for value in item.values()), "nonempty metadata required" + ) + if column in _COUNT_COLUMNS: + _require(item["kind"] == "weight", "count metadata must describe weight") + else: + _require( + item["kind"] == "code" and item["relation"] in RELATIONS, + "code mapping classification", + ) + result[column] = dict(item) + _require( + result["oa_code"]["vintage"] == vintage + and result["oa_code"]["relation"] == "exact", + "atomic identity/vintage differs", + ) + if system == SYSTEMS[2]: + _require( + result["constituency_code"]["relation"] != "inferred_modal", + "NI postcode-modal mapping is not admitted by this adapter", + ) + return result + + +def _counts(array: np.ndarray, *, positive: bool) -> np.ndarray: + _require( + array.dtype.kind in "iu" or array.dtype == np.dtype("float64"), "count dtype" + ) + if array.dtype.kind == "f": + _require(np.isfinite(array).all(), "non-finite count") + _require(np.equal(array, np.floor(array)).all(), "non-integral count") + _require( + np.all(array > 0 if positive else array >= 0) and np.all(array <= _LIMIT), + "count range", + ) + integers = [int(value) for value in array] + _require(0 < sum(integers) <= _LIMIT, "count total outside exact sampling range") + converted = np.asarray(integers, dtype=np.int64) + _require( + np.array_equal(converted.astype(array.dtype), array), + "count conversion is not exact", + ) + return converted + + +def assemble_uk_atomic_area_support( + *, + system: str, + arrays: Mapping[str, np.ndarray], + column_metadata: Mapping, +) -> bytes: + """Normalize one complete nation system from supplied legacy ladder columns. + + Rows are sorted by atomic code, never dropped. Counts remain exact; old + float64 count arrays are accepted only when integral and exactly representable. + Each declared FRS region must have positive household sampling mass. The + caller supplies source/vintage/relation evidence; syntax is not authority. + NI legacy ward_code means district electoral area, and lsoa_code aliases DZ. + """ + level, vintage, regions, prefixes = _profile(system) + descriptions = _metadata(column_metadata, vintage=vintage, system=system) + _require( + type(arrays) is dict and set(arrays) == set(_INPUT_COLUMNS), + "exact arrays required", + ) + values = {} + for column in _INPUT_COLUMNS: + array = arrays[column] + _require( + type(array) is np.ndarray and array.ndim == 1 and len(array) > 0, + "nonempty one-dimensional arrays required", + ) + if column in _COUNT_COLUMNS: + values[column] = _counts(array, positive=column == "population") + continue + _require(array.dtype.kind == "U", "codes must be Unicode arrays") + _require( + np.all(np.char.str_len(array) > 0) + and np.all(np.char.strip(array) == array), + "blank or padded code", + ) + pattern = ( + _ITL if column == "itl3_code" else _WARD if column == "ward_code" else _CODE + ) + _require( + all(pattern.fullmatch(str(value)) for value in array), "invalid code syntax" + ) + if column != "itl3_code": + _require( + all(str(value)[0] in prefixes for value in array), "foreign nation code" + ) + values[column] = array.copy() + count = len(values["oa_code"]) + _require( + all(len(array) == count for array in values.values()), "array lengths differ" + ) + _require(len(np.unique(values["oa_code"])) == count, "duplicate atomic code") + nations = [str(value)[0] for value in values["oa_code"]] + for column in _INPUT_COLUMNS: + if column not in {*_COUNT_COLUMNS, "itl3_code"}: + _require( + [str(value)[0] for value in values[column]] == nations, + "mapping crosses a nation boundary", + ) + if system == SYSTEMS[2]: + _require( + np.array_equal(values["lsoa_code"], values["oa_code"]), + "NI legacy lsoa_code must alias its atomic Data Zone", + ) + region_names = {FRS_REGION_TO_REGION_CODE[name]: name for name in regions} + _require( + set(values["region_code"].tolist()) == set(region_names), + "incomplete or foreign FRS region coverage", + ) + for code in region_names: + _require( + sum(int(v) for v in values["households"][values["region_code"] == code]) + > 0, + "region has no household sampling mass", + ) + order = np.argsort(values["oa_code"], kind="stable") + values = {column: array[order] for column, array in values.items()} + normalized = {"area": values["oa_code"]} + normalized.update({k: v for k, v in values.items() if k != "oa_code"}) + columns = {"area": descriptions["oa_code"]} + columns.update({k: v for k, v in descriptions.items() if k != "oa_code"}) + normalized["frs_region"] = np.asarray( + [region_names[code] for code in values["region_code"]], dtype="U" + ) + region_rule = canonical_json( + { + "mapping": dict(FRS_REGION_TO_REGION_CODE), + "input": descriptions["region_code"], + } + ) + columns["frs_region"] = { + "kind": "code", + "source": "sha256:" + hashlib.sha256(region_rule).hexdigest(), + "vintage": "frs_region_enum_v1", + "relation": "exact", + } + for width in (3, 4): + column = "itl1_code" if width == 3 else "itl2_code" + normalized[column] = np.asarray([code[:width] for code in values["itl3_code"]]) + columns[column] = { + **descriptions["itl3_code"], + "source": descriptions["itl3_code"]["source"] + f"#prefix-{width}", + "relation": "exact", + } + return encode_atomic_support( + { + "version": 1, + "system": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "columns": columns, + }, + normalized, + ) + + +def _layers(system: str) -> tuple[tuple[str, str], ...]: + shared = tuple( + (column, column) + for column in _INPUT_COLUMNS + if column not in {"oa_code", *_COUNT_COLUMNS} + ) + aliases = (("area", "oa_code"),) + if system == SYSTEMS[0]: + native = (("area", "output_area_code"),) + elif system == SYSTEMS[1]: + native = ( + ("area", "output_area_code"), + ("lsoa_code", "data_zone_code"), + ("msoa_code", "intermediate_zone_code"), + ) + else: + native = ( + ("area", "data_zone_code"), + ("msoa_code", "super_data_zone_code"), + ("ward_code", "district_electoral_area_code"), + ) + return ( + *shared, + *aliases, + *native, + ("itl1_code", "itl1_code"), + ("itl2_code", "itl2_code"), + ) + + +def uk_atomic_assignment_definition( + supports: Mapping[str, bytes], + *, + seed: int, +) -> dict: + """Declare shared assignment after the complete initial spine/support roster. + + Source names are fixed by system, not file paths. The host must bind admitted + bytes to those SourceRefs and authenticate the supplied full-spine identity. + This function neither creates a graph barrier nor admits a native artifact. + """ + _require( + type(supports) is dict and set(supports) == set(SYSTEMS), + "three systems required", + ) + _require( + isinstance(seed, Integral) + and not isinstance(seed, (bool, np.bool_)) + and 0 <= seed < 2**32, + "seed must be a bounded integer", + ) + systems = [] + for system in SYSTEMS: + _require(type(supports[system]) is bytes, "immutable support bytes required") + support = decode_atomic_support(supports[system]) + level, vintage, regions, _ = _profile(system) + _require( + all( + support.metadata[k] == value + for k, value in ( + ("system", system), + ("level", level), + ("code_system", "uk_gss"), + ("vintage", vintage), + ) + ), + "support system identity differs", + ) + expected_columns = { + "area", + *(_INPUT_COLUMNS[1:]), + "frs_region", + "itl1_code", + "itl2_code", + } + _require(set(support.arrays) == expected_columns, "support columns differ") + # The declaration accepts this adapter's normalized form. This closes + # forged derived aliases/region normalization without claiming source + # authority from a self-consistent payload. + restored_arrays = { + column: support.arrays["area" if column == "oa_code" else column] + for column in _INPUT_COLUMNS + } + restored_metadata = { + column: dict( + support.metadata["columns"]["area" if column == "oa_code" else column] + ) + for column in _INPUT_COLUMNS + } + _require( + assemble_uk_atomic_area_support( + system=system, arrays=restored_arrays, column_metadata=restored_metadata + ) + == supports[system], + "support is not the canonical UK adapter output", + ) + systems.append( + { + "id": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "source": SOURCES[system], + "selector": {"region": list(regions)}, + "constraints": [ + {"input": "region", "support": "frs_region", "required": True} + ], + "observed_area": None, + "stages": [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ], + "layers": [ + { + "input": column, + "output": output, + **{ + k: support.metadata["columns"][column][k] + for k in ("source", "vintage", "relation") + }, + } + for column, output in _layers(system) + ], + } + ) + return validate_assignment_spec( + { + "version": 1, + "identity": [IDENTITY_COLUMN], + "stream": ["sha256-u53-v1", "uk-post-clone-atomic-area-v1", 0, int(seed)], + "outputs": { + "area": "atomic_area_code", + "system": "atomic_area_system", + "basis": "atomic_area_basis", + }, + "systems": systems, + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py new file mode 100644 index 000000000..1d8435b8b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py @@ -0,0 +1,85 @@ +"""Explicit post-clone household keys for UK atomic assignment. + +The host supplies original-source identity and ordered structural branches. +This helper neither infers ancestry from financial values/ID offsets nor +authenticates the supplied lineage. Source/EXPAND receipts remain authoritative. +""" + +from __future__ import annotations + +from numbers import Integral + +import numpy as np + +from microcosm.graph.canonical import canonical_json + +_BRANCHES = ( + "spi_support_channel", + "cgt_incidence_clone", + "cgt_band_donors", + "geographic_support", +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK geography identity: " + message) + + +def household_draw_key( + *, + source: str, + source_vintage: str, + source_household_id: int, + clone_path: tuple[tuple[str, int], ...], +) -> str: + """Encode an original-source ID and explicit ordered clone branch path. + + Every full-spine household needs a distinct key. Branch ordinals are + supplied lineage identities, never sample row counters. Original branches + should use their declared ordinal consistently; growing a pool must not + change an existing path. Calibration K, weights, income, and final offset + IDs are deliberately not inputs. Exact IDs above 2**53 remain distinct. + """ + _require( + all( + type(value) is str + and value + and value.strip() == value + and len(value) <= 256 + for value in (source, source_vintage) + ), + "qualified source and vintage required", + ) + _require( + isinstance(source_household_id, Integral) + and not isinstance(source_household_id, (bool, np.bool_)) + and 0 < source_household_id < 2**63, + "original source household ID must be an exact positive int64", + ) + _require(type(clone_path) is tuple, "explicit immutable clone path required") + path = [] + previous = -1 + for branch in clone_path: + _require( + type(branch) is tuple and len(branch) == 2, + "named branch/ordinal pair required", + ) + name, ordinal = branch + _require(type(name) is str and name in _BRANCHES, "unknown structural branch") + position = _BRANCHES.index(name) + _require(position > previous, "clone branches must be unique and ordered") + _require( + isinstance(ordinal, Integral) + and not isinstance(ordinal, (bool, np.bool_)) + and 0 <= ordinal < 2**32, + "clone ordinal must be a bounded exact integer", + ) + path.append([name, str(int(ordinal))]) + previous = position + return ( + "uk-household-v1:" + + canonical_json( + [source, source_vintage, str(int(source_household_id)), path] + ).decode() + ) diff --git a/packages/microcosm-build/tests/test_uk_atomic_area_support.py b/packages/microcosm-build/tests/test_uk_atomic_area_support.py new file mode 100644 index 000000000..b25887a87 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_area_support.py @@ -0,0 +1,482 @@ +"""Invented array/identity controls only; no publisher files or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + SYSTEMS, + assemble_uk_atomic_area_support, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE + + +def parts(system): + index = SYSTEMS.index(system) + regions = ( + [ + r + for r in FRS_REGION_TO_REGION_CODE + if r not in {"SCOTLAND", "NORTHERN_IRELAND"} + ] + if index == 0 + else ["SCOTLAND"] + if index == 1 + else ["NORTHERN_IRELAND"] + ) + rows = [] + region_order = list(FRS_REGION_TO_REGION_CODE) + for region in regions: + prefix = FRS_REGION_TO_REGION_CODE[region][0] + for offset in range(3): + serial = region_order.index(region) * 3 + offset + 1 + area = prefix + ("20" if index == 2 else "00") + f"{serial:06d}" + rows.append( + { + "oa_code": area, + "population": [2.0, 6.0, 1000.0][offset], + "households": [0.0, 30.0, 10.0][offset], + "constituency_code": prefix + + "14" + + f"{serial - offset + (offset == 2):06d}", + "region_code": FRS_REGION_TO_REGION_CODE[region], + "lsoa_code": area + if index == 2 + else prefix + "01" + f"{serial:06d}", + "msoa_code": prefix + "02" + f"{serial:06d}", + "local_authority_code": prefix + "06" + f"{serial:06d}", + "ward_code": prefix + "05" + f"{serial:06d}", + "itl3_code": "TL" + + "CDEFGHIJKLMN"[region_order.index(region)] + + "01", + } + ) + arrays = {column: np.asarray([r[column] for r in rows]) for column in rows[0]} + descriptions = {} + for column in arrays: + if column in {"population", "households"}: + descriptions[column] = { + "kind": "weight", + "source": "invented-" + system + "-" + column, + "basis": "invented persons" + if column == "population" + else "invented occupied households", + } + else: + descriptions[column] = { + "kind": "code", + "source": "invented-" + system + "-" + column, + "vintage": "2022_census" if index == 1 else "2021_census", + "relation": "exact" + if column in {"oa_code", "lsoa_code", "msoa_code", "region_code"} + else "best_fit", + } + descriptions["constituency_code"]["vintage"] = "2024_pcon" + if index == 2: + descriptions["constituency_code"]["relation"] = "official_tabulation" + return arrays, descriptions + + +def payloads(): + return { + system: assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + for system in SYSTEMS + for arrays, metadata in [parts(system)] + } + + +def key(source_id=101, path=()): + return household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=source_id, + clone_path=path, + ) + + +def complete(households, payload): + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = { + system: geo.decode_atomic_support(data) for system, data in payload.items() + } + result = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + result = pd.concat([result, geo.derive_geography(result, spec, supports)], axis=1) + return result, spec, supports + + +def test_normalization_preserves_all_rows_counts_mappings_and_source_labels(): + for system in SYSTEMS: + arrays, metadata = parts(system) + original = {k: v.copy() for k, v in arrays.items()} + data = assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + support = geo.decode_atomic_support(data) + order = np.argsort(arrays["oa_code"], kind="stable") + for column in arrays: + normalized = "area" if column == "oa_code" else column + np.testing.assert_array_equal( + support.arrays[normalized], arrays[column][order] + ) + assert dict(support.metadata["columns"][normalized]) == metadata[column] + np.testing.assert_array_equal(arrays[column], original[column]) + assert support.arrays["population"].dtype == np.dtype("int64") + assert support.arrays["households"].dtype == np.dtype("int64") + assert support.arrays["area"].flags.writeable is False + assert support.sha256 == geo.decode_atomic_support(data).sha256 + + +def test_normalized_bytes_are_stable_under_input_row_order_and_integer_count_storage(): + arrays, metadata = parts(SYSTEMS[0]) + expected = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + reversed_arrays = {k: v[::-1].copy() for k, v in arrays.items()} + for column in ("population", "households"): + reversed_arrays[column] = reversed_arrays[column].astype(np.int64) + assert ( + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=reversed_arrays, column_metadata=metadata + ) + == expected + ) + + +@pytest.mark.parametrize( + "defect", + [ + "nonintegral", + "float32", + "bool", + "negative", + "nan", + "overflow_total", + "zero_population", + "zero_region_households", + "duplicate_area", + "length", + "missing_column", + "object_code", + "foreign_nation", + "cross_ew_nation", + "blank_code", + "missing_region", + "bad_atomic_vintage", + "missing_source", + ], +) +def test_converter_refuses_bad_counts_codes_coverage_and_metadata(defect): + arrays, metadata = parts(SYSTEMS[0]) + if defect == "nonintegral": + arrays["households"][0] = 0.5 + elif defect == "float32": + arrays["households"] = arrays["households"].astype(np.float32) + elif defect == "bool": + arrays["households"] = arrays["households"] > 0 + elif defect == "negative": + arrays["households"][0] = -1 + elif defect == "nan": + arrays["population"][0] = np.nan + elif defect == "overflow_total": + arrays["population"][0] = float(2**53) + elif defect == "zero_population": + arrays["population"][0] = 0 + elif defect == "zero_region_households": + arrays["households"][:3] = 0 + elif defect == "duplicate_area": + arrays["oa_code"][1] = arrays["oa_code"][0] + elif defect == "length": + arrays["msoa_code"] = arrays["msoa_code"][:-1] + elif defect == "missing_column": + del arrays["population"] + elif defect == "object_code": + arrays["oa_code"] = arrays["oa_code"].astype(object) + elif defect == "foreign_nation": + arrays["ward_code"][0] = "S05000001" + elif defect == "cross_ew_nation": + arrays["ward_code"][0] = "W05000001" + elif defect == "blank_code": + arrays["ward_code"][0] = "" + elif defect == "missing_region": + arrays = {k: v[3:].copy() for k, v in arrays.items()} + elif defect == "bad_atomic_vintage": + metadata["oa_code"]["vintage"] = "2011_census" + elif defect == "missing_source": + metadata["population"]["source"] = "" + with pytest.raises(ValueError, match="UK atomic support:"): + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + + +@pytest.mark.parametrize("defect", ["inferred_ni_constituency", "ni_lsoa_not_dz"]) +def test_ni_source_convention_refusals(defect): + arrays, metadata = parts(SYSTEMS[2]) + if defect == "inferred_ni_constituency": + metadata["constituency_code"]["relation"] = "inferred_modal" + else: + arrays["lsoa_code"][0] = "N01000001" + with pytest.raises(ValueError, match="UK atomic support: NI"): + assemble_uk_atomic_area_support( + system=SYSTEMS[2], arrays=arrays, column_metadata=metadata + ) + + +def test_three_system_assignment_keeps_observed_region_and_native_alias_roles(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 5, dtype=np.int64), + "region": pd.array( + ["LONDON", "WALES", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + IDENTITY_COLUMN: pd.array( + [key(i) for i in (101, 102, 103, 104)], dtype="string" + ), + } + ) + result, spec, supports = complete(households, payloads()) + pd.testing.assert_series_equal(result["region"], households["region"]) + assert result["atomic_area_system"].tolist() == [ + SYSTEMS[0], + SYSTEMS[0], + SYSTEMS[1], + SYSTEMS[2], + ] + assert result["atomic_area_basis"].tolist() == ["assigned"] * 4 + assert result.loc[:2, "output_area_code"].equals(result.loc[:2, "atomic_area_code"]) + assert pd.isna(result.loc[3, "output_area_code"]) + assert result.loc[2, "data_zone_code"] == result.loc[2, "lsoa_code"] + assert result.loc[2, "intermediate_zone_code"] == result.loc[2, "msoa_code"] + assert result.loc[3, "data_zone_code"] == result.loc[3, "atomic_area_code"] + assert result.loc[3, "super_data_zone_code"] == result.loc[3, "msoa_code"] + assert result.loc[3, "district_electoral_area_code"] == result.loc[3, "ward_code"] + assert pd.isna(result.loc[0, "super_data_zone_code"]) + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + assert {s["source"] for s in spec["systems"]} == set(SOURCES.values()) + + +def test_source_metadata_changes_are_identified_and_declaration_refuses_forged_derived_field(): + payload = payloads() + original = payload[SYSTEMS[0]] + arrays, metadata = parts(SYSTEMS[0]) + metadata["population"]["source"] += "-revision-2" + changed = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + assert changed != original + support = geo.decode_atomic_support(original) + forged_arrays = {k: v.copy() for k, v in support.arrays.items()} + forged_arrays["itl1_code"][0] = "TLN" + forged = geo.encode_atomic_support( + {k: dict(v) if k == "columns" else v for k, v in support.metadata.items()}, + forged_arrays, + ) + payload[SYSTEMS[0]] = forged + with pytest.raises(ValueError, match="canonical UK adapter output"): + uk_atomic_assignment_definition(payload, seed=43) + + +@pytest.mark.parametrize("defect", ["missing_system", "wrong_system", "bad_seed"]) +def test_declaration_refuses_mismatched_inputs(defect): + payload = payloads() + options = {"seed": 43} + if defect == "missing_system": + del payload[SYSTEMS[2]] + elif defect == "wrong_system": + payload[SYSTEMS[2]] = payload[SYSTEMS[1]] + elif defect == "bad_seed": + options["seed"] = True + with pytest.raises(ValueError): + uk_atomic_assignment_definition(payload, **options) + + +def test_declaration_cannot_substitute_an_income_or_counter_identity_column(): + with pytest.raises(TypeError, match="identity_column"): + uk_atomic_assignment_definition( + payloads(), seed=43, identity_column="household_weight" + ) + + +def test_distinct_post_clone_keys_assign_then_remain_stable_under_order_subset_and_growth(): + paths = [ + (("spi_support_channel", 0), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + ] + original = pd.DataFrame( + { + "household_id": np.asarray([11, 22, 33, 44], dtype=np.int64), + "region": pd.array(["LONDON"] * 4, dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(101, path) for path in paths], dtype="string" + ), + } + ) + assert original[IDENTITY_COLUMN].nunique() == 4 + payload = payloads() + expected, spec, supports = complete(original, payload) + changed = original.iloc[[3, 1, 0]].copy() + changed["household_weight"] = [0.0, 2.0, 8.0] + extra = pd.DataFrame( + { + "household_id": [55], + "region": pd.array(["LONDON"], dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(202, (("geographic_support", 1),))], dtype="string" + ), + "household_weight": [1.0], + }, + index=[8], + ) + actual, _, _ = complete(pd.concat([changed, extra]), payload) + for column in expected: + if column != "region": + pd.testing.assert_series_equal( + actual.loc[changed.index, column], expected.loc[changed.index, column] + ) + assert geo.validate_geography(actual, spec, supports)["outcome"] == "pass" + assigned_columns = ["atomic_area_code", "oa_code", "constituency_code"] + pruned = expected.iloc[[3, 1]].copy() + assert geo.validate_geography(pruned, spec, supports)["outcome"] == "pass" + pd.testing.assert_frame_equal( + pruned[assigned_columns], expected.loc[pruned.index, assigned_columns] + ) + with pytest.raises(ValueError, match="assignment output already exists"): + geo.assign_atomic(expected, spec, supports) + + +def test_country_declaration_retains_the_two_stage_sampling_law(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 2001, dtype=np.int64), + "region": pd.array(["LONDON"] * 2000, dtype="string"), + IDENTITY_COLUMN: pd.array([key(i) for i in range(1, 2001)], dtype="string"), + } + ) + result, spec, _ = complete(households, payloads()) + assert all( + s["stages"] + == [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + for s in spec["systems"] + ) + arrays, _ = parts(SYSTEMS[0]) + london = arrays["region_code"] == FRS_REGION_TO_REGION_CODE["LONDON"] + areas = arrays["oa_code"][london] + shares = result["atomic_area_code"].value_counts(normalize=True) + # Constituent household mass 30:10, then within-first-area population 2:6; + # the third area's population 1000 must not dominate the first-stage draw. + for area, expected in zip(areas, (0.1875, 0.5625, 0.25), strict=True): + assert abs(shares.get(area, 0) - expected) < 0.04 + + +def test_exact_source_ids_and_ordered_branches_are_preserved(): + first = key(2**53 + 1, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + second = key(2**53 + 2, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + assert first != second + assert str(2**53 + 1) in first + assert first == key( + np.int64(2**53 + 1), + (("spi_support_channel", np.int64(1)), ("cgt_incidence_clone", 0)), + ) + grown = [key(101, (("geographic_support", i),)) for i in range(6)] + assert grown[:3] == [key(101, (("geographic_support", i),)) for i in range(3)] + + +@pytest.mark.parametrize( + "defect", + [ + "float_id", + "bool_id", + "zero_id", + "oversize_id", + "no_vintage", + "mutable_path", + "counter_branch", + "income_branch", + "max_id_branch", + "out_of_order", + "duplicate_branch", + "float_ordinal", + "negative_ordinal", + "oversize_ordinal", + ], +) +def test_identity_refuses_inferred_or_inexact_conventions(defect): + args = { + "source": "invented-frs", + "source_vintage": "invented-2024-25", + "source_household_id": 101, + "clone_path": (), + } + if defect == "float_id": + args["source_household_id"] = 101.0 + elif defect == "bool_id": + args["source_household_id"] = True + elif defect == "zero_id": + args["source_household_id"] = 0 + elif defect == "oversize_id": + args["source_household_id"] = 2**63 + elif defect == "no_vintage": + args["source_vintage"] = "" + elif defect == "mutable_path": + args["clone_path"] = [] + elif defect in {"counter_branch", "income_branch", "max_id_branch"}: + args["clone_path"] = ( + ( + { + "counter_branch": "row_counter", + "income_branch": "income", + "max_id_branch": "sample_max_id", + }[defect], + 1, + ), + ) + elif defect == "out_of_order": + args["clone_path"] = (("cgt_incidence_clone", 1), ("spi_support_channel", 1)) + elif defect == "duplicate_branch": + args["clone_path"] = (("spi_support_channel", 0), ("spi_support_channel", 1)) + else: + ordinal = { + "float_ordinal": 1.5, + "negative_ordinal": -1, + "oversize_ordinal": 2**32, + }[defect] + args["clone_path"] = (("spi_support_channel", ordinal),) + with pytest.raises(ValueError, match="UK geography identity:"): + household_draw_key(**args) + + +def test_duplicate_or_unknown_household_identity_is_refused_by_shared_assignment(): + payload = payloads() + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = {s: geo.decode_atomic_support(p) for s, p in payload.items()} + households = pd.DataFrame( + { + "household_id": [11, 22], + "region": ["LONDON", "LONDON"], + IDENTITY_COLUMN: [key(101), key(101)], + } + ) + with pytest.raises(ValueError, match="duplicate draw identity"): + geo.assign_atomic(households, spec, supports) + households.loc[1, IDENTITY_COLUMN] = key(102) + households.loc[1, "region"] = "UNKNOWN" + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) diff --git a/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py new file mode 100644 index 000000000..90f57fdf8 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py @@ -0,0 +1,138 @@ +"""Normal-import contracts for the real UK runtime public package. + +A fresh no-country-resource guard supplies the cold-import check. In a larger +suite, already imported country modules are allowed; these tests never remove, +stub or replace modules to manufacture an isolated import environment. +""" + +import hashlib +import importlib +import json +import sys + +import pytest + +_PACKAGE = "microcosm.build.uk_runtime" +_UNRELATED_MODULES = ( + f"{_PACKAGE}.calibration_run", + f"{_PACKAGE}.national_calibration", + f"{_PACKAGE}.firm_generation", +) +# SHA256 of compact JSON for all 456 ordered entries from ec3b306f3f9742ff... +# This includes the existing duplicate ladder_vs_chronicle_household_dispersion. +_ORDERED_ALL_SHA256 = "282baaa8103b6a68ec916669dc9b96909d6faf0a9beecdbb8359df12f128f66c" + + +def _unrelated_modules() -> set[str]: + return { + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in _UNRELATED_MODULES + ) + } + + +def test_normal_package_and_pure_helper_imports_do_not_load_unrelated_modules(): + before = _unrelated_modules() + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + + assert sys.modules[_PACKAGE] is package + assert package.rowwise_geography is helper + assert callable(helper.assign_household_geography) + assert _unrelated_modules() == before + + +def test_public_from_import_returns_and_caches_the_real_defining_object(): + from microcosm.build.uk_runtime import assign_household_geography + from microcosm.build.uk_runtime.rowwise_geography import ( + assign_household_geography as direct, + ) + + package = importlib.import_module(_PACKAGE) + assert assign_household_geography is direct + assert package.assign_household_geography is direct + assert vars(package)["assign_household_geography"] is direct + assert package.assign_household_geography is assign_household_geography + + +def test_direct_submodule_import_and_from_import_preserve_module_identity(): + # In the fresh guard this exercises Python's fallback before any direct + # content_identity import, without removing an already loaded module in CI. + from microcosm.build.uk_runtime import content_identity + + assert content_identity is sys.modules[f"{_PACKAGE}.content_identity"] + + import microcosm.build.uk_runtime.content_identity as content_direct + import microcosm.build.uk_runtime.rowwise_geography as direct + from microcosm.build.uk_runtime import rowwise_geography + + package = importlib.import_module(_PACKAGE) + assert content_identity is content_direct + assert content_direct is sys.modules[f"{_PACKAGE}.content_identity"] + assert package.content_identity is content_direct + assert "content_identity" not in package.__all__ + assert rowwise_geography is direct + assert direct is sys.modules[f"{_PACKAGE}.rowwise_geography"] + assert package.rowwise_geography is direct + assert "rowwise_geography" not in package.__all__ + + +def test_discovery_preserves_the_frozen_ordered_export_contract_without_resolution(): + package = importlib.import_module(_PACKAGE) + imports_before = _unrelated_modules() + aliases_before = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + + discovered = dir(package) + + assert len(package.__all__) == 456 + assert len(set(package.__all__)) == 455 + assert package.__all__.count("ladder_vs_chronicle_household_dispersion") == 2 + encoded = json.dumps(package.__all__, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == _ORDERED_ALL_SHA256 + assert set(package.__all__).issubset(discovered) + assert discovered == sorted(set(discovered)) + aliases_after = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + assert aliases_after.keys() == aliases_before.keys() + assert all(aliases_after[name] is value for name, value in aliases_before.items()) + assert _unrelated_modules() == imports_before + with pytest.raises(TypeError): + package._EXPORTS["invented_missing_export"] = ("invented", "missing") + + +def test_unknown_attribute_and_unknown_from_import_keep_normal_failures(): + package = importlib.import_module(_PACKAGE) + with pytest.raises( + AttributeError, match="has no attribute 'invented_missing_export'" + ): + _ = package.invented_missing_export + with pytest.raises( + ImportError, match="cannot import name 'invented_missing_export'" + ): + from microcosm.build.uk_runtime import invented_missing_export # noqa: F401 + + assert "invented_missing_export" not in vars(package) + + +def test_reload_clears_public_aliases_and_rebinds_the_same_defining_objects(): + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + real_function = helper.assign_household_geography + assert package.assign_household_geography is real_function + assert "assign_household_geography" in vars(package) + imports_before = _unrelated_modules() + + assert importlib.reload(package) is package + + assert "assign_household_geography" not in vars(package) + assert package.rowwise_geography is helper + assert sys.modules[f"{_PACKAGE}.rowwise_geography"] is helper + assert package.assign_household_geography is real_function + assert vars(package)["assign_household_geography"] is real_function + assert _unrelated_modules() == imports_before From 162869d4d350b06c263b2fbdbeea93b7f1b05050 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 13:19:30 -0400 Subject: [PATCH 066/255] Align seed diagnostic with approved current lockfile --- .../893-diagnostic-current-lock.fixed.md | 1 + .../spec-seed-current-lock-20260910.json | 22 +++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 changelog.d/893-diagnostic-current-lock.fixed.md create mode 100644 experiments/spec-seed-current-lock-20260910.json diff --git a/changelog.d/893-diagnostic-current-lock.fixed.md b/changelog.d/893-diagnostic-current-lock.fixed.md new file mode 100644 index 000000000..cff257d2e --- /dev/null +++ b/changelog.d/893-diagnostic-current-lock.fixed.md @@ -0,0 +1 @@ +Align the source/seed diagnostic with the lockfile already approved for the primary QRF worker after the UK openpyxl dependency declaration. diff --git a/experiments/spec-seed-current-lock-20260910.json b/experiments/spec-seed-current-lock-20260910.json new file mode 100644 index 000000000..0005829fa --- /dev/null +++ b/experiments/spec-seed-current-lock-20260910.json @@ -0,0 +1,22 @@ +{ + "approved_lock_sha256": "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9", + "before_lock_sha256": "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b", + "ci_job": "https://github.com/PolicyEngine/microcosm/actions/runs/34506624007/job/102970618835", + "data_or_release_acceptance": false, + "diagnostic_rerun": "pending CI on corrected source", + "diagnostic_sha256": "18468f5319324169896942230ca3584188c9840fff03e913aaf16e8360adf4ef", + "local_subject_execution": false, + "matches_existing_worker_approval": true, + "observed_refusal": { + "code": "LOCK", + "completed": false, + "coverage_pass": false, + "phase": "environment" + }, + "only_lock_delta": "Declare existing openpyxl package in microcosm-build[uk] dependency and requires-dist; no new resolved package or version change", + "package_count_before_after": 125, + "resolved_package_versions_unchanged": true, + "scope": "Source-only audit of current-lock diagnostic correction", + "source_change": "One literal, retaining exact lock equality; no allowance for arbitrary locks", + "upstream_worker_approval_commit": "0d978f20bf4d1d899b707fe7d33ab084c153ee4f" +} diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 291cca3ca..7b1616c3a 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -20,7 +20,7 @@ from functools import cached_property from pathlib import Path -LOCK_SHA256 = "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b" +LOCK_SHA256 = "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9" UPLOAD_ACTION_SHA = "ea165f8d65b6e75b540449e92b4886f43607fa02" CAPS = { "candidate-digests.json": 64 * 1024, From b591280674db6d606c7ab356cabffbb6ef582e2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 13:55:59 -0400 Subject: [PATCH 067/255] Verify full and pruned US exports against their common parent --- .../common-frame-export-contract.added.md | 1 + docs/us-launch-review.md | 11 + docs/us-uk-release-path.md | 2 +- .../us-common-frame-export36-20260910.json | 54 +++ .../common_frame_export_contract.py | 234 ++++++++++ .../test_us_common_frame_export_contract.py | 405 ++++++++++++++++++ 6 files changed, 706 insertions(+), 1 deletion(-) create mode 100644 changelog.d/common-frame-export-contract.added.md create mode 100644 experiments/us-common-frame-export36-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py create mode 100644 packages/microcosm-build/tests/test_us_common_frame_export_contract.py diff --git a/changelog.d/common-frame-export-contract.added.md b/changelog.d/common-frame-export-contract.added.md new file mode 100644 index 000000000..ea5fadddb --- /dev/null +++ b/changelog.d/common-frame-export-contract.added.md @@ -0,0 +1 @@ +Add supplied-parent comparison for US full, pruned and local frame exports, preserving retained inputs and missingness through checkpoint readback. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index de1f7a64b..7b726edf5 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -38,6 +38,17 @@ to zero. A pruned export must not be required to make every survey origin independently resemble a national population, or to rerun imputation to satisfy such a requirement. +The supplied-parent export comparison now passes 36 focused checks, including +actual checkpoint write/readback, complete household selection, unchanged input +values and missingness, signed and unsigned stable IDs, and full/pruned/local +weight and scope handling. A household with zero original weight can receive +positive calibrated weight without mutating its parent or other entity weights. +The comparison binds the full ordered weight vector and specification, including +rows outside the selected scope. It deliberately reports calibration ancestry +and release eligibility as unverified: the owning build must establish those +and seal its inputs across export I/O. See the +[scoped export acceptance](../experiments/us-common-frame-export36-20260910.json). + The earlier ten-node survey prefix assigns a block before cloning, and its thirteen-node age-calibration extension passes cold execution and required replay on invented originals. A twenty-node extension now carries atomic diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 6420b4786..1db721cd6 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -46,7 +46,7 @@ common source population alone does not guarantee identical estimates. | Geography | Earlier pre-clone block assignment and inheritance pass in the small real build; corrected post-clone order and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | -| Full/compact files | Small prefix export and required replay pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | | Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | The US new graph has passed a small real survey → block assignment → clone → diff --git a/experiments/us-common-frame-export36-20260910.json b/experiments/us-common-frame-export36-20260910.json new file mode 100644 index 000000000..55818bd4e --- /dev/null +++ b/experiments/us-common-frame-export36-20260910.json @@ -0,0 +1,54 @@ +{ + "adoption_formatting": { + "full_ast_unchanged": true, + "test_source_after_sha256": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c", + "test_source_before_sha256": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0", + "tool": "ruff format" + }, + "component": "US supplied-parent full/pruned/local export comparison", + "files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0" + }, + "guard_sha256": "f8d8184e582a443ccae781342df2b8c0accf4c87c552ccaed843f1bad0586bd1", + "independent_review": { + "correctness_defects": 0, + "coverage_note": "Both new prune-flag cases select the same retained rows; existing full/pruned/local cases exercise differing selections.", + "sha256": "8ef45f1d1fd5b3790e9d15d42d4dcb98651c1f66e5badb059134df623c91d17b" + }, + "limits": { + "cpu_seconds": 600, + "numeric_threads": 1, + "wall_seconds": 900 + }, + "physical_postcheck_sha256": "486eb981a71ca13806ed2524cc7a7b9469751ff47d2496e85b566044b7afc0a8", + "prior_attempts": "Preserved original all-zero fixture failure, unexecuted34 packet, and two36 wrapper failures. Final36v3 corrected invocation serialization and passed all checks.", + "scope": "Invented frames and real frame-checkpoint write/readback. No native candidate, actual calibration ancestry or release is certified.", + "source_closure_parity": { + "new_files": 2, + "pyproject_difference": "Comment describing UK NISRA openpyxl use; dependency configuration unchanged", + "pyproject_toml_semantics_equal": true, + "unchanged_python_files": 57 + }, + "source_map_sha256": "d6bb7e1404569ec27c5231b37d92397399fca7f63b85744ff393dc630454215e", + "source_revision_before_adoption": "162869d4d350b06c263b2fbdbeea93b7f1b05050", + "validation": { + "actual_graph_calibration_ancestry_verified": false, + "cpu_seconds": 1.769343, + "model_sources": 0, + "owned_files": 11, + "peak_rss_bytes": 324517888, + "physical_postcheck": "pass", + "receipt_sha256": "2b6de385041b6bb5d59ff34a2672d215de85bf0d058e6853f05a7afaf05b84f9", + "release_eligible": false, + "resources": 0, + "reviewer": "root", + "source_files": 60, + "suite_accepted": true, + "tests_failed": 0, + "tests_passed": 36, + "unexpected_refusals": {}, + "wall_seconds": 2.5767143329721875, + "xml_sha256": "3ab7cc10509aad988dd767ce39425138e9c8a67b171fcb433387057c2c1d28cb" + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py new file mode 100644 index 000000000..f7671534f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py @@ -0,0 +1,234 @@ +"""Pure export comparison against a supplied common parent; no source issuer. + +The host must authenticate the actual enriched Population and calibration +ancestry, including targets, ordered weights and geographic scope. A supplied +reference, specification or returned digest cannot establish that ancestry. +The host must also seal those inputs across writer/readback I/O and finish its +owner checks before accepting an export. This module performs no I/O or solve. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChange, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import storage_equal + +PROTOCOL = "microcosm.us.supplied-parent-export-comparison.v1" +MAX_SPECIFICATION_BYTES = 1024 * 1024 + + +class RetainedFrameExportError(ValueError): + """An export differs from its supplied parent or declared weight/scope inputs.""" + + +def _require(condition, code): + if not condition: + raise RetainedFrameExportError(code) + + +def _vector(value, dtype, code): + _require( + type(value) is np.ndarray and value.ndim == 1 and value.dtype == dtype, + code, + ) + return value + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _array_binding(value): + _require(value.dtype.kind in ("i", "u", "f"), "NUMERICAL_BINDING_DTYPE") + return { + "dtype": value.dtype.str, + "rows": len(value), + "sha256": _sha(np.ascontiguousarray(value).tobytes()), + } + + +def _same_series(expected, actual, *, readback): + """Allow only masked-cell codec canonicalization on logical readback.""" + if expected.dtype != actual.dtype or len(expected) != len(actual): + return False + expected_null = expected.isna().to_numpy(dtype=np.bool_) + actual_null = actual.isna().to_numpy(dtype=np.bool_) + if not np.array_equal(expected_null, actual_null): + return False + if readback and expected.dtype == object: + # Checkpoint supports object-string strata. Raw object pointers cannot + # identify values across a serialization boundary. Arbitrary object + # equality is outside this small numerical/string contract. + left = expected.to_numpy()[~expected_null].tolist() + right = actual.to_numpy()[~actual_null].tolist() + return all(type(v) is str for v in (*left, *right)) and left == right + return storage_equal(expected, actual, ~expected_null if readback else None) + + +def verify_retained_frame_export( + parent: Frame, + candidate: Frame, + *, + parent_reference: str, + ordered_household_ids: np.ndarray, + calibrated_weights: np.ndarray, + calibration_specification: bytes, + scope_household_ids: np.ndarray | None = None, + prune_zero_weight: bool = True, + comparison: str = "storage", + expected_binding: bytes | None = None, +) -> bytes: + """Compare complete retained inputs and bind supplied numerical export inputs. + + IDs retain the parent's exact signed/unsigned integer dtype; weights are + exact float64 arrays in the *whole parent* + household order. Scope IDs must be a unique subset in that same order; + None explicitly denotes all households. Pruning removes exactly zero + weights, with no tolerance or default origin-quality requirement. Household + selection carries all its persons and precisely their referenced groups. + + Call before writing, then on the serializer's logical readback with + comparison="frame-checkpoint-readback" and the prior expected_binding. + That closed mode allows only existing + codec normalization under null masks; known values, dtype and masks remain + exact. Extra or missing columns and record reordering refuse. Pandas row + indices may be reset: entity IDs, not incidental indices, define alignment. + + No Population ownership, design anchor, mass ledger, target interpretation, + geographic predicate meaning or source authenticity is established here. + The host must bind parent_reference to actual graph ancestry and verify the + specification/scope against that ancestry; opaque bytes are not authority. + Empty support, scopes with no positive weight, and experimental separate + link tables refuse explicitly, matching the Frame weight contract. + """ + _require(isinstance(parent, Frame) and isinstance(candidate, Frame), "FRAME_TYPE") + _require( + type(parent_reference) is str and bool(parent_reference), "PARENT_REFERENCE" + ) + _require( + type(calibration_specification) is bytes + and 0 < len(calibration_specification) <= MAX_SPECIFICATION_BYTES, + "CALIBRATION_SPECIFICATION", + ) + _require(type(prune_zero_weight) is bool, "OPTIONS") + _require( + type(comparison) is str + and comparison in ("storage", "frame-checkpoint-readback"), + "COMPARISON_MODE", + ) + readback = comparison == "frame-checkpoint-readback" + _require( + expected_binding is None or type(expected_binding) is bytes, "BINDING_TYPE" + ) + _require(parent.schema == candidate.schema, "SCHEMA") + _require( + not parent.schema.links and parent.links == candidate.links == (), + "SEPARATE_LINK_TABLES_UNSUPPORTED", + ) + _require("household" in parent.schema.group_entities, "HOUSEHOLD_SCHEMA") + _require( + "household" in parent.weighted_entities + and parent.weighted_entities == candidate.weighted_entities, + "WEIGHT_TOPOLOGY", + ) + _require(parent.metadata == candidate.metadata, "FRAME_METADATA") + household_id = parent.schema.entity_id_column("household") + parent_ids = parent.table("household")[household_id].to_numpy() + _require(parent_ids.dtype.kind in ("i", "u"), "INTEGER_HOUSEHOLD_IDS") + ids = _vector(ordered_household_ids, parent_ids.dtype, "HOUSEHOLD_ID_VECTOR") + weights = _vector(calibrated_weights, np.dtype("float64"), "WEIGHT_VECTOR") + _require( + parent_ids.dtype == ids.dtype + and np.array_equal(parent_ids, ids) + and len(np.unique(ids)) == len(ids), + "COMPLETE_ORDERED_HOUSEHOLD_IDS", + ) + _require( + weights.shape == ids.shape + and np.isfinite(weights).all() + and (weights >= 0).all(), + "FINITE_ALIGNED_WEIGHTS", + ) + if scope_household_ids is None: + scope = ids + scope_mask = np.ones(len(ids), dtype=np.bool_) + else: + scope = _vector(scope_household_ids, ids.dtype, "SCOPE_ID_VECTOR") + scope_mask = np.isin(ids, scope) + _require(np.array_equal(ids[scope_mask], scope), "ORDERED_SCOPE_SUBSET") + keep = scope_mask & (weights > 0) if prune_zero_weight else scope_mask + _require(keep.any() and (weights[keep] > 0).any(), "EMPTY_EXPORT_SUPPORT") + membership = parent.schema.membership_column("household") + comparison_parent = parent.with_weights( + "household", + Weights(weights, WeightKind.CALIBRATED), + mass=MassChange( + factor=None, + reason="Temporary supplied-weight view for retained export comparison", + ), + ) + selected = comparison_parent.select( + np.isin(parent.person[membership].to_numpy(), ids[keep]) + ) + _require(selected.entities == candidate.entities, "ENTITY_ROSTER") + entity_ids = {} + for entity in selected.entities: + expected, actual = selected.table(entity), candidate.table(entity) + _require( + not actual.columns.has_duplicates + and tuple(expected.columns) == tuple(actual.columns), + "COLUMN_ROSTER:" + entity, + ) + id_column = parent.schema.entity_id_column(entity) + _require( + _same_series(expected[id_column], actual[id_column], readback=False), + "RETAINED_ENTITY_IDS:" + entity, + ) + for name in expected.columns: + _require( + _same_series(expected[name], actual[name], readback=readback), + "RETAINED_INPUT:" + entity + "." + str(name), + ) + entity_ids[entity] = _array_binding(expected[id_column].to_numpy()) + _require( + selected.strata.name == candidate.strata.name + and _same_series(selected.strata, candidate.strata, readback=readback), + "RETAINED_STRATA", + ) + for entity in selected.weighted_entities: + actual = candidate.weights_for(entity) + if entity == "household": + expected_values, expected_kind = weights[keep], WeightKind.CALIBRATED + else: + expected_weight = selected.weights_for(entity) + expected_values, expected_kind = ( + expected_weight.values, + expected_weight.kind, + ) + _require( + actual.kind is expected_kind + and storage_equal(pd.Series(expected_values), pd.Series(actual.values)), + "RETAINED_WEIGHTS:" + entity, + ) + binding = canonical_json( + { + "protocol": PROTOCOL, + "scope": "comparison_against_supplied_parent_only", + "parent_reference": parent_reference, + "actual_graph_calibration_ancestry_verified": False, + "release_eligible": False, + "calibration_specification_sha256": _sha(calibration_specification), + "complete_ordered_household_ids": _array_binding(ids), + "complete_calibrated_weights": _array_binding(weights), + "scope_household_ids": _array_binding(scope), + "prune_zero_weight": prune_zero_weight, + "retained_entity_ids": entity_ids, + } + ) + _require(expected_binding is None or binding == expected_binding, "EXPORT_BINDING") + return binding diff --git a/packages/microcosm-build/tests/test_us_common_frame_export_contract.py b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py new file mode 100644 index 000000000..349e78527 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py @@ -0,0 +1,405 @@ +"""Invented supplied-parent comparisons; no calibration or source issuer imitation.""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import frame_checkpoint +from microcosm.build.us_runtime import common_frame_export_contract as contract +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _parent(): + person = pd.DataFrame( + { + "person_id": np.arange(1, 7, dtype=np.int64), + "person_household_id": np.array([10, 10, 20, 20, 30, 30], dtype=np.int64), + "person_tax_unit_id": np.array( + [100, 100, 200, 201, 300, 300], dtype=np.int64 + ), + "money": np.array([-0.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + "hours": pd.Series([10, pd.NA, 30, 40, pd.NA, 60], dtype="Int32"), + "reported": pd.Series( + [True, pd.NA, False, True, pd.NA, False], dtype="boolean" + ), + "source_person_id": np.arange(101, 107, dtype=np.int64), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([10, 20, 30], dtype=np.int64), + "origin": pd.Series(["asec", "acs", "acs"], dtype="string"), + "assigned_block": pd.Series(["001", "002", "003"], dtype="string"), + "county": pd.Series(["A", "A", "B"], dtype="string"), + "clone": np.array([0, 1, 0], dtype=np.int8), + } + ) + tax = pd.DataFrame( + { + "tax_unit_id": np.array([100, 200, 201, 300], dtype=np.int64), + "deduction": np.array([1.0, 2.0, 3.0, 4.0]), + } + ) + return Frame( + {"person": person, "household": household, "tax_unit": tax}, + EntitySchema(group_entities=("household", "tax_unit")), + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.IMPORTANCE)}, + pd.Series(["a", "a", "b", "b", "b", "b"], name="stratum"), + metadata={"construction": "one-common-parent"}, + ) + + +def _weighted(parent, person_mask, weights): + selected = parent.select(np.asarray(person_mask, dtype=np.bool_)) + return Frame( + {e: selected.table(e).copy(deep=True) for e in selected.entities}, + selected.schema, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.CALIBRATED + ) + }, + selected.strata.copy(), + metadata=selected.metadata, + ) + + +def _arguments(parent): + return { + "parent_reference": "supplied-parent-example-not-an-issued-graph-key", + "ordered_household_ids": parent.table("household") + .household_id.to_numpy() + .copy(), + "calibrated_weights": np.array([0.0, 9.0, 10.0]), + "calibration_specification": b'{"targets":"invented","scope":"national"}', + } + + +@pytest.mark.parametrize("view", ["full", "pruned", "local"]) +def test_one_parent_full_pruned_and_local_preserve_all_inputs(view): + parent = _parent() + args = _arguments(parent) + if view == "full": + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args["prune_zero_weight"] = False + elif view == "pruned": + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + else: + candidate = _weighted(parent, [False, False, False, False, True, True], [10]) + args["scope_household_ids"] = np.array([30], dtype=np.int64) + binding = contract.verify_retained_frame_export(parent, candidate, **args) + report = json.loads(binding) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["retained_entity_ids"]["person"]["rows"] == candidate.n("person") + if view != "full": + # A complete survey origin may legitimately have zero calibrated weight. + assert candidate.table("household").origin.tolist() == ["acs"] * candidate.n( + "household" + ) + assert ( + contract.verify_retained_frame_export( + parent, candidate, expected_binding=binding, **args + ) + == binding + ) + + +def test_actual_checkpoint_readback_compares_retained_values_and_masks(tmp_path): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + path = tmp_path / "invented-export.h5" + frame_checkpoint.write_frame_checkpoint( + path, candidate, metadata={"comparison": before.decode()} + ) + loaded = frame_checkpoint.load_frame_checkpoint( + path, frame_metadata=parent.metadata + ) + assert loaded.metadata["comparison"] == before.decode() + assert ( + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + loaded.frame.person.loc[loaded.frame.person.index[0], "money"] += 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.money" + ): + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + + +@pytest.mark.parametrize( + "change", + [ + "value", + "mask", + "membership", + "id", + "geography", + "lineage", + "weight", + "strata", + "column", + ], +) +def test_retained_input_changes_refuse(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + if change == "value": + candidate.person.loc[2, "money"] += 1 + elif change == "mask": + candidate.person.loc[4, "hours"] = 0 + elif change == "membership": + candidate.person.loc[2, "person_household_id"] = 30 + elif change == "id": + candidate.person.loc[2, "person_id"] = 99 + elif change == "geography": + candidate.table("household").loc[0, "assigned_block"] = "004" + elif change == "lineage": + candidate.person.loc[2, "source_person_id"] = 999 + elif change == "weight": + candidate = _weighted(parent, [False, False, True, True, True, True], [8, 10]) + elif change == "strata": + candidate.strata.iloc[0] = "changed" + else: + candidate.person.drop(columns="money", inplace=True) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_|COLUMN_ROSTER" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +def test_partial_household_selection_refuses_even_with_valid_frame_links(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, False, True, True], [9, 10]) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_ENTITY_IDS:person" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +@pytest.mark.parametrize( + "change", + ["order", "dtype", "nan", "negative", "shape", "scope_order", "scope_unknown"], +) +def test_weight_and_scope_axes_are_explicit(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + if change == "order": + args["ordered_household_ids"] = args["ordered_household_ids"][::-1] + elif change == "dtype": + args["ordered_household_ids"] = args["ordered_household_ids"].astype(float) + elif change == "nan": + args["calibrated_weights"][0] = np.nan + elif change == "negative": + args["calibrated_weights"][0] = -1 + elif change == "shape": + args["calibrated_weights"] = args["calibrated_weights"][:2] + elif change == "scope_order": + args["scope_household_ids"] = np.array([30, 20], dtype=np.int64) + else: + args["scope_household_ids"] = np.array([99], dtype=np.int64) + with pytest.raises(contract.RetainedFrameExportError): + contract.verify_retained_frame_export(parent, candidate, **args) + + +@pytest.mark.parametrize( + "change", ["specification", "parent_reference", "excluded_weight", "scope", "prune"] +) +def test_readback_binding_covers_even_inputs_outside_retained_rows(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + args["scope_household_ids"] = np.array([20, 30], dtype=np.int64) + before = contract.verify_retained_frame_export(parent, candidate, **args) + if change == "specification": + args["calibration_specification"] = b'{"targets":"different"}' + elif change == "parent_reference": + args["parent_reference"] = "another-unverified-reference" + elif change == "excluded_weight": + args["calibrated_weights"][0] = 100 + elif change == "scope": + args["scope_household_ids"] = None + else: + args["prune_zero_weight"] = False + with pytest.raises(contract.RetainedFrameExportError, match="EXPORT_BINDING"): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + + +@pytest.mark.parametrize("scope", ["empty", "all_zero"]) +def test_empty_analysis_is_an_explicit_unsupported_frame_contract(scope): + parent = _parent() + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args = _arguments(parent) + if scope == "empty": + args["scope_household_ids"] = np.array([], dtype=np.int64) + else: + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export(parent, candidate, **args) + + +def test_uint64_stable_ids_above_int64_are_not_coerced(): + ids = np.array([2**63 + 1, 2**63 + 7], dtype=np.uint64) + parent = Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1, 2], dtype=np.int32), + "person_household_id": ids, + } + ), + "household": pd.DataFrame({"household_id": ids}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(2), WeightKind.IMPORTANCE)}, + ) + candidate = _weighted(parent, [True, True], [2, 3]) + args = { + "parent_reference": "large-ids", + "ordered_household_ids": ids, + "calibrated_weights": np.array([2.0, 3.0]), + "calibration_specification": b"{}", + } + report = json.loads( + contract.verify_retained_frame_export(parent, candidate, **args) + ) + assert report["complete_ordered_household_ids"]["dtype"] == ids.dtype.str + with pytest.raises(contract.RetainedFrameExportError, match="HOUSEHOLD_ID_VECTOR"): + contract.verify_retained_frame_export( + parent, candidate, **{**args, "ordered_household_ids": ids.astype(float)} + ) + + +def test_readback_normalization_is_closed_and_never_changes_knownness(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + values = candidate.person["hours"].array + assert values._mask[2] + values._data[2] ^= np.int32(1) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + assert ( + contract.verify_retained_frame_export( + parent, + candidate, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + with pytest.raises(contract.RetainedFrameExportError, match="COMPARISON_MODE"): + contract.verify_retained_frame_export( + parent, candidate, comparison="approximate", **args + ) + values._mask[2] = False + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, comparison="frame-checkpoint-readback", **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +@pytest.mark.parametrize("scope", ["all", "zero_weight_household"]) +def test_zero_weight_export_support_refuses(prune_zero_weight, scope): + parent = _parent() + # Keep a valid candidate so the export contract itself checks the supplied + # weight/scope inputs before comparing retained rows or weight storage. + candidate = _weighted(parent, [True] * 6, [1, 1, 1]) + args = _arguments(parent) + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + if scope == "zero_weight_household": + args["calibrated_weights"][1:] = [9, 10] + args["scope_household_ids"] = args["ordered_household_ids"][:1].copy() + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export( + parent, candidate, prune_zero_weight=prune_zero_weight, **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +def test_positive_calibrated_scope_can_have_zero_parent_weight(prune_zero_weight): + baseline = _parent() + parent = Frame( + {e: baseline.table(e).copy(deep=True) for e in baseline.entities}, + baseline.schema, + { + "household": Weights(np.array([0.0, 2.0, 3.0]), WeightKind.IMPORTANCE), + "tax_unit": Weights(np.array([7.0, 8.0, 9.0, 10.0]), WeightKind.DESIGN), + }, + baseline.strata.copy(), + metadata=baseline.metadata, + ) + before_tables = {e: parent.table(e).copy(deep=True) for e in parent.entities} + before_strata = parent.strata.copy(deep=True) + before_metadata, before_mass_log = parent.metadata, parent.mass_log + before_weights = { + e: (parent.weights_for(e).values.copy(), parent.weights_for(e).kind) + for e in parent.weighted_entities + } + retained = baseline.select(np.array([True, True, False, False, False, False])) + + def candidate(tax_weight): + return Frame( + {e: retained.table(e).copy(deep=True) for e in retained.entities}, + retained.schema, + { + "household": Weights(np.array([4.0]), WeightKind.CALIBRATED), + "tax_unit": Weights(np.array([tax_weight]), WeightKind.DESIGN), + }, + retained.strata.copy(), + metadata=retained.metadata, + ) + + args = _arguments(parent) + args["calibrated_weights"] = np.array([4.0, 0.0, 0.0]) + args["scope_household_ids"] = np.array([10], dtype=np.int64) + report = json.loads( + contract.verify_retained_frame_export( + parent, candidate(7.0), prune_zero_weight=prune_zero_weight, **args + ) + ) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["complete_ordered_household_ids"]["rows"] == 3 + assert report["retained_entity_ids"]["household"]["rows"] == 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_WEIGHTS:tax_unit" + ): + contract.verify_retained_frame_export( + parent, candidate(8.0), prune_zero_weight=prune_zero_weight, **args + ) + for entity, table in before_tables.items(): + pd.testing.assert_frame_equal(parent.table(entity), table, check_exact=True) + pd.testing.assert_series_equal(parent.strata, before_strata, check_exact=True) + assert parent.metadata == before_metadata and parent.mass_log == before_mass_log + for entity, (values, kind) in before_weights.items(): + assert parent.weights_for(entity).kind is kind + assert parent.weights_for(entity).values.tobytes() == values.tobytes() From 94b7c2a5d73c1512efcc41425224bfbb7a04cbb7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 14:13:45 -0400 Subject: [PATCH 068/255] Record independent export review and adopted source hashes --- .../us-common-frame-export36-20260910.json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/experiments/us-common-frame-export36-20260910.json b/experiments/us-common-frame-export36-20260910.json index 55818bd4e..10c567b06 100644 --- a/experiments/us-common-frame-export36-20260910.json +++ b/experiments/us-common-frame-export36-20260910.json @@ -6,9 +6,18 @@ "tool": "ruff format" }, "component": "US supplied-parent full/pruned/local export comparison", + "fable_source_review": { + "blocking_findings": 0, + "model": "Fable 5.1", + "record_note": "files identifies adopted bytes; tested_files identifies exact executed bytes. Full AST equality after Ruff formatting is recorded above.", + "reviewed_commit": "b591280674db6d606c7ab356cabffbb6ef582e2d", + "scope": "Six-file source review; no runtime or independent hash recomputation by the peer", + "sha256": "361eb8abccc7b227c7c2aef2e1704e807076b6481ce73e501839adf358143416", + "verdict": "clean" + }, "files": { "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", - "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0" + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c" }, "guard_sha256": "f8d8184e582a443ccae781342df2b8c0accf4c87c552ccaed843f1bad0586bd1", "independent_review": { @@ -32,6 +41,10 @@ }, "source_map_sha256": "d6bb7e1404569ec27c5231b37d92397399fca7f63b85744ff393dc630454215e", "source_revision_before_adoption": "162869d4d350b06c263b2fbdbeea93b7f1b05050", + "tested_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0" + }, "validation": { "actual_graph_calibration_ancestry_verified": false, "cpu_seconds": 1.769343, From 1cebe3547543bf2917a7af7362bfbb22a05bf964 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 15:10:55 -0400 Subject: [PATCH 069/255] Assign atomic geography after initial survey cloning --- docs/geography-assignment.md | 32 +-- docs/us-launch-review.md | 42 +++- docs/us-uk-release-path.md | 12 +- ...tclone-geography-70-controls-20260910.json | 197 ++++++++++++++++++ .../src/microcosm/build/atomic_geography.py | 7 +- .../microcosm/build/graph_atomic_geography.py | 30 ++- .../build/us_runtime/atomic_block_support.py | 5 +- .../us_runtime/current_survey_predictors.py | 5 + .../us_runtime/graph_atomic_survey_clone.py | 46 ++-- .../graph_atomic_survey_financial.py | 37 ++-- .../graph_atomic_survey_population.py | 167 ++++----------- .../graph_current_survey_predictors.py | 47 ++++- .../us_runtime/survey_age_calibration.py | 50 +++-- .../us_runtime/survey_atomic_geography.py | 136 ++++++++++-- .../build/us_runtime/survey_origin_budget.py | 105 +++++++--- .../tests/test_atomic_geography.py | 74 ++++++- .../test_us_atomic_survey_clone_graph.py | 44 ++-- .../test_us_graph_atomic_survey_financial.py | 71 ++++++- .../test_us_graph_atomic_survey_population.py | 180 +++++++++------- .../test_us_postclone_geography_identity.py | 129 ++++++++++++ ...survey_age_calibration_atomic_geography.py | 8 +- ...s_survey_origin_budget_atomic_geography.py | 119 ++++++++++- 22 files changed, 1191 insertions(+), 352 deletions(-) create mode 100644 experiments/us-postclone-geography-70-controls-20260910.json create mode 100644 packages/microcosm-build/tests/test_us_postclone_geography_identity.py diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md index cca500c02..5793b0b1f 100644 --- a/docs/geography-assignment.md +++ b/docs/geography-assignment.md @@ -12,8 +12,16 @@ historical evidence. They do not demonstrate this corrected ordering. Geographic assignment must use a stable post-clone identity, including a clone discriminator where original-source identity alone is shared by multiple households. -This document records the intended contract and the implementation gap. It does -not certify a new geographic source, assignment run or population file. +The corrected order now passes 70 controls on invented inputs, plus a separate +default-path compatibility check. The nine-node survey prefix expands the initial +clones before assignment; its nineteen-node financial and twelve-node age +extensions retain the assigned geography. Draw identity combines +`survey_geography_origin_key` with `household_support_clone_index`, so renumbering +households does not redraw their location. A typed validation artifact orders +financial enrichment after the geography gate. See the +[scoped acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). +Native execution of this corrected order and population-quality acceptance +remain pending. The older results below retain their original source scope. ## Declarative country capability @@ -155,9 +163,9 @@ The US development composed graph attaches geography after harmonization, but `us_runtime/graph_geography.py` selects a joint tract/congressional-district cell and emits PUMA, county and district. It does not assign a Census block. Its prior national source and joint-support checks therefore do not establish block-first -acceptance. The current survey/clone graph needs an explicit block assignment -and derivation connection before enrichment. Block support and current district -mapping require source review before acquisition or execution. +acceptance. The separate atomic survey graph now provides that block assignment +and derivation connection before enrichment. Its population-only national block +support and source review are recorded below. The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. Its sampler selects a constituency within FRS region using household counts, @@ -182,10 +190,10 @@ household counts or support for subsequent construction in unpopulated 2020 blocks. All 13 invented tests passed, including missing/inconsistent mapping refusals and subset-stable assignment. Receipt SHA256: `404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. -Actual normalized block support and attachment before the survey clone remain -pending; these tests do not establish native geographic fit. +These initial adapter tests do not establish native geographic fit. Subsequent +national support and native pilot results are recorded below. -`us_runtime/graph_atomic_survey_clone.py` now declares the combined sequence: +The earlier `us_runtime/graph_atomic_survey_clone.py` declared this sequence: shared import, assignment, derivation and integrity gate, then the existing combined-survey support clone and an inherited-mapping gate. The graph compiler makes the clone depend on every member of its base version, including the @@ -205,7 +213,7 @@ through the actual source issuers. Their receipts are recorded in `experiments/us-survey-geography-source-controls-20260909.json` and `experiments/us-survey-geography-graph-controls-20260909.json`. -`us_runtime/graph_atomic_survey_population.py` connects that observed projection +The earlier `us_runtime/graph_atomic_survey_population.py` connected that projection to block assignment, geographic derivation, the integrity gate and the combined survey clone. It retains the raw allocation separately from the enriched pre-clone population, even though both share the allocation version identifier. @@ -215,9 +223,9 @@ refusals. The test population contains six survey households before cloning and twelve afterward. The exact tested revisions and remaining checks are recorded in `experiments/us-atomic-survey-population-controls-20260909.json`. -The calibration budget and age-calibration runner now consume this optional -prefix and independently reconstruct it from the raw allocation. Eight budget -controls pass, as does the complete thirteen-node age graph through fresh +The earlier calibration budget and age runner consumed this optional prefix +and independently reconstructed it from the raw allocation. Eight budget +controls passed, as did the complete thirteen-node age graph through fresh execution and required replay. A separate compatibility control verifies that the existing predictor and PUF-host qualifiers still accept the default survey prefix. See the corresponding `us-atomic-budget-semantic-8`, diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md index 7b726edf5..8914d6bea 100644 --- a/docs/us-launch-review.md +++ b/docs/us-launch-review.md @@ -59,10 +59,29 @@ operator is a separate path. See [Geography assignment](geography-assignment.md) for the country contracts, implementation and source boundaries. -The 10 September sequencing correction requires the full initial multispine and -its clones before block assignment, allowing each resulting household its own -constrained draw. The accepted ten/twenty-node runs retain their original order; -the corrected order needs its own implementation and replay checks. +The 10 September sequencing correction now passes 70 inventoried controls on +invented inputs, across seven separately guarded runs, plus one separate default +compatibility case. The nine-node prefix completes its initial clones before +block assignment; the nineteen-node financial extension and twelve-node age +extension retain that geography through cold execution and required replay. +Assignment uses the qualified original-source key and stable clone discriminator, +so changing row order or numeric household coordinates does not redraw the same +clone. Observed state/PUMA constraints, input knownness, entity links, complete +population ownership and weights remain checked. Different clones may draw +different blocks; an occasional shared block is valid. + +The financial donor FILTER requires the exact typed geography-validation artifact +and retains original ASEC DESIGN weights. The shared gate emits this artifact +only when requested, preserving its default API for UK consumers. Two test-only +corrections repair the CREATE source declaration and isolate the obsolete-order +budget counterexample from unrelated serialized storage differences; production +matches the independently reviewed candidate. See the +[scoped postclone acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). + +The earlier ten/twenty-node native runs retain their original order. Native +postclone execution, the new-order PUF host, held-out model quality, national/local +fiscal calibration and full release acceptance remain pending. The age control +uses invented targets and does not establish that complete calibration path. | Area | Main source entry points | What to review | | --- | --- | --- | @@ -71,7 +90,7 @@ the corrected order needs its own implementation and replay checks. | Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | | Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | | Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | -| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment, observed-source constraints, versioned mappings and stable clone identities; moving assignment after cloning remains in progress | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment after complete initial cloning, observed-source constraints, versioned mappings, stable clone identities and the optional typed validation prerequisite | | Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | | Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | @@ -116,12 +135,13 @@ Record shared dependencies as named changes with pinned revisions. Extract a smaller shared prerequisite where useful, while preserving compatibility with Maria's intentional stack. Proposed sequencing does not certify or merge any PR. -The shared post-clone geography interface is under revision following the -10 September ordering correction. Consumers adopting that new path should agree -on its interface before wiring it. Existing target/source repairs can proceed on -their reviewed base. Passing this draft's invented controls does not certify -Maria's UK candidate, and the final US release need not gate her independent -source and calibration work. No whole-PR merge is implied by this review map. +The shared post-clone geography interface now has a tested additive validation +artifact contract. The default gate API remains unchanged; UK registration can +adopt the optional typed prerequisite independently of the US stage ordering. +Existing target/source repairs can proceed on their reviewed base. Passing this +draft's invented controls does not certify Maria's UK candidate, and the final +US release need not gate her independent source and calibration work. No +whole-PR merge is implied by this review map. ## Verification and review scope diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 1db721cd6..ddfff2328 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -43,7 +43,7 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | | Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; complete PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | -| Geography | Earlier pre-clone block assignment and inheritance pass in the small real build; corrected post-clone order and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Geography | Corrected post-clone order passes 70 invented controls plus separate default compatibility; native post-clone execution and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | @@ -51,10 +51,12 @@ common source population alone does not guarantee identical estimates. The US new graph has passed a small real survey → block assignment → clone → financial-enrichment build and required replay. The combined survey population -has 1,584 source households and 3,168 records after cloning. This is not the -complete enriched, calibrated national/CD file. The next critical sequence is -complete PUF/remaining input integration, a small real full build, model and -calibration evaluation, then progressive scale and release verification. See +has 1,584 source households and 3,168 records after cloning. That native result +uses the earlier assignment-before-cloning order. The corrected nine-node prefix +and nineteen-node financial extension now pass their invented controls; the +next native pilot must use their new order and a fresh graph store. The remaining +critical sequence is complete PUF/input integration, a small real full build, +model and calibration evaluation, then progressive scale and release verification. See [the US review guide](us-launch-review.md) for scoped acceptance records. UK work already includes raw-source enrichment, national/local calibration, diff --git a/experiments/us-postclone-geography-70-controls-20260910.json b/experiments/us-postclone-geography-70-controls-20260910.json new file mode 100644 index 000000000..1abe29a64 --- /dev/null +++ b/experiments/us-postclone-geography-70-controls-20260910.json @@ -0,0 +1,197 @@ +{ + "adopted_production_sha256": { + "packages/microcosm-build/src/microcosm/build/atomic_geography.py": "ed20cdec84eebe46fda5ae213522b6b6699812f04cc6699411c83f27df0ac82a", + "packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py": "9972f125566c4b697f8f862d236f6ae8cd3eece43fddb5d82df46552b0723496", + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py": "c70cfb2cf8f7915666846e563fd3d729fcb814244fc4bc3b3292d064fd0fdd40", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "ecb883827ca47c1fb1f2bf3ff25328706d0cc5c187d7914749b3ede4bd3ced31", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py": "9aef169897cb0f91d1414d607c5025e8c9d80bcc84c6eef269a27570aa8c12b0", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "d407086f05311e721e8961707e9471f5abffd74cef100eee2620cc75294492fc", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py": "e8a92fc299d6a388a0ef9ebf26d842460aefda9370006a4b1d929c78983cba0e", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "291285c35b19ff9dfccffc37ac2e768a72021db4196f3f662d86d26626790253", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "f7169ef0631a25acb4ca2fea0b49e105a5ca97d8fe85da2ecb4a5bdd5669cbeb", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py": "d4e80f94ba79c4bad4b47d18fcb5daad9469d45aeb81dea64fba101554080f91", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd" + }, + "adopted_test_sha256": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py": "9bfce65ca47f74a325e652684b7c47fcf4fae1561a465e8f87e7518786bd4efa", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "59adfd2af5225ad7d08aba2a55872546b9f252f61baea5f2647863b0b13a8c03", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py": "c2474f4943a2ccb0d89b77f74e1fa991b293bc977139da2c4f28b8626aed0c37", + "packages/microcosm-build/tests/test_us_postclone_geography_identity.py": "763d15b4d543b1e5b294d20a55169138d1a5d54bac15c11a9e9dc6fa93376c14", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "8dca207eb22b0709358d97c723c2367820e586f443462fd9039372145ed827ab", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + }, + "adoption_exclusions": [ + "PUF nineteen-node predecessor bridge", + "optional ACS source cache/fingerprint overlays", + "split control collectors", + "native guard or payload changes" + ], + "assignment_identity": [ + "survey_geography_origin_key", + "household_support_clone_index" + ], + "compatibility": { + "default_no_geography_case_passed_separately": true, + "default_shared_gate_api_preserved": true, + "optional_typed_gate_artifact": "microcosm.geography.validation_result@1" + }, + "controls": { + "all_original_selected_function_and_fixture_asts_match_adopted_tests": true, + "candidate_cases": 70, + "candidate_split": [ + 45, + 8, + 5, + 5, + 1, + 4, + 2 + ], + "separate_default_compatibility_cases": 1, + "single_combined_suite_run": false + }, + "graph_nodes": { + "age_extension_total": 12, + "financial_extension_total": 19, + "survey_prefix": 9 + }, + "historical_native_note": "Accepted twenty-node cold/replay evidence assigned geography before cloning and remains separately versioned; it does not certify this new ordering.", + "model_quality_accepted": false, + "national_or_local_fiscal_calibration_accepted": false, + "native_full_puf_enrichment_accepted": false, + "native_postclone_survey_accepted": false, + "new_order_puf_host_accepted": false, + "original_candidate_source_map_sha256": "dd1a782dceb020fcf8d9ff5ba94fb32035aa01e8d670ea3529a42a01b26ce9da", + "release_eligible": false, + "runs": [ + { + "cases": 45, + "cpu_seconds": 3.744692, + "included_in_candidate70": true, + "label": "us-postclone-atomic-geography45-v2", + "model_sources": 0, + "physical_source_and_owned": 510, + "postcheck_sha256": "45efe59214d52901a9712304e8ca6f66175c13caaecba3ff773d7780bc2a5ffa", + "receipt_sha256": "f5d1390d9b5183d7f23a2e2492a2d24e06df44bb66cec6e33bc100e74659cfe3", + "resources": 12, + "source_count": 502, + "source_map_sha256": "a294308425861522235f68d025c3c5becf7c2922708ed70543ab54f6a5a15ba4", + "wall_seconds": 4.635587666998617, + "xml_sha256": "8fd2783fc1917170f7d6abcb7ee0c7154a794dcddf326b2c154d5d7b612a2f82" + }, + { + "cases": 8, + "cpu_seconds": 235.803439, + "included_in_candidate70": true, + "label": "us-postclone-prefix-budget8-v3", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "85695bedda8003c9278d2d2ac4fc8d4d6e659898c3f3f5e8d8efcae4b08d74ae", + "receipt_sha256": "3b3fe93a0fe65ba6fd63889afdf99a8f45ba4a37c11aa5e243557fa05352b07c", + "resources": 12, + "source_count": 503, + "source_map_sha256": "71812aeb4d1600223f90882d8652405876c05379acbe0828fbee41b24680d7a1", + "wall_seconds": 237.32238883303944, + "xml_sha256": "caffc1d0eefa51cb0a71338324a213224e67cac50692d1f62d6d61d3cdb328bf" + }, + { + "cases": 5, + "cpu_seconds": 345.624984, + "included_in_candidate70": true, + "label": "us-postclone-financial5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "7e993eccb0085b7dfce203a50d43f868eb822068ab62b50f344f69ce8f478396", + "receipt_sha256": "e3d9e95210185976de73982f994b51386bf4aa1243cc10b4f8db893760e55a90", + "resources": 12, + "source_count": 503, + "source_map_sha256": "02ae2194338378ac652a4ee6c2049b68dd78a813962b665b010c947b5cab0a4b", + "wall_seconds": 352.15383283403935, + "xml_sha256": "224590dab0d79f68b30a8e50ef9e86e9144869d90521cd66c0689d2b21167ff5" + }, + { + "cases": 5, + "cpu_seconds": 203.630023, + "included_in_candidate70": true, + "label": "us-postclone-budget5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "9056592f58e29a73ba8b9ac69314e78602015be112261212911ca19a517e1df0", + "receipt_sha256": "0d577da19dd0086c8d5c47a7fc91101ad708ac51a22c62d8a487f1684c3f1b47", + "resources": 12, + "source_count": 503, + "source_map_sha256": "d316393b1aadf99fefadc342dcd0a88575491665971c8a3908c07b14d7f81f05", + "wall_seconds": 206.07668850000482, + "xml_sha256": "6a189b65471483178fc3518bc68692cf1c88678ae214db5d4407e178394722a1" + }, + { + "cases": 1, + "cpu_seconds": 549.278067, + "included_in_candidate70": true, + "label": "us-postclone-age1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f2c6533c11be9b325e9a507e8356a1c516ca4b20a3f4aecc7721458a8480f9ff", + "receipt_sha256": "e3a513ba6483688fb30b6bd7199b690278f32a02bc49101c78385539447fd699", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b8e4b18994687ea33bead77003ccaffc28d72de7b79d3a34316faa1c7f945b4f", + "wall_seconds": 552.7509316250216, + "xml_sha256": "621b6dff98084c6748eb4a57b3515ef98828ae634bda3362c1504a87c4aba937" + }, + { + "cases": 4, + "cpu_seconds": 438.682175, + "included_in_candidate70": true, + "label": "us-postclone-prefix-late4", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f25adff98c4645dabb503cd155e2652c9988342fca1be108d8b5aea8ee77f90e", + "receipt_sha256": "973fad9735b877c2867bb37af9c60025e10ae7b91216d846b056b5164e39ee33", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b6b9faabf3080fce9cb3c9d1ab9521ba69da615c12d0c2328d75a24f30c976ec", + "wall_seconds": 459.0505850840127, + "xml_sha256": "c0034d745588c5d3821fb73d7086ada5e44c334b2a3e0b52e4ae4116bbfff13d" + }, + { + "cases": 2, + "cpu_seconds": 458.346596, + "included_in_candidate70": true, + "label": "us-postclone-financial-late2", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "2ab808471fc7943f6a3e6a810480ef9a1dddc267a908f0df03fc11311915c37c", + "receipt_sha256": "306c8e250421248faf282a9ac89742a820684b2ba2f6072189426c8301bd5464", + "resources": 12, + "source_count": 503, + "source_map_sha256": "e62aac60cb960e323cbf127067b3c27aa22daf73cdc815e20b55683984ed3ce8", + "wall_seconds": 478.2829617500538, + "xml_sha256": "e303d11996dbe2289c612d5fc102f55da5724f3f259d2cd5de9610686c2d7bb4" + }, + { + "cases": 1, + "cpu_seconds": 28.545443, + "included_in_candidate70": false, + "label": "us-postclone-default1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "157961870ad76565d8c3634a457e4ef8d5ac2245abd75e6b7dc789288cc13fae", + "receipt_sha256": "9c96d01404c8a48fac7b991c2c7d7742d586bdd2b6b754a694e31c451d1bd5ad", + "resources": 12, + "source_count": 503, + "source_map_sha256": "38d7f20d01526ff275c74a8fcd92404b1a572cadade7f896d602f974d275fe2d", + "wall_seconds": 29.92554637498688, + "xml_sha256": "d8da781a858400f3670031566d774d7148b1333cecd6feee543901e68383ec8f" + } + ], + "scope": "Complete initial survey clones before constrained block assignment; current financial and limited age-calibration interfaces on invented source/target fixtures", + "source_adoption_base_head": "94b7c2a5d73c1512efcc41425224bfbb7a04cbb7", + "source_peer_review_sha256": "08c53c53cbd63481279c7c003f536404ec0b2163151b152ff39a6a72b6a3f713", + "status": "invented_postclone_controls_accepted", + "test_only_corrections": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + } +} diff --git a/packages/microcosm-build/src/microcosm/build/atomic_geography.py b/packages/microcosm-build/src/microcosm/build/atomic_geography.py index 81ce87fb9..dc419490f 100644 --- a/packages/microcosm-build/src/microcosm/build/atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/atomic_geography.py @@ -543,9 +543,10 @@ def validate_geography( ) -> dict: """Recheck constraints and functional mappings on a full or pruned population. - Draws are not repeated: enrichment clones have new identities and must inherit - their parent's location. Structural lineage verification remains the executor's - responsibility. This gate alone is not a population-quality certificate. + Draws are not repeated. Initial support clones receive their own assignment + after expansion; subsequent views retain it. Structural lineage verification + remains the executor's responsibility. This gate is not a population-quality + certificate. """ spec = _bind(spec, supports) expected = derive_geography(households, spec, supports) diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py index b507f845d..a245992a8 100644 --- a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -36,6 +36,9 @@ from microcosm.graph.randomness import keyed_uniform ATOMIC_SUPPORT_TYPE = ArtifactType("microcosm.geography.atomic_support_npz", 1) +ATOMIC_GEOGRAPHY_VALIDATION_TYPE = ArtifactType( + "microcosm.geography.validation_result", 1 +) _DEPENDENCIES = ("numpy", "pandas") @@ -174,20 +177,38 @@ class AtomicGeographyGateKernel(_GeographyKernel): def run(self, context: KernelContext) -> KernelResult: households, spec, supports = _inputs(context) + outputs = context.node.artifact_outputs + expected = (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if outputs not in ((), expected): + raise ValueError( + "Atomic geography validation artifact declaration differs." + ) + receipt = geography.validate_geography(households, spec, supports) return KernelResult( - receipt=geography.validate_geography(households, spec, supports) + receipt=receipt, + artifacts={"validation": canonical_json(receipt)} if outputs else {}, ) def atomic_geography_nodes( - spec: Mapping, columns: Sequence[Owned], *, base: str, prefix: str = "geography" + spec: Mapping, + columns: Sequence[Owned], + *, + base: str, + prefix: str = "geography", + emit_validation_artifact: bool = False, ) -> tuple[Node, ...]: """Append shared nodes to an existing population, refusing column rewrites. SourceRef declarations use each system's ``source`` and ``raw-bytes-v1``. ``identity`` must be stable across the country's sampling rungs; this builder cannot establish that property merely from a column name. + + The optional typed gate result orders downstream consumers. Its bytes alone + do not authenticate their receiving population or source ancestry. """ + if type(emit_validation_artifact) is not bool: + raise ValueError("Atomic geography validation artifact flag must be boolean.") spec = geography.validate_assignment_spec(spec) inventory = {(o.entity, o.column): o for o in columns} if len(inventory) != len(columns): @@ -235,6 +256,11 @@ def node(suffix, kernel, read, write=()): "stream": tuple(spec["stream"]), }, artifact_inputs=tuple(artifacts), + artifact_outputs=( + (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if suffix == "gate" and emit_validation_artifact + else () + ), ) return ( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py index 0033c318d..20fa3686c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py @@ -216,8 +216,9 @@ def assignment_definition( """Declare one block draw constrained by normalized observed state/PUMA. Input codes must already be strings from the survey's source projection. - Callers choose original source identity columns before cloning; subsequent - clone/prune operators retain the assigned block and derived geographies. + Callers declare stable source plus completed initial-clone identity columns. + Assignment follows those clones; subsequent views retain the block and all + functionally derived geographies. """ sources = _sources(source_ids) _require(type(seed) is int and 0 <= seed < 2**63, "invalid seed") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py index 1dc823e39..7598dba1a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -249,6 +249,7 @@ class QualifiedSurveyPredictors: evidence: dict demographic_conditioning: bool = False geography_config_payload: bytes | None = None + geography_validation: bytes | None = None def _qualified_seal(value): @@ -288,6 +289,7 @@ def _qualified_seal(value): value.matrix, value.demographic_conditioning, value.geography_config_payload, + value.geography_validation, codec.encode_json(value.evidence), tuple( geography._population_stamp( @@ -516,6 +518,9 @@ def qualify_current_survey_predictors( evidence, demographic_conditioning, config_payload, + None + if geography_binding is None + else codec.encode_json(json.loads(geography_binding)["validation_receipt"]), ) derived_seal = _qualified_seal(result) if geography_config is not None: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py index 84caad516..f65c71236 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py @@ -1,15 +1,19 @@ -"""Compose shared geography before the combined-survey support clone. +"""Assign shared geography after the complete combined-survey support clone. This declaration has no country-specific geography kernel or source admission. Callers provide normalized, qualified observed columns and declared support. """ -from dataclasses import replace - +from microcosm.build.atomic_geography import validate_assignment_spec from microcosm.build.graph_atomic_geography import atomic_geography_nodes from .graph_combined_clone import us_combined_survey_clone_nodes +ASSIGNMENT_IDENTITY = ( + "survey_geography_origin_key", + "household_support_clone_index", +) + def atomic_survey_clone_nodes( definition, @@ -19,26 +23,34 @@ def atomic_survey_clone_nodes( geography_prefix="geography", clone_prefix="combined_survey_puf_support_clone", ): - """Assign once on the base, inherit every location, then recheck the clone. + """Complete initial clones, then draw on each stable source/role identity. - Structural graph nodes depend on every member of their base version. The - pre-clone integrity gate therefore precedes EXPAND even though it owns no - cells. The post-clone gate derives mappings without drawing again. + The base carries qualified observed constraints, not assigned geography. + The clone-index Slice depends on its ownership claim; the population edge + depends on EXPAND. Numeric remapped household IDs are only row coordinates. """ columns = tuple(columns) - geography = atomic_geography_nodes( - definition, columns, base=base, prefix=geography_prefix - ) + definition = validate_assignment_spec(definition) + if tuple(definition.get("identity", ())) != ASSIGNMENT_IDENTITY: + raise ValueError( + "US postclone geography requires its stable source/clone identity." + ) clones = us_combined_survey_clone_nodes( - (*columns, *(output for node in geography for output in node.outputs)), + columns, base=base, source_channels=("acs", "asec"), prefix=clone_prefix, ) - final_gate = replace( - geography[-1], - id=f"{geography_prefix}.clone_gate", - population=clones[0].id, - description="Recheck inherited atomic geography on every support clone.", + inventory = {(o.entity, o.column): o for o in columns} + inventory.update({(o.entity, o.column): o for o in clones[1].outputs}) + origin = inventory.get(("household", ASSIGNMENT_IDENTITY[0])) + if origin is None or origin.dtype != "string": + raise ValueError("US postclone geography requires a string source identity.") + geography = atomic_geography_nodes( + definition, + tuple(inventory.values()), + base=clones[0].id, + prefix=geography_prefix, + emit_validation_artifact=True, ) - return (*geography, *clones, final_gate) + return (*clones, *geography) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py index fbfaa9081..d89986093 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -1,6 +1,6 @@ """Checked atomic survey geography plus current financial development output. -This twenty-node composition retains the raw allocation and the pre-financial +This nineteen-node composition retains the raw allocation and the pre-financial clone separately. It grants neither a budget successor nor PUF recipient or release authority. Support bytes establish integrity, not publisher provenance. """ @@ -300,6 +300,8 @@ def _issue_run( (name, getattr(prefix, name)) for name in ( "allocated_population", + "observed_population", + "expanded_population", "geography_population", "clone_population", ) @@ -548,7 +550,7 @@ def run_atomic_survey_financial( resume="auto", return_values=False, ): - """Execute and verify all twenty nodes, including required cache replay.""" + """Execute and verify all nineteen nodes, including required cache replay.""" require(type(return_values) is bool, "RETURN_VALUES_FLAG") values.feature_columns(demographic_conditioning) live = _live() @@ -580,7 +582,13 @@ def run_atomic_survey_financial( getattr(prefix, name), reconstruction._population_stamp(getattr(prefix, name)), ) - for name in ("allocated_population", "geography_population", "clone_population") + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) } qualified = values.qualify_current_survey_predictors( prefix.preparation, @@ -597,7 +605,10 @@ def run_atomic_survey_financial( for node_id, record in prefix.manifest.nodes.items(): for name, key in record.opaque_artifacts.items(): prefix_artifacts[node_id, name] = prefix.store.load_bytes(key) - for edge in financial.host.current_survey_host_edges(): + for edge in ( + *financial.host.current_survey_host_edges(), + financial._geography_edge(), + ): record = prefix.manifest.node(edge.producer) pins[edge.name] = { "producer_key": record.key, @@ -614,9 +625,16 @@ def run_atomic_survey_financial( replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, *nodes)) ) require( - len(prefix.compiled.order) == 10 and len(compiled.order) == 20, + len(prefix.compiled.order) == 9 and len(compiled.order) == 19, "ATOMIC_COMPILER_ROSTER", ) + gate_edge = financial._geography_edge() + require( + gate_edge.producer in compiled.predecessors[financial.DONOR_NODE] + and prefix_artifacts[gate_edge.producer, gate_edge.artifact] + == qualified.geography_validation, + "ATOMIC_GEOGRAPHY_GATE_EDGE", + ) declaration = graph_to_json(compiled.graph) kernels, store, sources = prefix.kernels, prefix.store, dict(prefix.sources) source_items = tuple(sorted(sources.items())) @@ -650,11 +668,6 @@ def run_atomic_survey_financial( **{n: r.receipt for n, r in prefix.manifest.nodes.items()}, **{s.node.id: json.loads(s.receipt) for s in geography.stages}, } - cloned, clone_receipts = atomic._clone_expectations( - geography, prefix.compiled.graph.nodes, compiled - ) - base_expected.update(cloned) - receipts.update(clone_receipts) donor_node = compiled.graph.node(financial.DONOR_NODE) donor = population_ops.patch( base_expected[survey.CREATE_NODE], @@ -750,8 +763,6 @@ def observe(node_id, population): ) ), ) - elif node_id == "geography.clone_gate": - population = current[version] elif node_id in base_expected: population = base_expected[node_id] else: @@ -844,7 +855,7 @@ def observe(node_id, population): ) for version, population in ( (survey.CREATE_NODE, base_expected[survey.CREATE_NODE]), - (survey.ALLOCATION_NODE, prefix.geography_population), + (survey.ALLOCATION_NODE, prefix.observed_population), (atomic.clone.COMBINED_CLONE_NODE, prefix.clone_population), ): survey._same_frame(population.frame, prefix.manifest.population(version)) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py index 4d5d19f91..5bd732768 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py @@ -1,4 +1,4 @@ -"""Execute qualified survey geography before the first support clone. +"""Complete initial support clones before assigning qualified survey geography. The existing runner independently admits the raw allocation. This extension reconstructs every added column from the live preparation and pinned normalized @@ -9,21 +9,20 @@ from __future__ import annotations import json -from dataclasses import dataclass, replace +from dataclasses import dataclass from types import SimpleNamespace -import numpy as np - -from microcosm.build import atomic_geography from microcosm.build.graph_atomic_geography import ( - ATOMIC_SUPPORT_TYPE, - AtomicSupportImportKernel, + ATOMIC_SUPPORT_TYPE as ATOMIC_SUPPORT_TYPE, +) +from microcosm.build.graph_atomic_geography import ( + AtomicSupportImportKernel as AtomicSupportImportKernel, +) +from microcosm.build.graph_atomic_geography import ( register_atomic_geography_kernels, ) -from microcosm.frame import US_SCHEMA from microcosm.graph import ( Graph, - KernelResult, SourceRef, StructuralDelta, compile_graph, @@ -33,8 +32,6 @@ from microcosm.graph.codecs import load_raw_bytes from microcosm.graph.executor import ( _all_node_keys, - _expand_declared_payload, - _expand_rewrite_coordinates, _input_writers, _source_paths_and_keys, _tolerance_writer_payload, @@ -50,9 +47,6 @@ from microcosm.graph.manifest import _freeze_json from microcosm.graph.population import ( Population, - _mass_record, - expand_lineage_receipt, - expand_writes_receipt, mass_record_receipt, weight_cap_receipt, ) @@ -60,9 +54,7 @@ from . import graph_combined_clone as clone from . import graph_current_survey_geography as projection from . import graph_survey_population as survey -from . import puf_support from . import survey_atomic_geography as reconstruction -from .graph_sources import frame_column_declarations from .survey_population_replay import same_replayed_population @@ -73,6 +65,8 @@ class AtomicSurveyPopulationRunValues: manifest: object preparation: object allocated_population: Population + observed_population: Population + expanded_population: Population geography_population: Population clone_population: Population geography_config: reconstruction.AtomicSurveyReconstruction @@ -82,76 +76,6 @@ class AtomicSurveyPopulationRunValues: sources: dict -def _clone_expectations(geography, nodes, compiled): - """Independently reconstruct the complete clone, including inherited cells.""" - before = geography.population - expanded = puf_support.clone_us_frame_for_puf_support( - before.frame, - clone_attachment_fraction=1.0, - clone_attachment_seed=0, - ) - design = survey._verify_cloned_frame( - before.frame, expanded, before.design_weights["household"] - ) - expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) - claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) - gate = next(n for n in nodes if n.id == "geography.clone_gate") - ledger = ( - *before.mass_ledger, - _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), - ) - owners = { - (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) - } - expected, receipts = {}, {} - lineage, facts = {}, {} - for entity in US_SCHEMA.entities: - lineage[entity], facts[entity] = clone._entity_lineage( - before.frame, expanded, entity - ) - authority = puf_support.validate_puf_clone_attachment( - expanded, boundary=survey.PHASE, expected_fraction=1.0, expected_seed=0 - ) - receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( - before.frame, expanded, ("acs", "asec"), authority, facts - ) - receipt["expand"] = expand_lineage_receipt(lineage) - receipt["expand_declared"] = _expand_declared_payload(expand) - receipt["expand_writes"] = expand_writes_receipt( - before.frame, - expanded, - expand, - receipt, - rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), - ) - receipts[expand.id] = receipt - receipts[claim.id] = { - "phase": clone.COMBINED_CLONE_PHASE, - "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), - } - definition = json.loads(geography.definition) - system = definition["systems"][0]["id"] - receipts[gate.id] = atomic_geography.validate_geography( - expanded.table("household"), - definition, - {system: atomic_geography.decode_atomic_support(geography.support_payload)}, - ) - # The gate and the ownership claim can have either topological order. - for node_id in compiled.order: - if node_id not in receipts: - continue - if node_id == claim.id: - owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) - expected[node_id] = Population.from_frame( - expanded, - expand.id, - owners=owners, - mass_ledger=ledger, - design_weights={"household": np.array(design, copy=True)}, - ) - return expected, receipts - - def _states(compiled, kernels, source_keys, expected_populations, raw_receipts): """Bind independent domain receipts to current implementation and graph keys.""" keys, implementations = _all_node_keys(compiled, kernels, source_keys) @@ -221,7 +145,7 @@ def run_atomic_survey_population( resume="auto", return_values=False, ): - """Build one block location before cloning and independently verify replay.""" + """Assign each completed source/clone household and independently verify replay.""" survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") prefix = survey.run_authenticated_survey_population( source_dir, @@ -251,17 +175,7 @@ def run_atomic_survey_population( (survey.ALLOCATION_NODE, "allocation"): allocation_payload, (survey.ALLOCATION_NODE, "frame_context"): allocated_context, } - clone_nodes = clone.us_combined_survey_clone_nodes( - frame_column_declarations(geography.population.frame), - base=survey.ALLOCATION_NODE, - source_channels=("acs", "asec"), - ) - gate = replace( - geography.nodes[-1], - id="geography.clone_gate", - population=clone_nodes[0].id, - ) - additions = (*geography.nodes, *clone_nodes, gate) + additions = geography.nodes compiled = compile_graph( Graph( "us", @@ -272,6 +186,7 @@ def run_atomic_survey_population( (*prefix.compiled.graph.nodes, *additions), ) ) + survey._require(len(compiled.order) == 9, "ATOMIC_COMPILER_ROSTER") sources = {**prefix.sources, **dict(geography.sources)} kernels, store = prefix.kernels, prefix.store store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) @@ -289,9 +204,6 @@ def run_atomic_survey_population( **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, } - cloned, clone_receipts = _clone_expectations(geography, additions, compiled) - expected.update(cloned) - receipts.update(clone_receipts) expected_stamps = { name: reconstruction._population_stamp(value) for name, value in expected.items() @@ -319,16 +231,17 @@ def observe(node_id, population): ) survey._require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") survey._check_node_states(manifest, states) - for node in geography.nodes: - if node.kernel == AtomicSupportImportKernel.ref: + for stage in geography.stages: + for name, payload in stage.artifacts: + output = next(o for o in stage.node.artifact_outputs if o.name == name) survey._final_artifact( manifest, store, - node_id=node.id, - name="support", - type_=ATOMIC_SUPPORT_TYPE, - payload=geography.support_payload, - capabilities=AtomicSupportImportKernel.capabilities, + node_id=stage.node.id, + name=name, + type_=output.type, + payload=payload, + capabilities=kernels.get(stage.node.kernel).capabilities, ) # Validate prefix artifacts separately from column materialization. for (node_id, name), payload in prefix_artifacts.items(): @@ -340,22 +253,22 @@ def observe(node_id, population): survey._require( store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" ) - clone_terminal = next( - name - for name in reversed(compiled.order) - if compiled.versions[name] == clone_nodes[0].id - ) + geography_terminal = geography.nodes[-1].id result = AtomicSurveyPopulationRunValues( - manifest, - prefix.preparation, - prefix.allocated_population, - observed[geography.nodes[-1].id], - observed[clone_terminal], - geography_config, - compiled, - store, - kernels, - sources, + manifest=manifest, + preparation=prefix.preparation, + allocated_population=prefix.allocated_population, + observed_population=observed[projection.NODE], + expanded_population=observed[clone.COMBINED_CLONE_CLAIM_NODE], + geography_population=observed[geography_terminal], + # This historical field is the terminal receiving population. The + # explicit expanded_population is the preassignment clone snapshot. + clone_population=observed[geography_terminal], + geography_config=geography_config, + compiled=compiled, + store=store, + kernels=kernels, + sources=sources, ) current_keys, current_implementations = _all_node_keys( compiled, kernels, source_keys @@ -385,8 +298,10 @@ def observe(node_id, population): and result.geography_config is geography_config and result.preparation is prefix.preparation and result.allocated_population is prefix.allocated_population - and result.geography_population is observed[geography.nodes[-1].id] - and result.clone_population is observed[clone_terminal] + and result.observed_population is observed[projection.NODE] + and result.expanded_population is observed[clone.COMBINED_CLONE_CLAIM_NODE] + and result.geography_population is observed[geography_terminal] + and result.clone_population is result.geography_population and result.manifest is manifest and result.compiled is compiled and result.store is store @@ -395,6 +310,8 @@ def observe(node_id, population): and reconstruction._population_stamp(prefix.allocated_population) == raw_stamp, "ATOMIC_FINAL_RECONSTRUCTION", ) + same_replayed_population(fresh.observed_population, result.observed_population) + same_replayed_population(fresh.expanded_population, result.expanded_population) same_replayed_population(fresh.population, result.geography_population) for node_id, population in expected.items(): survey._require( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py index 80c9a1108..b6d9cfd66 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py @@ -100,12 +100,30 @@ def _attach_edges(): return tuple(result) +def _geography_edge(): + return ArtifactInput( + "geography_validation", + "geography.gate", + "validation", + host.survey_budget.geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + ) + + def _params(qualified, host_pins, n_estimators): require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") values.feature_columns(qualified.demographic_conditioning) + geography_enabled = qualified.geography_config_payload is not None + require( + (type(qualified.geography_validation) is bytes) + if geography_enabled + else qualified.geography_validation is None, + "GEOGRAPHY_VALIDATION_VALUES", + ) + names = {e.name for e in host.current_survey_host_edges()} + if geography_enabled: + names.add(_geography_edge().name) require( - type(host_pins) is dict - and set(host_pins) == {e.name for e in host.current_survey_host_edges()}, + type(host_pins) is dict and set(host_pins) == names, "HOST_PINS", ) for pin in host_pins.values(): @@ -160,7 +178,14 @@ def current_survey_predictor_nodes( params=params, # Branch from the authenticated CREATE before its weight transition. # The later projection depends on this FILTER, never conversely. - artifact_inputs=(host.current_survey_host_edges()[0],), + artifact_inputs=( + host.current_survey_host_edges()[0], + *( + (_geography_edge(),) + if qualified.geography_config_payload is not None + else () + ), + ), description="Select native ASEC whole households before allocation; retain original design weights for survey financial fitting.", ) columns = Node( @@ -257,7 +282,7 @@ def implementation_hash(self): model_input, sys.modules[decode_matrix_apply_state.__module__], sys.modules[legacy_qrf_train_nodes.__module__], - # The optional pre-clone geography admission invokes + # The optional postclone geography admission invokes # the budget owner's complete reconstruction closure. *host.survey_budget._modules(), dependencies=self.capabilities.dependencies, @@ -310,6 +335,20 @@ def _qualified(self, context): and value.payload == self.preparation.payload, "DONOR_PREPARATION_BINDING", ) + if result.geography_config_payload is not None: + edge = _geography_edge() + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and type(value.payload) is bytes + and value.payload == result.geography_validation, + "GEOGRAPHY_VALIDATION_BINDING", + ) elif context.node.id != PROJECTION_NODE: projection = host.shared.artifact(context, "projection", PROJECTION_TYPE) matrix = host.shared.artifact( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py index 42ee6c992..c2051861b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -464,10 +464,6 @@ def _run_survey_age_calibration( prefix_populations = { stage.node.id: stage.population for stage in geography.stages } - expanded, _receipts = atomic_source._clone_expectations( - geography, prefix.compiled.graph.nodes, prefix.compiled - ) - prefix_populations.update(expanded) source_owner, source_view = source_graph._checked_preparation(prefix.preparation) preparation_entry = source_owner._ISSUED.get(id(prefix.preparation)) instructions = source_graph.allocation_instructions( @@ -518,6 +514,16 @@ def _run_survey_age_calibration( for node in geography.nodes if node.kernel == atomic_source.AtomicSupportImportKernel.ref ) + gate = geography.stages[-1] + prefix_artifacts += ( + ( + gate.node.id, + "validation", + atomic_source.reconstruction.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + gate.receipt, + atomic_source.reconstruction.atomic_graph.AtomicGeographyGateKernel.capabilities, + ), + ) initial = prefix.clone_population _require(initial is not None, "COMPLETE_CLONE_REQUIRED") budget = budgets.freeze_survey_origin_budget( @@ -591,6 +597,11 @@ def _run_survey_age_calibration( successor_payload = None receiving_initial = None receiving_budget_entry = None + receiving_node = ( + clone.COMBINED_CLONE_CLAIM_NODE + if geography is None + else geography.stages[-1].node.id + ) def observe(node_id, population): nonlocal admitted, budget, receiving_initial, receiving_budget_entry @@ -610,26 +621,38 @@ def observe(node_id, population): successor_entry = budgets._entry(admitted, budgets.SamplingOriginSuccessor) successor_payload = admitted.payload elif node_id in {transport.BUDGET_NODE, COUNT_NODE}: - # Count may precede the ownership claim; it is a read-only artifact. - # A named replay check does not pretend it shares the claim owners. - replay.same_replayed_frame(initial.frame, population.frame) + # Read-only count/transport may precede the geography gate. Compare + # their complete current version, including whichever prefix writes + # have actually occurred. Calibration waits for every base member. + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + preceding = next( + name + for name in reversed(observed) + if name in prefix_populations + and compiled.versions[name] == initial.version + ) + replay.same_replayed_population( + prefix_populations[preceding], population + ) _require( population.frame.weights_for("household").values.tobytes() == numeric_bounds.incoming.tobytes(), "RECEIVING_INCOMING_WEIGHTS", ) - if node_id == transport.BUDGET_NODE: + if node_id == transport.BUDGET_NODE and geography is None: _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") _require( budgets._population_identity(population) == snapshots[clone.COMBINED_CLONE_CLAIM_NODE], "COMPLETE_CLAIMED_CLONE", ) - else: + elif node_id == COUNT_NODE: _require(_measure(population) == count_payload, "REMEASURED_COUNTS") elif node_id == source_graph.ALLOCATION_NODE: replay.same_replayed_population(prefix.allocated_population, population) - elif node_id == clone.COMBINED_CLONE_CLAIM_NODE: + elif node_id == receiving_node: replay.same_replayed_population(initial, population) _require(source_graph.ALLOCATION_NODE in observed, "ALLOCATION_REQUIRED") # Actual source reconstruction rebinds an issued budget to this run's @@ -653,15 +676,16 @@ def observe(node_id, population): actual_allocation = observed[ source_graph.ALLOCATION_NODE if geography is None - else geography.stages[-1].node.id + else atomic_source.projection.NODE ] source_graph._verify_cloned_frame( actual_allocation.frame, population.frame, actual_allocation.design_weights["household"], ) - replay.same_replayed_frame(initial.frame, population.frame) - if geography is not None: + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: replay.same_replayed_population(prefix_populations[node_id], population) elif node_id in prefix_populations: replay.same_replayed_population(prefix_populations[node_id], population) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py index 9671eceb9..d6fe50b0d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -1,4 +1,4 @@ -"""Reconstruct the complete pre-clone geography Population from live sources. +"""Reconstruct observed constraints, initial clones and geography from live sources. The immutable recipe names normalized support bytes; their digest proves byte integrity, not publisher provenance. A native source adapter must establish the @@ -24,21 +24,42 @@ from microcosm.build import atomic_geography as atomic from microcosm.build import graph_atomic_geography as atomic_graph -from microcosm.frame import Frame, WeightKind, Weights -from microcosm.graph import Graph, KernelResult, Node, SourceRef, compile_graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Graph, + KernelResult, + Node, + SourceRef, + StructuralDelta, + compile_graph, +) from microcosm.graph.canonical import canonical_json from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES -from microcosm.graph.population import Population, _mass_record, _storage_parts, patch +from microcosm.graph.executor import ( + _expand_declared_payload, + _expand_rewrite_coordinates, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + expand_lineage_receipt, + expand_writes_receipt, + patch, +) from . import atomic_block_support as blocks from . import current_survey_geography as observed +from . import graph_atomic_survey_clone as composition +from . import graph_combined_clone as clone from . import graph_current_survey_geography as observed_graph from . import graph_survey_population as graph +from . import puf_support from . import survey_population_preparation as source from . import survey_population_replay as replay from .graph_sources import frame_column_declarations -PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v1" +PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v2" MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES MAX_SUPPORT_EXPANDED_BYTES = 2 * 1024**3 MAX_SUPPORT_MEMBERS = 32 @@ -279,6 +300,67 @@ def _raw_allocation(view, population): return columns, nodes +def _clone_expectations(before, nodes, compiled): + """Independently reconstruct the complete clone, including inherited cells.""" + expanded = puf_support.clone_us_frame_for_puf_support( + before.frame, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + design = graph._verify_cloned_frame( + before.frame, expanded, before.design_weights["household"] + ) + expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) + claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) + ledger = ( + *before.mass_ledger, + _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), + ) + owners = { + (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) + } + expected, receipts = {}, {} + lineage, facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], facts[entity] = clone._entity_lineage( + before.frame, expanded, entity + ) + authority = puf_support.validate_puf_clone_attachment( + expanded, boundary=graph.PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + before.frame, expanded, ("acs", "asec"), authority, facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(expand) + receipt["expand_writes"] = expand_writes_receipt( + before.frame, + expanded, + expand, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), + ) + receipts[expand.id] = receipt + receipts[claim.id] = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), + } + # Preserve exact structural and claim ownership before assignment. + for node_id in compiled.order: + if node_id not in receipts: + continue + if node_id == claim.id: + owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) + expected[node_id] = Population.from_frame( + expanded, + expand.id, + owners=owners, + mass_ledger=ledger, + design_weights={"household": np.array(design, copy=True)}, + ) + return expected, receipts + + @dataclass(frozen=True, slots=True) class AtomicSurveyGeographyStage: """Expected complete Population and pure receipt, never executor evidence.""" @@ -294,6 +376,8 @@ class AtomicSurveyGeographyReconstruction: """Detached reconstruction values; consumers must independently requalify.""" population: Population + observed_population: Population + expanded_population: Population stages: tuple[AtomicSurveyGeographyStage, ...] nodes: tuple[Node, ...] definition: bytes @@ -305,10 +389,10 @@ class AtomicSurveyGeographyReconstruction: def reconstruct_atomic_survey_geography(preparation, allocated_population, config): - """Qualify sources and reconstruct geography on the exact raw allocation. + """Qualify sources, complete initial clones, then assign on the clone version. - Stage values follow the actual compiled order of the source/allocation plus - geography graph. A support import may precede the observed-column projection. + Stage values follow the actual compiled order, including the structural + expansion. Support import and clone ownership may have either relative order. Callers compare these complete expected Populations to executor observations on cold and required-warm runs; these values do not replace those checks. """ @@ -329,7 +413,7 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi support = _decode_support(support_payload) supports = {blocks.SYSTEM: support} definition = blocks.assignment_definition( - identity=(observed.COLUMNS[0],), + identity=composition.ASSIGNMENT_IDENTITY, state_column=observed.COLUMNS[1], puma_column=observed.COLUMNS[2], source_ids=dict(config.source_ids), @@ -351,10 +435,10 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi projection_receipt_sha256=_sha(projection_receipt), population=graph.ALLOCATION_NODE, ) - geography = atomic_graph.atomic_geography_nodes( + additions = composition.atomic_survey_clone_nodes( definition, (*columns, *observation.outputs), base=graph.ALLOCATION_NODE ) - nodes = (observation, *geography) + nodes = (observation, *additions) compiled = compile_graph( Graph( "us", @@ -370,8 +454,10 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi compiled.graph.node(name) for name in compiled.order if name in wanted ) stages = [] + cloned, clone_receipts = {}, {} for node in ordered: values, artifacts = {}, () + structural_or_claim = False if node.id == observation.id: values = { ("household", name): projection.household[name].copy(deep=True) @@ -385,6 +471,16 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi "population_admission_issued": False, "release_eligible": False, } + elif node.structural is StructuralDelta.EXPAND: + cloned, clone_receipts = _clone_expectations(current, nodes, compiled) + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.id == clone.COMBINED_CLONE_CLAIM_NODE: + _require(node.id in cloned, "CLONE_BEFORE_CLAIM") + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True elif node.kernel == atomic_graph.AtomicSupportImportKernel.ref: _require( support.metadata["system"] == node.params["system"], "SUPPORT_SYSTEM" @@ -422,19 +518,27 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi ) output = None receipt = atomic.validate_geography(households, definition, supports) + artifacts = (("validation", canonical_json(receipt)),) if output is not None: ids = pd.Index(households.household_id.to_numpy(), name="household_id") values = { ("household", name): pd.Series(output[name].array.copy(), index=ids) for name in output } - current = _copy_population(patch(current, node, KernelResult(columns=values))) + if not structural_or_claim: + current = _copy_population( + patch(current, node, KernelResult(columns=values)) + ) stages.append( AtomicSurveyGeographyStage( node, current, canonical_json(receipt), artifacts ) ) stage_tuple = tuple(stages) + by_id = {stage.node.id: stage for stage in stage_tuple} + observed_population = by_id[observation.id].population + expanded_population = by_id[clone.COMBINED_CLONE_CLAIM_NODE].population + validation_payload = by_id["geography.gate"].artifacts[0][1] receipt = canonical_json( { "protocol": PROTOCOL, @@ -443,9 +547,13 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi "projection_receipt_sha256": _sha(projection_receipt), "support_sha256": _sha(support_payload), "definition_sha256": _sha(definition_bytes), + "validation_receipt_sha256": _sha(validation_payload), + "assignment_identity": list(composition.ASSIGNMENT_IDENTITY), "stages": [stage.node.id for stage in stage_tuple], "frame_sha256": source._frame_identity(current.frame), "population_sha256": _population_stamp(current), + "observed_population_sha256": _population_stamp(observed_population), + "expanded_population_sha256": _population_stamp(expanded_population), "publisher_provenance_established": False, "source_admission_issued": False, "population_admission_issued": False, @@ -454,6 +562,8 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi ) result = AtomicSurveyGeographyReconstruction( population=_copy_population(current), + observed_population=observed_population, + expanded_population=expanded_population, stages=stage_tuple, nodes=nodes, definition=definition_bytes, @@ -503,6 +613,8 @@ def reconstruct_atomic_survey_geography(preparation, allocated_population, confi ) _require( result.nodes == nodes + and result.observed_population is observed_population + and result.expanded_population is expanded_population and tuple(stage.node for stage in result.stages) == ordered and tuple( (stage.node, stage.receipt, stage.artifacts) for stage in result.stages diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py index 00ac974bc..218a1e7f0 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -5,7 +5,7 @@ provisional; this module grants neither graph execution nor release authority. Decoded bytes cannot issue a budget or a weight-only successor. An optional immutable geography recipe independently reconstructs the complete -pre-clone Frame; the retained allocation always remains the raw source allocation. +postclone geography Frame; the retained allocation remains the raw source allocation. """ from __future__ import annotations @@ -143,6 +143,7 @@ def _modules(): geography.blocks, geography.atomic, geography.atomic_graph, + geography.composition, geography.observed, geography.observed_graph, geography.observed.demographics, @@ -153,6 +154,7 @@ def _modules(): sys.modules[geography.atomic.keyed_uniform.__module__], sys.modules[geography.canonical_json.__module__], sys.modules[geography.compile_graph.__module__], + sys.modules[geography._expand_declared_payload.__module__], ) return tuple({module.__name__: module for module in modules}.values()) @@ -201,6 +203,7 @@ def _live(): geography.atomic._U53, geography.observed.PROTOCOL, geography.observed.COLUMNS, + geography.composition.ASSIGNMENT_IDENTITY, geography.observed.MAX_HOUSEHOLDS, geography.observed.MAX_RECEIPT_BYTES, geography.observed_graph.NODE, @@ -214,6 +217,10 @@ def _live(): geography.atomic_graph.ATOMIC_SUPPORT_TYPE.name, geography.atomic_graph.ATOMIC_SUPPORT_TYPE.schema_version, ), + ( + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.name, + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.schema_version, + ), geography.atomic_graph._DEPENDENCIES, tuple( (kernel.ref, asdict(kernel.capabilities)) @@ -298,6 +305,18 @@ def _geography_binding(value, config_payload, preparation_payload): ) document = json.loads(value.receipt) population_sha256 = geography._population_stamp(value.population) + stages = {stage.node.id: stage for stage in value.stages} + _require( + value.observed_population is stages[geography.observed_graph.NODE].population + and value.expanded_population + is stages[clone.COMBINED_CLONE_CLAIM_NODE].population, + "GEOGRAPHY_STAGE_BINDINGS", + ) + validation = stages["geography.gate"].receipt + _require( + stages["geography.gate"].artifacts == (("validation", validation),), + "GEOGRAPHY_VALIDATION_ARTIFACT", + ) _require( value.config_sha256 == _sha(config_payload) and document["protocol"] == geography.PROTOCOL @@ -305,6 +324,13 @@ def _geography_binding(value, config_payload, preparation_payload): and document["config_sha256"] == value.config_sha256 and document["frame_sha256"] == source._frame_identity(value.population.frame) and document["population_sha256"] == population_sha256 + and document["observed_population_sha256"] + == geography._population_stamp(value.observed_population) + and document["expanded_population_sha256"] + == geography._population_stamp(value.expanded_population) + and document["validation_receipt_sha256"] == _sha(validation) + and document["assignment_identity"] + == list(geography.composition.ASSIGNMENT_IDENTITY) and document["projection_receipt_sha256"] == _sha(value.projection_receipt) and document["definition_sha256"] == _sha(value.definition) and document["support_sha256"] == _sha(value.support_payload) @@ -317,8 +343,13 @@ def _geography_binding(value, config_payload, preparation_payload): # The full receipt above seals this detached object exactly. Frame-store # replay can canonicalize storage beneath masked nulls, so its physical # stamp cannot identify an independently reconstructed object across runs. + physical_fields = ( + "population_sha256", + "observed_population_sha256", + "expanded_population_sha256", + ) semantic_receipt = { - key: item for key, item in document.items() if key != "population_sha256" + key: item for key, item in document.items() if key not in physical_fields } population = value.population semantic_population = { @@ -341,9 +372,12 @@ def _geography_binding(value, config_payload, preparation_payload): "projection_receipt_sha256": _sha(value.projection_receipt), "definition_sha256": _sha(value.definition), "support_sha256": _sha(value.support_payload), - "preclone_population_semantic_sha256": _sha(_json(semantic_population)), + "postclone_geography_population_semantic_sha256": _sha( + _json(semantic_population) + ), + "validation_receipt": json.loads(validation), "identity_scope": { - "reconstruction_omitted_fields": ["population_sha256"], + "reconstruction_omitted_fields": list(physical_fields), "population_fields": list(semantic_population), "null_backing": "retained_in_in_process_physical_seals_only", }, @@ -405,36 +439,43 @@ def _initial( geography_binding = _geography_binding( reconstructed, config_payload, view.payload ) - expected = reconstructed.population.frame - columns = frame_column_declarations(expected) - clone_nodes = clone.us_combined_survey_clone_nodes( - columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") - ) - copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) - graph._check_design_anchors(expanded, copied_design) - owners = { - (entity, str(column)): clone.COMBINED_CLONE_NODE - for entity in expected.entities - for column in expected.table(entity) - } - owners.update( - { - (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE - for o in clone_nodes[1].outputs + # Verify the exact clone before assignment, then the complete geography + # population. Geography additions never relax the clone equality rule. + graph._verify_cloned_frame( + reconstructed.observed_population.frame, + reconstructed.expanded_population.frame, + design, + ) + geography.replay.same_replayed_population(reconstructed.population, expanded) + else: + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(expanded, copied_design) + owners = { + (entity, str(column)): clone.COMBINED_CLONE_NODE + for entity in expected.entities + for column in expected.table(entity) } - ) - graph._check_population_state( - expanded, - version=clone.COMBINED_CLONE_NODE, - owners=owners, - kind=WeightKind.IMPORTANCE, - ledger=( - *allocation_ledger, - _mass_record( - expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + owners.update( + { + (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE + for o in clone_nodes[1].outputs + } + ) + graph._check_population_state( + expanded, + version=clone.COMBINED_CLONE_NODE, + owners=owners, + kind=WeightKind.IMPORTANCE, + ledger=( + *allocation_ledger, + _mass_record( + expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + ), ), - ), - ) + ) if geography_config is not None: _require( geography.AtomicSurveyReconstruction.to_bytes(geography_config) diff --git a/packages/microcosm-build/tests/test_atomic_geography.py b/packages/microcosm-build/tests/test_atomic_geography.py index 84745e434..ccc4b172c 100644 --- a/packages/microcosm-build/tests/test_atomic_geography.py +++ b/packages/microcosm-build/tests/test_atomic_geography.py @@ -8,11 +8,14 @@ from microcosm.build import atomic_geography as geo from microcosm.build.graph_atomic_geography import ( + ATOMIC_GEOGRAPHY_VALIDATION_TYPE, atomic_geography_nodes, register_atomic_geography_kernels, ) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.graph import ( + ArtifactInput, + ArtifactType, Capabilities, ContentStore, Determinism, @@ -27,6 +30,8 @@ compile_graph, run_graph, ) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key def support_parts(system="invented_block", *, district=True): @@ -360,7 +365,10 @@ def run(self, context): ) -def test_real_executor_cold_warm_and_changed_source_identity(tmp_path): +@pytest.mark.parametrize("emit_validation_artifact", (False, True)) +def test_real_executor_cold_warm_and_changed_source_identity( + tmp_path, emit_validation_artifact +): households, spec, supports, payload = fixture() columns = ( Owned("person", "age", "int64"), @@ -376,7 +384,9 @@ def test_real_executor_cold_warm_and_changed_source_identity(tmp_path): structural=StructuralDelta.CREATE, outputs=columns, ) - nodes = atomic_geography_nodes(spec, columns, base=base.id) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=emit_validation_artifact + ) graph = compile_graph( Graph( "invented", @@ -397,7 +407,9 @@ def test_real_executor_cold_warm_and_changed_source_identity(tmp_path): sources = {"invented_spine": raw_spine, "invented_block_source": raw_support} store = ContentStore(tmp_path / "store") cold = run_graph(graph, sources=sources, store=store, kernels=registry) - warm = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph( + graph, sources=sources, store=store, kernels=registry, resume="require" + ) actual = cold.population(base.id).table("household") expected = complete(households, spec, supports) for column in expected: @@ -413,6 +425,19 @@ def test_real_executor_cold_warm_and_changed_source_identity(tmp_path): pd.testing.assert_series_equal(left, right, check_index_type=False) assert cold.nodes["geography.assign"].key == warm.nodes["geography.assign"].key assert cold.nodes["geography.gate"].receipt["outcome"] == "pass" + gate = cold.nodes["geography.gate"] + if emit_validation_artifact: + output = graph.graph.node("geography.gate").artifact_outputs + assert len(output) == 1 and output[0].type == ATOMIC_GEOGRAPHY_VALIDATION_TYPE + key = gate.opaque_artifacts["validation"] + assert key == opaque_artifact_key(gate.key, "validation") + assert store.load_bytes(key) == canonical_json( + geo.validate_geography(expected, spec, supports) + ) + assert warm.nodes["geography.gate"].opaque_artifacts["validation"] == key + else: + assert not gate.opaque_artifacts + np.testing.assert_array_equal( cold.population(base.id).weights_for("household").values, np.ones(8) ) @@ -441,3 +466,46 @@ def test_graph_builder_refuses_input_overwrites_and_duplicate_inventory(): altered["outputs"]["area"] = "observed_region" with pytest.raises(ValueError, match="overwritten"): atomic_geography_nodes(altered, columns, base="spine") + + +@pytest.mark.parametrize("defect", ("missing", "type")) +def test_typed_geography_gate_edge_refuses_invalid_producer_contract(defect): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=(defect != "missing") + ) + edge_type = ( + ArtifactType("invented.wrong_gate", 1) + if defect == "type" + else ATOMIC_GEOGRAPHY_VALIDATION_TYPE + ) + consumer = Node( + "consumer", + "invented.consumer@1", + population=base.id, + artifact_inputs=( + ArtifactInput("validation", "geography.gate", "validation", edge_type), + ), + ) + with pytest.raises(ValueError, match="no declared artifact|type does not match"): + compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes, consumer), + ) + ) diff --git a/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py b/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py index 43e9dda28..740d69ebe 100644 --- a/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py +++ b/packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py @@ -1,5 +1,7 @@ """Invented two-arm population through the actual geography/clone executor.""" +import json + import numpy as np import pandas as pd @@ -7,6 +9,7 @@ from microcosm.build.graph_atomic_geography import register_atomic_geography_kernels from microcosm.build.us_runtime import atomic_block_support as us from microcosm.build.us_runtime.graph_atomic_survey_clone import ( + ASSIGNMENT_IDENTITY, atomic_survey_clone_nodes, ) from microcosm.build.us_runtime.graph_combined_clone import ( @@ -32,7 +35,7 @@ from microcosm.graph.population import dtype_for_token -def invented_population(): +def invented_population(*, last_id=42): arms = {} string = dtype_for_token("string") for channel, state, puma in (("asec", "01", None), ("acs", "02", "0200002")): @@ -41,7 +44,7 @@ def invented_population(): "person_id": [1, 2, 3], "age": [40, 10, 50], **{ - f"person_{entity}_id": [11, 11, 42] + f"person_{entity}_id": [11, 11, last_id] for entity in US_SCHEMA.group_entities }, } @@ -49,10 +52,14 @@ def invented_population(): tables = { "person": person, **{ - entity: pd.DataFrame({f"{entity}_id": [11, 42]}) + entity: pd.DataFrame({f"{entity}_id": [11, last_id]}) for entity in US_SCHEMA.group_entities }, } + tables["household"][ASSIGNMENT_IDENTITY[0]] = pd.Series( + [json.dumps([channel, 2024, 2024, str(i)]) for i in (11, last_id)], + dtype=string, + ) tables["household"]["observed_state"] = pd.Series([state] * 2, dtype=string) tables["household"]["observed_puma"] = pd.Series([puma] * 2, dtype=string) arms[channel] = Frame( @@ -87,7 +94,7 @@ def run(self, context): return KernelResult(frame=invented_population()) -def test_block_assignment_precedes_clone_and_survives_required_replay(tmp_path): +def test_block_assignment_follows_complete_clone_and_survives_required_replay(tmp_path): source_ids = {k: "invented-" + k for k in ("population", "district", "puma")} blocks = [int(x) for x in ("010010201001000", "010010201001001", "020130001001000")] payload = us.assemble_atomic_block_support( @@ -97,7 +104,7 @@ def test_block_assignment_precedes_clone_and_survives_required_replay(tmp_path): source_ids=source_ids, ) spec = us.assignment_definition( - identity=("household_support_channel", "household_spine_source_id"), + identity=ASSIGNMENT_IDENTITY, state_column="observed_state", puma_column="observed_puma", source_ids=source_ids, @@ -126,8 +133,11 @@ def test_block_assignment_precedes_clone_and_survives_required_replay(tmp_path): ) ) clone_id = "combined_survey_puf_support_clone" - assert graph.order.index("geography.gate") < graph.order.index(clone_id) - assert graph.order.index(clone_id) < graph.order.index("geography.clone_gate") + assert graph.order.index(clone_id) < graph.order.index(clone_id + ".owned") + assert clone_id in graph.predecessors["geography.assign"] + assert clone_id + ".owned" in graph.predecessors["geography.assign"] + assert "geography.support.0" in graph.predecessors["geography.assign"] + assert "geography.clone_gate" not in graph.order registry = KernelRegistry() registry.register(InventedHost()) register_atomic_geography_kernels(registry) @@ -151,19 +161,27 @@ def test_block_assignment_precedes_clone_and_survives_required_replay(tmp_path): *spec["outputs"].values(), *(x["output"] for x in spec["systems"][0]["layers"]), ] - for column in geo_columns: - assert ( - native[column].tolist() - == copies[column].tolist() - == before.table("household")[column].tolist() + assert not set(geo_columns) & set(before.table("household")) + for column in (ASSIGNMENT_IDENTITY[0], "observed_state", "observed_puma"): + pd.testing.assert_series_equal( + native[column].reset_index(drop=True), + copies[column].reset_index(drop=True), ) + assert not hh.duplicated(list(ASSIGNMENT_IDENTITY)).any() + expected = geo.assign_atomic( + hh.drop(columns=geo_columns), + spec, + {us.SYSTEM: geo.decode_atomic_support(payload)}, + ) + for column in spec["outputs"].values(): + assert hh[column].tolist() == expected[column].tolist() for entity in original.entities: assert len(after.table(entity)) == 2 * len(original.table(entity)) np.testing.assert_array_equal( after.weights_for("household").values, np.tile(before.weights_for("household").values / 2, 2), ) - assert manifest.nodes["geography.clone_gate"].receipt["outcome"] == "pass" + assert manifest.nodes["geography.gate"].receipt["outcome"] == "pass" geo.validate_geography( hh.iloc[[6, 1]], spec, {us.SYSTEM: geo.decode_atomic_support(payload)} ) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py index 0553d1691..d9b46a3dc 100644 --- a/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py @@ -1,7 +1,8 @@ -"""Twenty actual graph nodes over invented, source-issued survey fixtures.""" +"""Nineteen actual graph nodes over invented, source-issued survey fixtures.""" import hashlib import sys +from dataclasses import replace from types import SimpleNamespace import numpy as np @@ -11,6 +12,8 @@ from test_us_graph_atomic_survey_population import _support_payload from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.graph import ArtifactType, ArtifactValue, NumericScope +from microcosm.graph.keys import opaque_artifact_key financial = runner.financial values = financial.values @@ -45,12 +48,29 @@ def known_financial_run(tmp_path_factory): values.source.verify_survey_population_preparation(run.prefix.preparation) -def test_twenty_node_financial_cold_and_required_replay(known_financial_run): +def test_nineteen_node_financial_cold_and_required_replay(known_financial_run): case = known_financial_run assert case.cold.manifest.key == case.warm.manifest.key assert all(n.hit for n in case.warm.manifest.nodes.values()) for run in (case.cold, case.warm): - assert len(run.compiled.order) == 20 + assert len(run.compiled.order) == 19 + edge = financial._geography_edge() + assert edge.producer in run.compiled.predecessors[financial.DONOR_NODE] + gate = run.manifest.node(edge.producer) + assert gate.opaque_artifacts[edge.artifact] == opaque_artifact_key( + gate.key, edge.artifact + ) + assert run.store.load_bytes( + gate.opaque_artifacts[edge.artifact] + ) == financial.codec.encode_json( + financial.codec.decode_json(run.projection)["atomic_geography"][ + "validation_receipt" + ] + ) + assert run.prefix.clone_population is run.prefix.geography_population + assert "census_block_geoid" not in run.prefix.expanded_population.frame.table( + "household" + ) assert set(run.compiled.order) == { *run.prefix.compiled.order, financial.PROJECTION_NODE, @@ -124,9 +144,7 @@ def test_twenty_node_financial_cold_and_required_replay(known_financial_run): for column in values.OUTPUTS: assert rows[column].notna().all() assert rows[column].nunique() == 1 - runner.survey._same_frame( - after.frame, run.manifest.population(after.version) - ) + runner.survey._same_frame(after.frame, run.manifest.population(after.version)) runner.atomic.same_replayed_population( case.cold.financial_population, case.warm.financial_population ) @@ -233,3 +251,44 @@ def wrapper_profile(frame, event, arg): finally: sys.setprofile(previous) assert fired == [True] + + +@pytest.mark.parametrize( + "defect,reason", + ( + ("payload", "GEOGRAPHY_VALIDATION_BINDING"), + ("producer", "GEOGRAPHY_VALIDATION_BINDING"), + ("type", "DETAIL_ARTIFACT_IDENTITY"), + ), +) +def test_financial_donor_requires_actual_geography_gate( + known_financial_run, defect, reason +): + run = known_financial_run.cold + node = run.compiled.graph.node(financial.DONOR_NODE) + artifacts = {} + for edge in node.artifact_inputs: + producer = run.manifest.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + run.store.load_bytes(key), edge.type, key, producer.key, NumericScope() + ) + # A detached consumer-context negative, using actual source-owned run keys. + # This does not pretend that constructing ArtifactValue issues an authority. + edge = financial._geography_edge() + original = artifacts[edge.name] + if defect == "payload": + artifacts[edge.name] = replace(original, payload=original.payload + b" ") + elif defect == "producer": + other = "0" * 64 if original.producer_key != "0" * 64 else "1" * 64 + artifacts[edge.name] = replace( + original, producer_key=other, key=opaque_artifact_key(other, edge.artifact) + ) + else: + artifacts[edge.name] = replace( + original, type=ArtifactType("invented.wrong_gate", 1) + ) + context = SimpleNamespace(node=node, sources={}, artifacts=artifacts) + kernel = run.kernels.get(financial.CurrentSurveyPredictorDonorFilterKernel.ref) + with pytest.raises(ValueError, match=reason): + kernel._qualified(context) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py index 8f2a5b75f..43dc09b50 100644 --- a/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py +++ b/packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py @@ -150,119 +150,140 @@ def _assert_complete_clone(before, after): } -def test_cold_and_required_replay_keep_raw_allocation_and_inherit_geography( - known_atomic_run, -): +def test_cold_and_required_replay_assign_after_complete_clone(known_atomic_run): case = known_atomic_run assert case.cold.manifest.key == case.warm.manifest.key assert all(record.hit for record in case.warm.manifest.nodes.values()) - # The wrapper first verifies its prefix, then adds geography and clone nodes. - # Only those additions are cold in the complete graph's first execution. assert all( not record.hit for name, record in case.cold.manifest.nodes.items() if name not in {survey.CREATE_NODE, survey.ALLOCATION_NODE} ) - for field in ("allocated_population", "geography_population", "clone_population"): + for field in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ): replay.same_replayed_population( getattr(case.cold, field), getattr(case.warm, field) ) - support = atomic.decode_atomic_support(case.payload) for run in (case.cold, case.warm): - raw, geography = run.allocated_population, run.geography_population - order = run.compiled.order + raw, observed_population = run.allocated_population, run.observed_population + expanded, geography = run.expanded_population, run.geography_population + assert run.clone_population is geography + assert raw.version == observed_population.version == survey.ALLOCATION_NODE + assert expanded.version == geography.version == clone.COMBINED_CLONE_NODE + assert len(run.compiled.order) == 9 stages = ( survey.ALLOCATION_NODE, projection.NODE, + clone.COMBINED_CLONE_NODE, + clone.COMBINED_CLONE_CLAIM_NODE, "geography.assign", "geography.derive", "geography.gate", - clone.COMBINED_CLONE_NODE, - "geography.clone_gate", ) - positions = tuple(order.index(name) for name in stages) + positions = tuple(run.compiled.order.index(name) for name in stages) assert positions == tuple(sorted(positions)) - assert raw is not geography and raw.frame is not geography.frame - assert raw.version == geography.version == survey.ALLOCATION_NODE + assert ( + clone.COMBINED_CLONE_CLAIM_NODE + in run.compiled.predecessors["geography.assign"] + ) rebuilt = reconstruction.reconstruct_atomic_survey_geography( run.preparation, raw, case.config ) - replay.same_replayed_population(rebuilt.population, geography) + for expected, actual in ( + (rebuilt.observed_population, observed_population), + (rebuilt.expanded_population, expanded), + (rebuilt.population, geography), + ): + replay.same_replayed_population(expected, actual) receipt, definition = ( json.loads(rebuilt.receipt), json.loads(rebuilt.definition), ) + assert receipt["protocol"] == "microcosm.us.atomic-survey-reconstruction.v2" + assert receipt["assignment_identity"] == list( + reconstruction.composition.ASSIGNMENT_IDENTITY + ) assert rebuilt.support_payload == case.payload assert receipt["support_sha256"] == case.config.support_sha256 - for field in ( + for name in ( "publisher_provenance_established", "source_admission_issued", "population_admission_issued", "release_eligible", ): - assert receipt[field] is False + assert receipt[name] is False assigned = tuple(definition["outputs"].values()) derived = tuple(layer["output"] for layer in definition["systems"][0]["layers"]) - extra = (*observed.COLUMNS, *assigned, *derived) - assert not set(extra) & set(raw.frame.table("household")) - assert set(geography.frame.table("household")) - set( + assert not set((*observed.COLUMNS, *assigned, *derived)) & set( raw.frame.table("household") - ) == set(extra) - for entity in raw.frame.entities: - pd.testing.assert_frame_equal( - geography.frame.table(entity).loc[:, raw.frame.table(entity).columns], - raw.frame.table(entity), - check_exact=True, - ) - assert geography.frame.schema == raw.frame.schema - assert geography.frame.entities == raw.frame.entities - assert geography.frame.links == raw.frame.links == () - assert geography.frame.metadata == raw.frame.metadata - assert geography.frame.mass_log == raw.frame.mass_log - pd.testing.assert_series_equal( - geography.frame.strata, raw.frame.strata, check_exact=True - ) - assert geography.mass_ledger == raw.mass_ledger - assert ( - geography.weight_kind - == raw.weight_kind - == {"household": WeightKind.IMPORTANCE} - ) - assert ( - geography.frame.weighted_entities - == raw.frame.weighted_entities - == ("household",) ) - np.testing.assert_array_equal( - geography.frame.weights_for("household").values, - raw.frame.weights_for("household").values, - ) - assert set(geography.design_weights) == set(raw.design_weights) == {"household"} - np.testing.assert_array_equal( - geography.design_weights["household"], raw.design_weights["household"] - ) - assert dict(raw.owners) == { - (entity, column): survey.ALLOCATION_NODE - for entity in raw.frame.entities - for column in raw.frame.table(entity) - } - assert dict(geography.owners) == { + assert not set((*assigned, *derived)) & set(expanded.frame.table("household")) + _assert_complete_clone(observed_population, expanded) + # Both ordinary projection steps preserve every incumbent cell, entity, + # membership, axis, weight, design anchor and context field exactly. + for before, after, added in ( + (raw, observed_population, observed.COLUMNS), + (expanded, geography, (*assigned, *derived)), + ): + assert set(after.frame.table("household")) - set( + before.frame.table("household") + ) == set(added) + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity).loc[ + :, before.frame.table(entity).columns + ], + before.frame.table(entity), + check_exact=True, + ) + assert after.frame.schema == before.frame.schema + assert after.frame.entities == before.frame.entities + assert after.frame.links == before.frame.links == () + assert after.frame.metadata == before.frame.metadata + assert after.frame.mass_log == before.frame.mass_log + assert after.mass_ledger == before.mass_ledger + assert after.weight_kind == before.weight_kind + assert after.frame.weighted_entities == before.frame.weighted_entities + pd.testing.assert_series_equal( + after.frame.strata, before.frame.strata, check_exact=True + ) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(observed_population.owners) == { **raw.owners, **{("household", name): projection.NODE for name in observed.COLUMNS}, + } + assert dict(geography.owners) == { + **expanded.owners, **{("household", name): "geography.assign" for name in assigned}, **{("household", name): "geography.derive" for name in derived}, } - # The manifest's latest allocation version is enriched; the separately - # returned raw allocation remains suitable for independent reconstruction. replay.same_replayed_frame( - geography.frame, run.manifest.population(survey.ALLOCATION_NODE) + observed_population.frame, run.manifest.population(survey.ALLOCATION_NODE) + ) + replay.same_replayed_frame( + geography.frame, run.manifest.population(clone.COMBINED_CLONE_NODE) ) households = geography.frame.table("household") - assert len(households) == 6 + assert len(households) == 12 + assert not households.duplicated( + list(reconstruction.composition.ASSIGNMENT_IDENTITY) + ).any() keys = households[observed.COLUMNS[0]].map(json.loads) acs = keys.map(lambda key: key[0] == "acs") - assert acs.sum() == 4 + assert acs.sum() == 8 assert households.loc[acs, "survey_observed_puma"].eq("0612345").all() assert households.loc[acs, "assigned_puma_geoid"].eq("0612345").all() assert households.loc[~acs, "survey_observed_puma"].isna().all() @@ -270,15 +291,24 @@ def test_cold_and_required_replay_keep_raw_allocation_and_inherit_geography( selected = keys.map( lambda key, native=native: key[0] == "asec" and key[-1] == native ) - assert selected.sum() == 1 - assert households.loc[selected, "survey_observed_state"].tolist() == [state] - assert households.loc[selected, "assigned_state_fips"].tolist() == [state] + assert selected.sum() == 2 + assert households.loc[selected, "survey_observed_state"].tolist() == [ + state, + state, + ] + assert households.loc[selected, "assigned_state_fips"].tolist() == [ + state, + state, + ] assert households.loc[acs, "assigned_state_fips"].eq("06").all() assert set(households.census_block_geoid) <= set(support.arrays["area"]) atomic.validate_geography(households, definition, {blocks.SYSTEM: support}) - for name in ("geography.gate", "geography.clone_gate"): - assert run.manifest.node(name).receipt["outcome"] == "pass" - _assert_complete_clone(geography, run.clone_population) + gate = run.manifest.node("geography.gate") + assert gate.receipt["outcome"] == "pass" + assert ( + run.store.load_bytes(gate.opaque_artifacts["validation"]) + == rebuilt.stages[-1].receipt + ) def test_changed_normalized_support_is_refused_by_exact_digest( @@ -311,6 +341,8 @@ def test_enriched_population_cannot_substitute_for_raw_allocation(known_atomic_r "change,reason", ( ("observed_cell", "SURVEY_POPULATION_REPLAY_"), + ("observed_snapshot", "SURVEY_POPULATION_REPLAY_"), + ("expanded_snapshot", "SURVEY_POPULATION_REPLAY_"), ("source_container", "ATOMIC_FINAL_RECONSTRUCTION"), ), ) @@ -335,6 +367,14 @@ def trace(frame, event, arg): if change == "observed_cell": households = result.geography_population.frame.table("household") households.loc[households.index[0], "survey_observed_state"] = "99" + elif change in {"observed_snapshot", "expanded_snapshot"}: + value = ( + result.observed_population + if change == "observed_snapshot" + else result.expanded_population + ) + households = value.frame.table("household") + households.loc[households.index[0], "survey_observed_state"] = "99" else: result.sources[blocks.SOURCE] = ( str(result.sources[blocks.SOURCE]) + ".changed" diff --git a/packages/microcosm-build/tests/test_us_postclone_geography_identity.py b/packages/microcosm-build/tests/test_us_postclone_geography_identity.py new file mode 100644 index 000000000..8b4499343 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_postclone_geography_identity.py @@ -0,0 +1,129 @@ +"""Invented identity controls using the maintained clone and assignment operators.""" + +import numpy as np +import pandas as pd +import pytest +from test_us_atomic_survey_clone_graph import invented_population + +from microcosm.build import atomic_geography as geography +from microcosm.build.us_runtime import atomic_block_support as blocks +from microcosm.build.us_runtime import puf_support +from microcosm.build.us_runtime.graph_atomic_survey_clone import ( + ASSIGNMENT_IDENTITY, + atomic_survey_clone_nodes, +) +from microcosm.build.us_runtime.graph_sources import frame_column_declarations + + +def _case(*, last_id=42): + original = invented_population(last_id=last_id) + cloned = puf_support.clone_us_frame_for_puf_support( + original, clone_attachment_fraction=1.0, clone_attachment_seed=0 + ) + ids = {name: "invented-" + name for name in ("district", "population", "puma")} + areas = [ + int(base) + i + for base in ("010010201001000", "020130001001000") + for i in range(64) + ] + payload = blocks.assemble_atomic_block_support( + block_population={area: 1 for area in areas}, + cd_by_block={area: 101 if area < 2 * 10**13 else 200 for area in areas}, + puma_by_tract={areas[0] // 10000: 100001, areas[64] // 10000: 200002}, + source_ids=ids, + ) + definition = blocks.assignment_definition( + identity=ASSIGNMENT_IDENTITY, + state_column="observed_state", + puma_column="observed_puma", + source_ids=ids, + seed=17, + ) + return ( + original, + cloned, + definition, + {blocks.SYSTEM: geography.decode_atomic_support(payload)}, + ) + + +def _draws(households, definition, supports): + result = geography.assign_atomic(households, definition, supports) + return ( + pd.concat([households.loc[:, list(ASSIGNMENT_IDENTITY)], result], axis=1) + .set_index(list(ASSIGNMENT_IDENTITY), verify_integrity=True) + .sort_index() + ) + + +@pytest.mark.parametrize("change", ("order", "subset", "cohort_stride")) +def test_same_source_clone_draw_survives_cohort_coordinate_changes(change): + _original, cloned, definition, supports = _case() + households = cloned.table("household") + expected = _draws(households, definition, supports) + if change == "order": + selected = households.iloc[::-1] + elif change == "subset": + selected = households.iloc[[6, 1, 4]] + else: + _other, alternate, same_definition, same_support = _case(last_id=9999) + assert same_definition == definition + assert same_support[blocks.SYSTEM].sha256 == supports[blocks.SYSTEM].sha256 + selected = alternate.table("household") + previous_ids = households.set_index(list(ASSIGNMENT_IDENTITY)).household_id + next_ids = selected.set_index(list(ASSIGNMENT_IDENTITY)).household_id + common = previous_ids.index.intersection(next_ids.index) + assert len(common) == 4 + assert np.any( + previous_ids.loc[common].to_numpy() != next_ids.loc[common].to_numpy() + ) + selected = selected.loc[ + pd.MultiIndex.from_frame(selected.loc[:, list(ASSIGNMENT_IDENTITY)]).isin( + common + ) + ] + actual = _draws(selected, definition, supports) + pd.testing.assert_frame_equal(actual, expected.loc[actual.index], check_exact=True) + + +def test_completed_clone_keys_permit_distinct_block_draws(): + _original, cloned, definition, supports = _case() + households = cloned.table("household") + assert not households.duplicated(list(ASSIGNMENT_IDENTITY)).any() + actual = _draws(households, definition, supports) + pairs = actual.census_block_geoid.unstack(ASSIGNMENT_IDENTITY[1]) + assert set(pairs.columns) == {0, 1} + assert pairs.notna().all().all() + # Fixed invented support and seed: require evidence of independent clone + # assignment, while permitting individual pairs to choose the same block. + assert pairs[0].ne(pairs[1]).any() + + +def test_required_unknown_state_survives_initial_clone_and_refuses_assignment(): + original, _cloned, definition, supports = _case() + original.table("household").loc[0, "observed_state"] = pd.NA + cloned = puf_support.clone_us_frame_for_puf_support( + original, clone_attachment_fraction=1.0, clone_attachment_seed=0 + ) + households = cloned.table("household") + missing = households.observed_state.isna() + assert missing.sum() == 2 + assert set(households.loc[missing, ASSIGNMENT_IDENTITY[1]]) == {0, 1} + with pytest.raises(ValueError, match="missing required observed geography"): + geography.assign_atomic(households, definition, supports) + + +@pytest.mark.parametrize("defect", ("missing_discriminator", "duplicate_key")) +def test_postclone_identity_contract_refuses_ambiguous_rows(defect): + original, cloned, definition, supports = _case() + if defect == "missing_discriminator": + definition = {**definition, "identity": [ASSIGNMENT_IDENTITY[0]]} + with pytest.raises(ValueError, match="stable source/clone identity"): + atomic_survey_clone_nodes( + definition, frame_column_declarations(original), base="source" + ) + else: + households = cloned.table("household").copy(deep=True) + households[ASSIGNMENT_IDENTITY[1]] = np.zeros(len(households), dtype=np.int64) + with pytest.raises(ValueError, match="identity"): + geography.assign_atomic(households, definition, supports) diff --git a/packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py b/packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py index f6ae13591..39613e695 100644 --- a/packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py +++ b/packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py @@ -37,7 +37,7 @@ def test_atomic_age_cold_and_required_replay_retain_raw_budget_and_geography(req age.numerical.CALIBRATION_NODE, } prefix_ids = set(case.cold.compiled.order) - assert len(prefix_ids) == 10 + assert len(prefix_ids) == 9 assert cold.manifest.key == warm.manifest.key assert cold.budget.payload == warm.budget.payload assert all(record.hit for record in warm.manifest.nodes.values()) @@ -53,7 +53,11 @@ def test_atomic_age_cold_and_required_replay_retain_raw_budget_and_geography(req assert "census_block_geoid" in geography_columns raw_identity = age.budgets._population_identity(case.cold.allocated_population) for run in runs: - assert len(run.compiled.order) == len(run.manifest.nodes) == 13 + assert len(run.compiled.order) == len(run.manifest.nodes) == 12 + assert ( + "geography.gate" + in run.compiled.predecessors[age.numerical.CALIBRATION_NODE] + ) assert set(run.compiled.order) == prefix_ids | age_ids assert tuple(name for name in run.compiled.order if name in prefix_ids) == ( case.cold.compiled.order diff --git a/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py b/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py index 0739b797d..e6c958cc9 100644 --- a/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py +++ b/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py @@ -6,6 +6,7 @@ import sys from pathlib import Path +import pandas as pd import pytest from test_us_graph_atomic_survey_population import known_atomic_run # noqa: F401 from test_us_survey_origin_budget import _assert_final_mutation_refused @@ -13,6 +14,7 @@ from microcosm.build.us_runtime import survey_atomic_geography as reconstruction from microcosm.build.us_runtime import survey_origin_budget as owner from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.frame import Frame def test_atomic_support_fifo_refuses_before_any_source_borrow(tmp_path, monkeypatch): @@ -79,15 +81,25 @@ def test_atomic_budget_binds_complete_geography_on_cold_and_required_replay( expected_receipt = json.loads(expected.receipt) # Keep the helper's own physical seal exact; compare independently replayed # objects with the maintained named null-storage convention instead. - assert expected_receipt.pop( - "population_sha256" - ) == reconstruction._population_stamp(expected.population) + for key, population in ( + ("population_sha256", expected.population), + ("observed_population_sha256", expected.observed_population), + ("expanded_population_sha256", expected.expanded_population), + ): + assert expected_receipt.pop(key) == reconstruction._population_stamp(population) + assert "preclone_population_semantic_sha256" not in binding + assert len(binding["postclone_geography_population_semantic_sha256"]) == 64 + assert binding["validation_receipt"] == json.loads(expected.stages[-1].receipt) assert ( binding["reconstruction_semantic_sha256"] == hashlib.sha256(owner._json(expected_receipt)).hexdigest() ) assert binding["identity_scope"] == { - "reconstruction_omitted_fields": ["population_sha256"], + "reconstruction_omitted_fields": [ + "population_sha256", + "observed_population_sha256", + "expanded_population_sha256", + ], "population_fields": [ "frame_sha256", "version", @@ -112,13 +124,25 @@ def test_atomic_budget_binds_complete_geography_on_cold_and_required_replay( assert warm.payload == budget.payload -@pytest.mark.parametrize("defect", ("geography_cell", "geography_owner")) -def test_atomic_budget_refuses_changed_geography_in_first_clone(atomic_budget, defect): +@pytest.mark.parametrize( + "defect,reason", + ( + ("geography_cell", "SURVEY_POPULATION_REPLAY_STRING_VALUE"), + ("geography_owner", "SURVEY_POPULATION_REPLAY_POPULATION_CONTEXT"), + ("clone_discriminator", "SURVEY_POPULATION_REPLAY_NATIVE_BITS"), + ), +) +def test_atomic_budget_refuses_changed_geography_in_first_clone( + atomic_budget, defect, reason +): case, _budget = atomic_budget arguments = _arguments(case) changed = reconstruction._copy_population(arguments["clone_population"]) if defect == "geography_cell": changed.frame.table("household").loc[0, "census_block_geoid"] = "0" * 15 + elif defect == "clone_discriminator": + table = changed.frame.table("household") + table.loc[0, "household_support_clone_index"] = 1 else: owners = dict(changed.owners) owners["household", "census_block_geoid"] = ( @@ -126,7 +150,88 @@ def test_atomic_budget_refuses_changed_geography_in_first_clone(atomic_budget, d ) object.__setattr__(changed, "owners", owners) arguments["clone_population"] = changed - with pytest.raises(ValueError, match="CLONE_FRAME_VALUES|POPULATION_OWNERS"): + with pytest.raises(ValueError, match=reason): + owner.freeze_survey_origin_budget(**arguments) + + +def test_atomic_budget_refuses_preclone_assignment_candidate(atomic_budget): + case, _budget = atomic_budget + run = case.cold + before = run.observed_population.frame + definition = reconstruction.blocks.assignment_definition( + identity=(reconstruction.observed.COLUMNS[0],), + state_column=reconstruction.observed.COLUMNS[1], + puma_column=reconstruction.observed.COLUMNS[2], + source_ids=dict(case.config.source_ids), + seed=case.config.seed, + ) + supports = { + reconstruction.blocks.SYSTEM: reconstruction.atomic.decode_atomic_support( + case.payload + ) + } + households = before.table("household") + assigned = pd.concat( + [ + households, + reconstruction.atomic.assign_atomic(households, definition, supports), + ], + axis=1, + ) + tables = { + entity: before.table(entity).copy(deep=True) for entity in before.entities + } + tables["household"] = pd.concat( + [ + assigned, + reconstruction.atomic.derive_geography(assigned, definition, supports), + ], + axis=1, + ) + old_geography = Frame( + tables, + before.schema, + {entity: before.weights_for(entity) for entity in before.weighted_entities}, + before.strata, + metadata=before.metadata, + mass_log=before.mass_log, + ) + old_clone = reconstruction.puf_support.clone_us_frame_for_puf_support( + old_geography, clone_attachment_fraction=1.0, clone_attachment_seed=0 + ) + # Isolate obsolete placement/ownership from unrelated storage differences + # introduced by a direct clone versus a ContentStore materialization. + candidate = reconstruction._copy_population(run.clone_population) + actual_households = candidate.frame.table("household") + old_households = old_clone.table("household") + geography_columns = tuple( + column for column in tables["household"] if column not in households + ) + assert geography_columns and "census_block_geoid" in geography_columns + for identity in ( + "household_id", + reconstruction.observed.COLUMNS[0], + "household_support_clone_index", + ): + assert old_households[identity].tolist() == actual_households[identity].tolist() + owners = dict(candidate.owners) + for column in geography_columns: + reference = actual_households[column] + actual_households[column] = pd.Series( + old_households[column].array, + index=reference.index, + dtype=reference.dtype, + name=column, + ) + assert actual_households[column].dtype == reference.dtype + owners["household", column] = owner.clone.COMBINED_CLONE_NODE + object.__setattr__(candidate, "owners", owners) + arguments = _arguments(case) + arguments["clone_population"] = candidate + with pytest.raises( + ValueError, + match="SURVEY_POPULATION_REPLAY_STRING_VALUE|SURVEY_POPULATION_REPLAY_POPULATION_CONTEXT", + ): owner.freeze_survey_origin_budget(**arguments) From 151166377036a6f4703ed8effdcb328c7699e416 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 16:43:54 -0400 Subject: [PATCH 070/255] Bind survey replay dependencies in budget producer checks --- ...stclone-budget-replay-fence4-20260910.json | 52 +++++++++++++++++++ .../build/us_runtime/survey_origin_budget.py | 1 + ...s_survey_origin_budget_atomic_geography.py | 15 +++--- ...us_survey_origin_budget_replay_producer.py | 25 +++++++++ 4 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 experiments/us-postclone-budget-replay-fence4-20260910.json create mode 100644 packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py diff --git a/experiments/us-postclone-budget-replay-fence4-20260910.json b/experiments/us-postclone-budget-replay-fence4-20260910.json new file mode 100644 index 000000000..517507d08 --- /dev/null +++ b/experiments/us-postclone-budget-replay-fence4-20260910.json @@ -0,0 +1,52 @@ +{ + "code_adopted_on_top_of": "1cebe3547543bf2917a7af7362bfbb22a05bf964", + "code_changes": [ + { + "after_sha256": "4010b6518b295ec7ad18d65e8a4af0e5d0eaa09fddce60e46f70a197e4d79706", + "before_sha256": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py" + }, + { + "after_sha256": "281be16a58d8d72e7ee00a3eb4e62632b7a210f9c3b914b3187e48c775927d16", + "before_sha256": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f", + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py" + }, + { + "after_sha256": "0f79edc1ab92e16998b41baf9f1735501ef875fddc92a049237699d982ed70a1", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py" + } + ], + "description": "Include the actual geography replay dependency in the budget producer fence and strengthen the preclone block counterexample while retaining valid postclone owners.", + "earlier_70_control_evidence_unchanged": true, + "guard_sha256": "7424ca92c6aca7e3ad2e76cbb24232c65e2bda6af6af23c8542a5c28c5b8f0ea", + "native_dataset_accepted": false, + "physical_postcheck_counts": { + "models": 0, + "resources": 12, + "source_and_owned": 513 + }, + "physical_postcheck_sha256": "301c7b2812f030fbd329641ae4ec90c32e91f7117fec2eb2b9387e17a6b3ddc9", + "receipt_sha256": "24d783cac659a3a76a166259d61ecdf83a4bfb95ee85da0101f5415ba9412fa2", + "release_eligible": false, + "resource_use": { + "cpu_seconds": 156.80812, + "peak_rss_bytes": 431046656, + "wall_seconds": 161.48273254197557 + }, + "source_map_sha256": "315a018c3a3521a172b3de48bfffd6abaa38a72d37a02e246ef0009b17809fbe", + "source_review_sha256": "60e949960540f69e234ab132789993d95dfa2451ba5a755cb39af4bb0bcab245", + "status": "four_scoped_controls_accepted", + "tests": { + "errors": 0, + "failed": 0, + "passed": 4, + "selectors": [ + "test_budget_producer_refuses_replaced_replay_dependency (3 parametrizations)", + "test_atomic_budget_refuses_preclone_assignment_candidate" + ], + "skipped": 0 + }, + "unexpected_refusals": {}, + "xml_sha256": "41334743e72be57619c50839304d10e48d67ade99e7c74710baa4a5d8335679e" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py index 218a1e7f0..6a72b771d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -139,6 +139,7 @@ def _modules(): sys.modules[support_source_id_column.__module__], sys.modules[clone.clone_us_frame_for_puf_support.__module__], geography, + geography.replay, financial_successor, geography.blocks, geography.atomic, diff --git a/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py b/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py index e6c958cc9..f06c80ddd 100644 --- a/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py +++ b/packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py @@ -199,8 +199,8 @@ def test_atomic_budget_refuses_preclone_assignment_candidate(atomic_budget): old_clone = reconstruction.puf_support.clone_us_frame_for_puf_support( old_geography, clone_attachment_fraction=1.0, clone_attachment_seed=0 ) - # Isolate obsolete placement/ownership from unrelated storage differences - # introduced by a direct clone versus a ContentStore materialization. + # Isolate obsolete placement values from unrelated storage differences + # and retain the actual postclone owners, design anchors and ledger. candidate = reconstruction._copy_population(run.clone_population) actual_households = candidate.frame.table("household") old_households = old_clone.table("household") @@ -214,7 +214,11 @@ def test_atomic_budget_refuses_preclone_assignment_candidate(atomic_budget): "household_support_clone_index", ): assert old_households[identity].tolist() == actual_households[identity].tolist() - owners = dict(candidate.owners) + expected_owners = tuple(sorted(candidate.owners.items())) + old_blocks = old_households["census_block_geoid"].tolist() + actual_blocks = actual_households["census_block_geoid"].tolist() + assert all(type(value) is str for value in (*old_blocks, *actual_blocks)) + assert any(a != b for a, b in zip(old_blocks, actual_blocks, strict=True)) for column in geography_columns: reference = actual_households[column] actual_households[column] = pd.Series( @@ -224,13 +228,12 @@ def test_atomic_budget_refuses_preclone_assignment_candidate(atomic_budget): name=column, ) assert actual_households[column].dtype == reference.dtype - owners["household", column] = owner.clone.COMBINED_CLONE_NODE - object.__setattr__(candidate, "owners", owners) + assert tuple(sorted(candidate.owners.items())) == expected_owners arguments = _arguments(case) arguments["clone_population"] = candidate with pytest.raises( ValueError, - match="SURVEY_POPULATION_REPLAY_STRING_VALUE|SURVEY_POPULATION_REPLAY_POPULATION_CONTEXT", + match="^SURVEY_POPULATION_REPLAY_STRING_VALUE$", ): owner.freeze_survey_origin_budget(**arguments) diff --git a/packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py b/packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py new file mode 100644 index 000000000..6b1d6067d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py @@ -0,0 +1,25 @@ +"""Cheap loaded-code producer refusals; no financial/source fixture request.""" + +import pytest + +from microcosm.build.us_runtime import survey_origin_budget as owner + + +@pytest.mark.parametrize( + "name", ("same_replayed_population", "same_replayed_frame", "_series") +) +def test_budget_producer_refuses_replaced_replay_dependency(monkeypatch, name): + # The real producer first accepts its own source/module baseline. Replacing + # a function with behavior-preserving delegation must still invalidate that + # baseline; this checks producer identity, without inventing a budget issuer. + owner._producer() + replay = owner.geography.replay + original = getattr(replay, name) + + def changed(*args, **kwargs): + return original(*args, **kwargs) + + monkeypatch.setattr(replay, name, changed) + assert getattr(replay, name) is not original + with pytest.raises(owner.SurveyOriginBudgetError, match="^PRODUCER_CHANGED$"): + owner._producer() From a2c50a9d8286ff9a589d6c51e5b23957b476d3fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 10 Sep 2026 18:25:24 -0400 Subject: [PATCH 071/255] Verify survey input ownership and calibration diagnostics --- docs/us-input-coverage-diagnostic.md | 3 + experiments/us-input-coverage25-20260910.json | 125 ++++ .../us-survey-catalogue-memo20-20260910.json | 32 ++ ...survey-numeric-diagnostics27-20260910.json | 38 ++ .../us_runtime/input_coverage_profile.py | 202 +++++++ .../us_runtime/population_input_coverage.py | 535 ++++++++++++++++++ .../survey_calibration_diagnostics.py | 6 + .../survey_population_preparation.py | 63 ++- .../test_us_population_input_coverage.py | 535 ++++++++++++++++++ .../tests/test_us_survey_calibration.py | 68 +++ ...test_us_survey_catalogue_immutable_memo.py | 255 +++++++++ 11 files changed, 1858 insertions(+), 4 deletions(-) create mode 100644 docs/us-input-coverage-diagnostic.md create mode 100644 experiments/us-input-coverage25-20260910.json create mode 100644 experiments/us-survey-catalogue-memo20-20260910.json create mode 100644 experiments/us-survey-numeric-diagnostics27-20260910.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py create mode 100644 packages/microcosm-build/tests/test_us_population_input_coverage.py create mode 100644 packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py diff --git a/docs/us-input-coverage-diagnostic.md b/docs/us-input-coverage-diagnostic.md new file mode 100644 index 000000000..1304d2fd0 --- /dev/null +++ b/docs/us-input-coverage-diagnostic.md @@ -0,0 +1,3 @@ +# US population input-coverage diagnostic + +[`diagnose_us_input_coverage`](../packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py) takes a `Population`, its `CompiledGraph` and attached `RunManifest`, and returns a typed `PopulationInputCoverage` report. Its default `USInputProfile.NATIONAL_CD` describes 161 national/congressional-district input names; [`required_us_inputs()`](../packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py) keeps the historical 163-name default. The national/CD profile omits only engine `block_geoid` and `tract_geoid`, while separately reporting assigned `census_block_geoid`; neither profile adds `employment_income_last_year`. The diagnostic checks complete declared current ownership and replayed storage, then reports actual entity grains, source-channel/clone groups, null and invalid values, typed weights, and masked-writer versus carried rows. Missing grain and applicability remain unresolved. It verifies consistency of supplied graph objects; source ancestry and source requalification remain the host's responsibility. [Twenty-five frozen controls](../experiments/us-input-coverage25-20260910.json) passed, including actual cold/required replay and late-mutation refusals, with 515 physical source/owned checks and zero model/resource inputs. This is scoped evidence, not maintained-CI validation: four eager calibration imports differ from the frozen run and retain their newer maintained versions. The report establishes neither complete input coverage, statistical signal, applicability, native build acceptance nor release eligibility. diff --git a/experiments/us-input-coverage25-20260910.json b/experiments/us-input-coverage25-20260910.json new file mode 100644 index 000000000..6ff56ed08 --- /dev/null +++ b/experiments/us-input-coverage25-20260910.json @@ -0,0 +1,125 @@ +{ + "api": { + "assigned_block_diagnostic_retained": "census_block_geoid", + "diagnostic_module": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py", + "employment_income_last_year_in_required_profiles": false, + "existing_release_gate_or_manifest_modified": false, + "historical_required_us_inputs_default_count": 163, + "national_cd_engine_omissions": [ + "block_geoid", + "tract_geoid" + ], + "national_congressional_district_diagnostic_default_count": 161, + "profile_module": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py", + "report_protocol": "microcosm.us.input_coverage_diagnostic.v1", + "signature": "diagnose_us_input_coverage(population: Population, *, compiled: CompiledGraph, manifest: RunManifest, profile: USInputProfile = USInputProfile.NATIONAL_CD) -> PopulationInputCoverage" + }, + "code": [ + { + "after_sha256": "88a8456ad02846d8f41af7ead743c86caa8d82b9b2c18d196feee663acb02a33", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py" + }, + { + "after_sha256": "34455450437b5e4b7ce0377522d2bf2d815ac0a9d6e554b25c25d690e196e318", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py" + }, + { + "after_sha256": "7271ca0197d1396dc42656f3220207b0ba96927c1cd11233543f72fcf57ce94d", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_population_input_coverage.py" + } + ], + "description": "Descriptive input coverage on a supplied actual graph Population and attached RunManifest; complete ownership, storage, origin/clone and writer-mask checks.", + "frozen_evidence": { + "cpu_seconds": 6.143905999999999, + "guard_sha256": "0fb8e3f9f25ec47d89d6a3f61c8483caa86ccdee83958f7e801a8be579c4a559", + "models": 0, + "peak_rss_bytes": 332939264, + "physical_postcheck_sha256": "017ccd3d1d97819c1d6332436449e0ba961a1873ce6962547be07743fe097fc7", + "receipt_sha256": "8fed087e9f44d87d47a2c3071653255c959cbfee53ca1166fd22994025ca8d43", + "resources": 0, + "selectors": { + "test_actual_masked_coverage_separates_origin_clone_grain_and_unknownness": 1, + "test_closed_profiles_match_manifest_without_changing_historical_default": 1, + "test_complete_current_ownership_refuses_wrong_carrier_or_writer": 5, + "test_declared_absence_is_not_an_applicability_exemption": 1, + "test_duplicate_grains_are_rejected_by_the_actual_frame_contract": 1, + "test_inconsistent_population_manifest_or_compiler_refuses": 5, + "test_last_manifest_borrow_mutation_refuses": 1, + "test_nonfinite_values_and_missing_grains_remain_unresolved": 1, + "test_population_version_must_be_an_actual_structural_version": 2, + "test_profile_omissions_do_not_omit_assigned_block_diagnostic": 1, + "test_required_replay_preserves_coverage_and_manifest_identity": 1, + "test_source_role_and_membership_are_checked_on_actual_rows": 3, + "test_unclaimed_physical_column_is_not_accepted_as_a_structural_carrier": 1, + "test_zero_weight_origin_and_zero_observations_remain_known": 1 + }, + "source_and_owned_physical_checks": 515, + "source_count": 507, + "source_map_sha256": "55af74ead417eeab1b0e977da24e91ed5de61e52cf5953cd060797f58bc40038", + "source_peer_proof_sha256": "e1b8ae0cadd118ca7d4cfd85af2f4fd707462d87ace779c94277a0ee87fbc4a3", + "source_peer_review_sha256": "5a3177a15b478da0a8dcda6030397483892b60ae4cfd7481aa10a4c889ad59e9", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 25 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.524979708017781, + "xml_sha256": "e5ea4cc97ceddbd3ad9153b35ab14dd2e7eb3ff13e0d878972f85e9a3e72ade8" + }, + "limitations": { + "actual_source_ancestry_verified": false, + "applicability_complete": false, + "calibration_modules_rolled_back": false, + "complete_population_input_coverage_proven": false, + "host_integration_added": false, + "native_build_or_model_evaluation_verified": false, + "release_eligible": false, + "statistical_signal_verified": false, + "supplied_graph_consistency_only": true + }, + "maintained_integration_execution_verified": false, + "maintained_source_comparison": { + "baseline_eager_modules_compared": 59, + "comparison_is_not_recomputed_current_import_closure": true, + "different_calibration_imports_preserved": [ + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "397f02916de1535de3bd0c34fdd71a2e685fe37f1a61ff4d389a8343f9642d4b", + "maintained_sha256": "716018cfbb136c631ce56724e58048c94202d6d41b64a56c9ba901cc586fd253", + "module": "microcosm.calibrate", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c5084a0dd45617922f8efca08560f243195eb4456430c1d87258e6240f379ab4", + "maintained_sha256": "8a400c46e4c94773dc0d83122f173c8c3bd405694d411989b10f792597b94196", + "module": "microcosm.calibrate.exact_k", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "379e4adbb3ed7cdd13f7c7c548da45163f2a50330c874ee51095d38243d392e6", + "maintained_sha256": "14608a5ee02132f06eaad4a59de3b051c8b8e1d07bd1768f4eb57f237173cef7", + "module": "microcosm.calibrate.gates", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c04df2f61544d4ba83ad467365333a69528ccf9b78913d449539603cd848b6cc", + "maintained_sha256": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "module": "microcosm.calibrate.solve", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py" + } + ], + "identical": 53, + "maintained_CI_pending": true, + "new_owners": 2 + }, + "proposed_adoption_base": "151166377036a6f4703ed8effdcb328c7699e416", + "status": "25_frozen_scoped_controls_accepted" +} diff --git a/experiments/us-survey-catalogue-memo20-20260910.json b/experiments/us-survey-catalogue-memo20-20260910.json new file mode 100644 index 000000000..ff6b93661 --- /dev/null +++ b/experiments/us-survey-catalogue-memo20-20260910.json @@ -0,0 +1,32 @@ +{ + "code": [ + { + "maintained_sha256": "508d18bac53690f8841e3eb2770bc2aa6079b9180ad0a31c65c5566f3304a206", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py" + }, + { + "maintained_sha256": "8a9742e108347916937317a0901a462c315d688b32795758972ff9554b07d32f", + "path": "packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py" + } + ], + "evidence": { + "cpu_seconds": 18.188797, + "model_sources": 0, + "peak_rss_bytes": 405815296, + "physical_postcheck_sha256": "fcd8dcdb6b706f65888b1e33add9c042302d51358597973db14901e0c70f17b8", + "receipt_sha256": "ee5279f9678d035b0d6ede176704b4d04196d1eea351adfba56510dba973f972", + "resources": 12, + "source_peer_review_sha256": "c5e0e1188fdac1e84929f46d8e13f5e42e33b286f86a121ddd16aa38222c7e1d", + "source_plus_owned": 514, + "tests_failed": 0, + "tests_passed": 20, + "unexpected_refusals": {}, + "wall_seconds": 18.8203088750015, + "xml_sha256": "04dec52f2e8ca14815b8368a7ac2023d2f08f752c94367f9a6d7b2189402077f" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Reuse a sealed immutable ACS catalogue value within its issued preparation. All source and producer checks, complete Frame seals and final checks remain; replacing roots or mutable contents takes the original hashing path. Native performance remains to be measured.", + "status": "scoped_controls_accepted" +} diff --git a/experiments/us-survey-numeric-diagnostics27-20260910.json b/experiments/us-survey-numeric-diagnostics27-20260910.json new file mode 100644 index 000000000..c74a4409e --- /dev/null +++ b/experiments/us-survey-numeric-diagnostics27-20260910.json @@ -0,0 +1,38 @@ +{ + "code": [ + { + "maintained_sha256": "22ff28977582c3b903ecf5d52b791a9f864a6a3c3df3b64541571e9199876b81", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py" + }, + { + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa", + "path": "packages/microcosm-build/tests/test_us_survey_calibration.py" + } + ], + "evidence": { + "cpu_seconds": 3.194314, + "model_sources": 0, + "peak_rss_bytes": 414924800, + "physical_postcheck_sha256": "e8a6f475e8bb8564c919ccfcbfce28182abd5b57e61029d0dab40404bae45d31", + "receipt_sha256": "50c1a09d4e71cffb00a375a703cb04623521a612478c34b2ac558c907bd4188b", + "resources": 0, + "source_peer_review_sha256": "4bfde0c677cecde4f2912a0a524c937292d33ee40ba35c10d5d29d34fa4c7b71", + "source_plus_owned": 512, + "tests_failed": 0, + "tests_passed": 27, + "unexpected_refusals": {}, + "wall_seconds": 3.6059712909627706, + "xml_sha256": "e8dc6d33a24fcf4c0a226216cb6be574b599c73f4b16cb666bb9db1724d35ba1" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Independently reconstruct four closed solver options when verifying calibration diagnostics: no supplied gate initialization, nonzero-count budget basis, no feasible-draw upper bound and no budget search. Real grouped Adam over five invented households and four altered-option refusals pass against maintained calibration source. Complete age graph and native calibration remain separate.", + "status": "scoped_controls_accepted", + "test_formatting": { + "accepted_sha256": "e8a5fd5eea56ac357c5e919de0206d7ea44ff77b5dbf3809bea214b5d0e01109", + "change": "Ruff parenthesized one import; no AST change.", + "complete_AST_equal": true, + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa" + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py new file mode 100644 index 000000000..21f7ad81f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py @@ -0,0 +1,202 @@ +"""Explicit input-name profiles; neither a release gate nor source admission. + +The historical release gate retains its existing 163-input default. This +lightweight declaration adds the national/CD projection without importing its +engine-dependent anti-rot helpers or reading package resources. Its exact source +manifest pin documents the extraction, not authority over a receiving Frame. +Grain/applicability are deliberately not guessed from engine variable names. +""" + +from enum import StrEnum + + +class USInputProfile(StrEnum): + HISTORICAL = "us_release_163_v1" + NATIONAL_CD = "us_national_cd_161_v1" + + +MANIFEST_SHA256 = "d96d0e98078906a9b47d47662e173553b1937851c7a8dd576010b50705ca1b1d" +HISTORICAL_REQUIRED_INPUTS = ( + "age", + "alimony_expense", + "alimony_income", + "attends_eligible_educational_institution_for_american_opportunity_credit", + "auto_loan_balance", + "auto_loan_interest", + "bank_account_assets", + "block_geoid", + "bond_assets", + "business_is_sstb", + "casualty_loss", + "charitable_cash_donations", + "charitable_non_cash_donations", + "child_support_expense", + "child_support_received", + "congressional_district_geoid", + "county_fips", + "cps_race", + "detailed_occupation_recode", + "disability_benefits", + "domestic_production_ald", + "educational_assistance", + "educator_expense", + "employment_income_before_lsr", + "estate_income", + "estate_income_would_be_qualified", + "farm_income", + "farm_operations_income", + "farm_operations_income_would_be_qualified", + "farm_rent_income", + "farm_rent_income_would_be_qualified", + "first_home_mortgage_balance", + "first_home_mortgage_interest", + "first_home_mortgage_origination_year", + "fsla_overtime_premium", + "has_american_opportunity_credit_1098_t_or_exception", + "has_american_opportunity_credit_institution_ein", + "has_champva_health_coverage_at_interview", + "has_esi", + "has_indian_health_service_coverage_at_interview", + "has_marketplace_health_coverage_at_interview", + "has_medicaid_health_coverage_at_interview", + "has_never_worked", + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "has_other_means_tested_health_coverage_at_interview", + "has_tricare_health_coverage_at_interview", + "has_va_health_coverage_at_interview", + "health_insurance_premiums", + "health_insurance_premiums_without_medicare_part_b", + "health_savings_account_ald", + "home_mortgage_interest", + "hourly_wage", + "hours_worked_last_week", + "household_vehicles_owned", + "household_vehicles_value", + "household_weight", + "immigration_status_str", + "investment_income_elected_form_4952", + "investment_interest_expense", + "is_blind", + "is_computer_scientist", + "is_disabled", + "is_enrolled_at_least_half_time_for_american_opportunity_credit", + "is_executive_administrative_professional", + "is_farmer_fisher", + "is_female", + "is_full_time_college_student", + "is_hispanic", + "is_household_head", + "is_incapable_of_self_care", + "is_military", + "is_paid_hourly", + "is_pregnant", + "is_pursuing_credential_for_american_opportunity_credit", + "is_self_employed", + "is_separated", + "is_surviving_spouse", + "is_union_member_or_covered", + "keogh_distributions", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "meets_ssi_disability_criteria", + "miscellaneous_income", + "net_worth", + "non_qualified_dividend_income", + "non_sch_d_capital_gains", + "other_health_insurance_premiums", + "other_medical_expenses", + "over_the_counter_health_expenses", + "own_children_in_household", + "partnership_s_corp_income_would_be_qualified", + "pre_subsidy_care_expenses", + "pre_subsidy_rent", + "previous_year_income_available", + "qualified_bdc_income", + "qualified_dividend_income", + "qualified_passenger_vehicle_loan_interest", + "qualified_reit_and_ptp_income", + "qualified_tuition_expenses", + "real_estate_taxes", + "receives_housing_assistance", + "rental_income", + "rental_income_would_be_qualified", + "roth_401k_contributions_desired", + "roth_ira_contributions_desired", + "salt_refund_income", + "schedule_d_capital_gain_distributions", + "selected_marketplace_plan_benchmark_ratio", + "self_employed_pension_contributions_desired", + "self_employment_income_before_lsr", + "self_employment_income_last_year", + "self_employment_income_would_be_qualified", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "spm_unit_energy_subsidy", + "spm_unit_pre_subsidy_childcare_expenses", + "spm_unit_tenure_type", + "ssn_card_type", + "sstb_self_employment_income_before_lsr", + "sstb_self_employment_income_would_be_qualified", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "state_fips", + "stock_assets", + "student_loan_interest", + "takes_up_aca_if_eligible", + "takes_up_eitc", + "takes_up_head_start_if_eligible", + "takes_up_housing_assistance_if_eligible", + "takes_up_medicaid_if_eligible", + "takes_up_medicare_if_eligible", + "takes_up_snap_if_eligible", + "takes_up_ssi_if_eligible", + "takes_up_tanf_if_eligible", + "takes_up_wic_if_eligible", + "tax_exempt_interest_income", + "tax_exempt_ira_distributions", + "tax_exempt_private_pension_income", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "taxable_sep_distributions", + "tenure_type", + "tip_income", + "tract_geoid", + "traditional_401k_contributions_desired", + "traditional_ira_contributions_desired", + "treasury_tipped_occupation_code", + "unadjusted_basis_qualified_property", + "unemployment_compensation", + "unrecaptured_section_1250_gain", + "unreimbursed_business_employee_expenses", + "veterans_benefits", + "w2_wages_from_qualified_business", + "weekly_hours_worked_before_lsr", + "weeks_unemployed", + "workers_compensation", + "would_file_taxes_voluntarily", +) +NATIONAL_CD_ENGINE_OMISSIONS = ("block_geoid", "tract_geoid") +NATIONAL_CD_REQUIRED_INPUTS = tuple( + name + for name in HISTORICAL_REQUIRED_INPUTS + if name not in NATIONAL_CD_ENGINE_OMISSIONS +) +# Independent build state, even when not an engine input in the selected profile. +ASSIGNED_BLOCK_COLUMN = "census_block_geoid" + + +def required_us_inputs( + profile: USInputProfile = USInputProfile.HISTORICAL, +) -> tuple[str, ...]: + """Return a closed profile; no custom name list or silent string fallback.""" + if type(profile) is not USInputProfile: + raise TypeError("US_INPUT_PROFILE_TYPE") + if profile is USInputProfile.HISTORICAL: + return HISTORICAL_REQUIRED_INPUTS + return NATIONAL_CD_REQUIRED_INPUTS diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py new file mode 100644 index 000000000..f02dce43f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py @@ -0,0 +1,535 @@ +"""Describe required inputs on one actual graph Population and attached manifest. + +This is a diagnostic, never an issuer, source qualifier, applicability model, +release gate or claim of non-default statistical signal. Call inside an actual +host's checked lifetime after its source/producer checks. Population and manifest +constructors are public: even coherent supplied objects cannot prove original +source ancestry. There is deliberately no supplied expected hash/list shortcut. + +The roster is a versioned coverage contract, not a claim that every name is +scientifically independent or consumed by every engine route. The current +source manifest does not declare grain or applicability. Present +columns report their actual grains; missing names retain unresolved grain. +Every row's applicability remains unresolved, including declared ABSENT cells. +Column ownership identifies the last writer, not the measurement source of all +rows: writer-mask counts and carried counts stay separate. Source names and +typed artifact ancestry below describe declarations/receipts, not fresh reads. + +Only the current group-linked survey schema is supported. Experimental link +tables refuse, rather than being dropped by the maintained replay comparator. +The single possible borrowed I/O is the attached manifest population lookup. +All detached output is built afterward, with final pure input seals. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import CompiledGraph, Population, RunManifest, compile_graph +from microcosm.graph.availability import execution_state +from microcosm.graph.canonical import canonical_json +from microcosm.graph.decl import Ownership, StructuralDelta +from microcosm.graph.population import _storage_parts, owned_ids +from microcosm.graph.serialize import graph_to_json +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + +from .input_coverage_profile import ( + ASSIGNED_BLOCK_COLUMN, + MANIFEST_SHA256, + USInputProfile, + required_us_inputs, +) +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, +) +from .survey_population_replay import same_replayed_frame + + +@dataclass(frozen=True) +class CoverageGroup: + entity: str + origin: str + clone: int + id_dtype: str + ordered_ids: tuple[int | str, ...] + source_id_dtype: str + ordered_source_ids: tuple[int | str, ...] + + +@dataclass(frozen=True) +class CoverageCounts: + group: int + known: int + unknown: int + invalid: int + writer_rows: int + carried_rows: int + applicability_unresolved: int + # Neither nulls nor Ownership.ABSENT are evidence of non-applicability. + not_applicable: int = 0 + + +@dataclass(frozen=True) +class InputCoverage: + name: str + entity: str | None + dtype: str | None + owner: str | None + declaration: str + writer_mask: str | None + counts: tuple[CoverageCounts, ...] + + +@dataclass(frozen=True) +class CoverageProducer: + node: str + key: str + kernel_ref: str + implementation: str + declared_sources: tuple[tuple[str, str], ...] + predecessors: tuple[tuple[str, str], ...] + typed_artifacts_json: str + + +@dataclass(frozen=True) +class PopulationInputCoverage: + profile: USInputProfile + version: str + graph_sha256: str + manifest_key: str + population_storage_sha256: str + groups: tuple[CoverageGroup, ...] + inputs: tuple[InputCoverage, ...] + producers: tuple[CoverageProducer, ...] + missing_inputs: tuple[str, ...] + ambiguous_grains: tuple[str, ...] + assigned_block: InputCoverage + block_storage_issues: tuple[str, ...] + protocol: str = "microcosm.us.input_coverage_diagnostic.v1" + profile_reference_sha256: str = MANIFEST_SHA256 + profile_authority: str = "tracked_input_coverage_contract_v1" + source_ancestry_verified: bool = False + applicability_complete: bool = False + statistical_signal_verified: bool = False + release_eligible: bool = False + + def to_bytes(self) -> bytes: + return canonical_json(asdict(self)) + + +def _require(condition, reason): + if not condition: + raise ValueError("US_INPUT_COVERAGE_" + reason) + + +def _series_stamp(series): + digest = hashlib.sha256() + digest.update(canonical_json((str(series.dtype), _axis_name_payload(series.name)))) + if pd.api.types.is_object_dtype(series.dtype): + # Typed scalar encoding, never process-specific PyObject addresses. + for value in series: + part = _encode_object_scalar(value) + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + else: + for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _axis_stamp(axis): + _require(not isinstance(axis, pd.MultiIndex), "MULTIINDEX") + return (type(axis).__name__, _series_stamp(pd.Series(axis.array, name=axis.name))) + + +def _frame_stamp(frame): + _require(frame.links == (), "LINK_TABLES_UNSUPPORTED") + tables = [] + for entity in frame.entities: + table = frame.table(entity) + tables.append( + ( + entity, + _axis_stamp(table.index), + _axis_stamp(table.columns), + table.flags.allows_duplicate_labels, + tuple((column, _series_stamp(table[column])) for column in table), + ) + ) + return hashlib.sha256( + canonical_json( + { + "schema": asdict(frame.schema), + "tables": tables, + "strata": ( + _axis_stamp(frame.strata.index), + _series_stamp(frame.strata), + ), + "weights": tuple( + ( + entity, + frame.weights_for(entity).kind.value, + frame.weights_for(entity).values.dtype.str, + frame.weights_for(entity).values.tobytes().hex(), + ) + for entity in frame.weighted_entities + ), + "metadata": _encode_frame_metadata(frame.metadata), + "mass_log": tuple(asdict(record) for record in frame.mass_log), + } + ) + ).hexdigest() + + +def _population_stamp(population): + return hashlib.sha256( + canonical_json( + { + "frame": _frame_stamp(population.frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "kinds": sorted( + (k, v.value) for k, v in population.weight_kind.items() + ), + "ledger": tuple(asdict(record) for record in population.mass_ledger), + "design": tuple( + (k, v.dtype.str, v.shape, v.tobytes().hex()) + for k, v in sorted(population.design_weights.items()) + ), + } + ) + ).hexdigest() + + +def _ids(series): + _require(not series.isna().any(), "UNKNOWN_ID") + if pd.api.types.is_integer_dtype(series.dtype) and not pd.api.types.is_bool_dtype( + series.dtype + ): + return tuple(int(value) for value in series) + _require(isinstance(series.dtype, pd.StringDtype), "ID_DTYPE") + return tuple(str(value) for value in series) + + +def _groups(frame): + groups, selections, roles = [], [], {} + for entity in frame.entities: + table = frame.table(entity) + names = ( + support_channel_column(entity), + support_clone_index_column(entity), + spine_source_id_column(entity), + ) + _require(all(name in table for name in names), "ORIGIN_COLUMNS:" + entity) + channel, clone, native = (table[name] for name in names) + _require( + not channel.isna().any() and set(channel) <= {"acs", "asec"}, + "ORIGIN:" + entity, + ) + _require( + pd.api.types.is_integer_dtype(clone.dtype) + and not pd.api.types.is_bool_dtype(clone.dtype) + and not clone.isna().any() + and set(clone) <= {0, 1}, + "CLONE:" + entity, + ) + ids = _ids(table[frame.schema.entity_id_column(entity)]) + native_ids = _ids(native) + _require(len(set(ids)) == len(ids), "DUPLICATE_ID:" + entity) + row_roles = tuple((str(a), int(b)) for a, b in zip(channel, clone, strict=True)) + _require( + len(set(zip(row_roles, native_ids, strict=True))) == len(ids), + "DUPLICATE_SOURCE_ROLE:" + entity, + ) + roles[entity] = dict(zip(ids, row_roles, strict=True)) + for origin, role in sorted(set(row_roles)): + selected = np.array( + [v == (origin, role) for v in row_roles], dtype=np.bool_ + ) + groups.append( + CoverageGroup( + entity, + origin, + role, + str(table[frame.schema.entity_id_column(entity)].dtype), + tuple(v for v, use in zip(ids, selected, strict=True) if use), + str(native.dtype), + tuple( + v for v, use in zip(native_ids, selected, strict=True) if use + ), + ) + ) + selections.append(selected) + person = frame.table(frame.schema.person_entity) + person_ids = _ids(person[frame.schema.person_id_column]) + for entity in frame.schema.group_entities: + members = _ids(person[frame.schema.membership_column(entity)]) + _require( + all( + member in roles[entity] + and roles[entity][member] == roles[frame.schema.person_entity][pid] + for pid, member in zip(person_ids, members, strict=True) + ), + "MEMBERSHIP_ORIGIN_CLONE:" + entity, + ) + return tuple(groups), tuple(selections) + + +def _writer(population, compiled, entity, name): + owner = population.owners[entity, name] + node = compiled.graph.node(owner) + outputs = [o for o in node.outputs if (o.entity, o.column) == (entity, name)] + if not outputs: + _require(node.structural is not StructuralDelta.NONE, "OWNER_DECLARATION") + return ( + owner, + "structural_carrier", + None, + np.zeros(population.frame.n(entity), dtype=np.bool_), + ) + _require(len(outputs) == 1, "OWNER_DECLARATION") + output = outputs[0] + table = population.frame.table(entity) + selected = ( + table[population.frame.schema.entity_id_column(entity)] + .isin(owned_ids(population, output)) + .to_numpy(dtype=np.bool_) + ) + if output.ownership is Ownership.ABSENT: + _require(table[name].loc[selected].isna().all(), "ABSENT_VALUE") + return owner, output.ownership.value, output.rows, selected + + +def _input(population, compiled, groups, selections, name, entity): + if entity is None: + return InputCoverage( + name, None, None, None, "missing_unresolved_grain", None, () + ) + frame, table = population.frame, population.frame.table(entity) + if name == "household_weight": + series = pd.Series(frame.weights_for(entity).values) + owner, declaration, mask = population.version, "typed_weight_carrier", None + writer = np.zeros(len(series), dtype=np.bool_) + else: + series = table[name] + owner, declaration, mask, writer = _writer(population, compiled, entity, name) + unknown = series.isna().to_numpy(dtype=np.bool_) + invalid = np.zeros(len(series), dtype=np.bool_) + if pd.api.types.is_numeric_dtype(series.dtype): + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + invalid = ~unknown & ~np.isfinite(numeric) + known = ~unknown & ~invalid + counts = tuple( + CoverageCounts( + i, + int(known[selected].sum()), + int(unknown[selected].sum()), + int(invalid[selected].sum()), + int(writer[selected].sum()), + int((~writer[selected]).sum()), + int(selected.sum()), + ) + for i, (group, selected) in enumerate(zip(groups, selections, strict=True)) + if group.entity == entity + ) + return InputCoverage( + name, entity, str(series.dtype), owner, declaration, mask, counts + ) + + +def diagnose_us_input_coverage( + population: Population, + *, + compiled: CompiledGraph, + manifest: RunManifest, + profile: USInputProfile = USInputProfile.NATIONAL_CD, +) -> PopulationInputCoverage: + """Bind descriptive coverage to actual tables and current graph ownership. + + This verifies supplied graph consistency, not actual-run/source issuance. + The final host must retain/requalify the real upstream owners separately. + No applicability exemption is accepted from caller metadata or a hash. + """ + names = required_us_inputs(profile) + _require( + type(population) is Population + and type(compiled) is CompiledGraph + and type(manifest) is RunManifest, + "TYPES", + ) + graph_json = graph_to_json(compiled.graph) + fresh = compile_graph(compiled.graph) + _require(compiled == fresh, "COMPILED_GRAPH") + _require( + manifest.country == compiled.graph.country == "us" + and tuple(manifest.nodes) == compiled.order, + "MANIFEST_ROSTER", + ) + _require(not manifest.known_failures, "MANIFEST_FAILURE") + for node_id in compiled.order: + record = manifest.node(node_id) + _require( + record.kernel_ref == compiled.graph.node(node_id).kernel + and execution_state(record.receipt) is None, + "PRODUCER_RECEIPT", + ) + frame = population.frame + frame.revalidate() + _require( + set(population.owners) + == { + (entity, str(column)) + for entity in frame.entities + for column in frame.table(entity) + }, + "PHYSICAL_OWNERS", + ) + _require( + compiled.versions.get(population.version) == population.version, + "POPULATION_VERSION", + ) + # CompiledGraph.owners contains explicit claims, not carried columns. + # Every structural step reowns its complete carrier; only writers in the + # current version replace those owners. Derive the roster from declarations + # along the base chain, never by accepting arbitrary physical columns. + expected_owners = {} + version = population.version + while True: + for (declared_version, entity, column), owner in compiled.owners.items(): + if declared_version == version: + expected_owners.setdefault( + (entity, column), + owner if version == population.version else population.version, + ) + holder = compiled.graph.node(version) + if holder.structural is StructuralDelta.CREATE: + break + version = holder.base + structural = { + (entity, frame.schema.entity_id_column(entity)) for entity in frame.entities + } + structural.update( + (frame.schema.person_entity, frame.schema.membership_column(entity)) + for entity in frame.schema.group_entities + ) + expected_owners.update(dict.fromkeys(structural, population.version)) + _require( + expected_owners and dict(population.owners) == expected_owners, "CURRENT_OWNERS" + ) + _require( + population.weight_kind + == { + entity: frame.weights_for(entity).kind for entity in frame.weighted_entities + }, + "WEIGHT_KIND", + ) + before, manifest_json = _population_stamp(population), manifest.to_json() + # A lazy manifest can perform store I/O here. Never construct detached output + # before this final borrow. Replay normalization is restricted to this edge. + materialized = manifest.population(population.version) + ledger = manifest.mass_ledger(population.version) + same_replayed_frame(frame, materialized) + _require( + canonical_json(tuple(asdict(r) for r in ledger)) + == canonical_json(tuple(asdict(r) for r in population.mass_ledger)), + "MASS_LEDGER", + ) + _require( + _population_stamp(population) == before and manifest.to_json() == manifest_json, + "BORROW_MUTATION", + ) + materialized_stamp = _frame_stamp(materialized) + groups, selections = _groups(frame) + inputs, missing, ambiguous = [], [], [] + for name in names: + entities = ( + ["household"] + if name == "household_weight" and "household" in frame.weighted_entities + else [entity for entity in frame.entities if name in frame.table(entity)] + ) + if name == "household_weight" and entities != ["household"]: + # Never accept a redundant ordinary column instead of typed weights. + entities = [] + if not entities: + missing.append(name) + inputs.append(_input(population, compiled, groups, selections, name, None)) + else: + if len(entities) > 1: + ambiguous.append(name) + inputs.extend( + _input(population, compiled, groups, selections, name, entity) + for entity in entities + ) + household = frame.table("household") + block = _input( + population, + compiled, + groups, + selections, + ASSIGNED_BLOCK_COLUMN, + "household" if ASSIGNED_BLOCK_COLUMN in household else None, + ) + block_issues = [] + if block.entity is None: + block_issues.append("missing_assigned_block") + else: + values = household[ASSIGNED_BLOCK_COLUMN] + if values.isna().any(): + block_issues.append("unknown_assigned_block") + if any( + not isinstance(v, str) or len(v) != 15 or not v.isascii() or not v.isdigit() + for v in values[values.notna()] + ): + block_issues.append("malformed_assigned_block") + sources = {source.name: source.codec for source in compiled.graph.sources} + producers = tuple( + CoverageProducer( + node_id, + manifest.node(node_id).key, + manifest.node(node_id).kernel_ref, + manifest.node(node_id).kernel_impl_hash, + tuple( + (name, sources[name]) for name in compiled.graph.node(node_id).sources + ), + tuple( + (name, manifest.node(name).key) + for name in compiled.predecessors[node_id] + ), + canonical_json(manifest.node(node_id).typed_artifacts).decode(), + ) + for node_id in compiled.order + ) + # Pure checks after decoding and grouping. No source/store/owner borrow follows. + _require( + _population_stamp(population) == before + and _frame_stamp(materialized) == materialized_stamp + and manifest.to_json() == manifest_json + and graph_to_json(compiled.graph) == graph_json + and compiled == fresh, + "FINAL_MUTATION", + ) + return PopulationInputCoverage( + profile, + population.version, + hashlib.sha256(graph_json.encode()).hexdigest(), + manifest.key, + before, + groups, + tuple(inputs), + producers, + tuple(missing), + tuple(ambiguous), + block, + tuple(block_issues), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py index 25d0bb9b7..c0d6cd4dd 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -135,6 +135,12 @@ def validate_survey_calibration_diagnostics( final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") options = { + # The closed grouped profile supplies no gates or record-budget search. + # Reconstruct these solver fields; never inherit them from the payload. + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, "grouped_preserve_zeros": { "enabled": True, "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 0d223ecdf..3bb7d8603 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -182,6 +182,42 @@ def _catalogue_digest(value): return digest.hexdigest() +def _catalogue_memo(value, expected_digest): + """Retain one proved immutable tree, never an unchecked caller digest. + + The exact-tuple/int/ASCII-str eligibility test also excludes mutable leaves + and subclasses. Strong references prevent identity reuse. Rehash once while + creating the memo to bind those exact immutable roots to the earlier seal. + A replacement tree or an ineligible shape keeps the original digest path. + """ + if not _catalogue_fast_path(value): + return None + if _catalogue_digest(value) != expected_digest: + return None + return (value[0], value[1], expected_digest) + + +def _memoized_catalogue_digest(value, memo=None, *, expected_digest=None): + """Reuse only the issued preparation's unchanged immutable record roots. + + This private memo is not source authority. The owning preparation still + performs every source/producer borrow and full physical Frame seal. Its + original nested digest is checked independently of the memo's digest. + """ + if ( + type(value) is tuple + and len(value) == 2 + and type(memo) is tuple + and len(memo) == 3 + and value[0] is memo[0] + and value[1] is memo[1] + and type(memo[2]) is str + and memo[2] == expected_digest + ): + return memo[2] + return _catalogue_digest(value) + + def _value(value): if isinstance(value, Enum): return value.value @@ -981,13 +1017,16 @@ class _State: plan: selection.CatalogueSelectionPlan plan_sha256: str nested: tuple + acs_catalogue_memo: tuple | None = None def _attached(state): return tuple(value.payload for value in (*state.catalogues, *state.native)) -def _nested_seals(catalogues, native): +def _nested_seals( + catalogues, native, *, catalogue_memo=None, expected_catalogue_digest=None +): """Pure final borrowed-object seals after all source/producer file checks. These inspect the actual retained issued owners, never construct issuer @@ -1011,7 +1050,11 @@ def _nested_seals(catalogues, native): for paths in acs.snapshots ] ), - _catalogue_digest((acs_cat.records, acs_cat.vacancies)), + _memoized_catalogue_digest( + (acs_cat.records, acs_cat.vacancies), + catalogue_memo, + expected_digest=expected_catalogue_digest, + ), ] for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): entry = module._ISSUED.get(id(value)) @@ -1045,7 +1088,13 @@ def _pure_final(state): ) _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") _require( - _nested_seals(state.catalogues, state.native) == state.nested, + _nested_seals( + state.catalogues, + state.native, + catalogue_memo=state.acs_catalogue_memo, + expected_catalogue_digest=state.nested[9], + ) + == state.nested, "NESTED_EVIDENCE_CHANGED", ) _require(_digest(_plan_document(state.plan)) == state.plan_sha256, "PLAN_CHANGED") @@ -1322,6 +1371,11 @@ def prepare_authenticated_survey_population( "release_eligible": False, } ) + nested = _nested_seals((acs, asec), (actual_acs, actual_asec)) + acs_owned = acs_catalogue._lookup(acs) + catalogue_memo = _catalogue_memo( + (acs_owned.records, acs_owned.vacancies), nested[9] + ) state = _State( root, files_before, @@ -1338,7 +1392,8 @@ def prepare_authenticated_survey_population( context, plan, _digest(_plan_document(plan)), - _nested_seals((acs, asec), (actual_acs, actual_asec)), + nested, + catalogue_memo, ) _validate(state) _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") diff --git a/packages/microcosm-build/tests/test_us_population_input_coverage.py b/packages/microcosm-build/tests/test_us_population_input_coverage.py new file mode 100644 index 000000000..9aa9301bb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_population_input_coverage.py @@ -0,0 +1,535 @@ +"""Invented CREATE/FILTER/masked-rewrite coverage, no engine or native data.""" + +from __future__ import annotations + +import json +import sys +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import input_coverage_profile as profile +from microcosm.build.us_runtime import population_input_coverage as coverage +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + Ownership, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.population import token_for_dtype + +BIG = 2**53 + 101 +SCHEMA = EntitySchema(group_entities=("household", "tax_unit")) + + +def _frame(spec): + household_ids = np.array([10, 20, 30, 40], dtype=np.int64) + household_members = [10, 10, 20, 30, 30, 40] + if spec.get("cross_origin"): + household_members[0] = 20 + people = pd.DataFrame( + { + "person_id": np.arange(BIG, BIG + 6, dtype=np.int64), + "person_household_id": np.array(household_members, dtype=np.int64), + "person_tax_unit_id": np.array([1, 1, 2, 3, 3, 4], dtype=np.int64), + "person_support_channel": pd.Series( + ["acs", "acs", "asec"] * 2, dtype="string" + ), + "person_support_clone_index": np.array([0, 0, 0, 1, 1, 1], dtype=np.int64), + "person_spine_source_id": np.array([1, 2, 3, 1, 2, 3], dtype=np.int64), + "is_detail": np.array([False] * 3 + [True] * 3), + "age": pd.Series([0.0, None, 40.0] * 2, dtype="float64"), + "taxable_interest_income": pd.Series([7.0, None, 0.0] * 2, dtype="float64"), + "bank_account_assets": pd.Series( + [0, pd.NA, 10, 0, pd.NA, 10], dtype="Int64" + ), + } + ) + if spec.get("unknown_origin"): + people.loc[0, "person_support_channel"] = pd.NA + households = pd.DataFrame( + { + "household_id": household_ids, + "household_support_channel": pd.Series(["acs", "asec"] * 2, dtype="string"), + "household_support_clone_index": np.array([0, 0, 1, 1], dtype=np.int64), + "household_spine_source_id": np.array([10, 20, 10, 20], dtype=np.int64), + "household_weight": np.full(4, 999.0), + "net_worth": np.array([0.0, 10.0, 0.0, 10.0]), + } + ) + if not spec.get("missing_block"): + households["census_block_geoid"] = pd.Series( + [ + "010010001001001", + "020010001001001", + "010010001001002", + "020010001001002", + ], + dtype="string", + ) + tax_units = pd.DataFrame( + { + "tax_unit_id": np.arange(1, 5, dtype=np.int64), + "tax_unit_support_channel": pd.Series(["acs", "asec"] * 2, dtype="string"), + "tax_unit_support_clone_index": np.array([0, 0, 1, 1], dtype=np.int64), + "tax_unit_spine_source_id": np.array([1, 2, 1, 2], dtype=np.int64), + } + ) + if spec.get("float_source_ids"): + households["household_spine_source_id"] = households[ + "household_spine_source_id" + ].astype("float64") + if spec.get("ambiguous_age"): + households["age"] = np.array([99.0] * 4) + if spec.get("invalid_numeric"): + people.loc[0, "age"] = np.inf + return Frame( + {"person": people, "household": households, "tax_unit": tax_units}, + SCHEMA, + {"household": Weights(np.array([0.0, 2.0, 0.0, 3.0]), WeightKind.DESIGN)}, + pd.Series(["invented"] * 6, name="stratum"), + metadata={"fixture": "invented-only"}, + ) + + +class InventedCreate(KernelBase): + ref = "test.coverage.create.v1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + spec = json.loads(context.sources["invented"].read_bytes()) + return KernelResult(frame=_frame(spec)) + + +class InventedKeep(KernelBase): + ref = "test.coverage.keep.v1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ) + + def run(self, context): + ids = context.tables["person"]["person_id"] + return KernelResult(keep=pd.Series(True, index=pd.Index(ids), dtype="bool")) + + +class InventedMasked(KernelBase): + ref = "test.coverage.masked.v1" + capabilities = Capabilities(determinism=Determinism.DETERMINISTIC) + + def run(self, context): + people = context.tables["person"] + ids = people.loc[people.is_detail, "person_id"] + return KernelResult( + columns={ + ("person", "taxable_interest_income"): pd.Series( + ([np.nan] * 3 if context.params["absent"] else [0.0, 100.0, 200.0]), + index=pd.Index(ids), + dtype="float64", + ) + } + ) + + +@pytest.fixture +def actual(tmp_path): + def build(**spec): + source = tmp_path / "invented.json" + source.write_text(json.dumps(spec)) + frame = _frame(spec) + structural = {(e, frame.schema.entity_id_column(e)) for e in frame.entities} + structural.update( + ("person", frame.schema.membership_column(e)) + for e in frame.schema.group_entities + ) + create = Node( + id="create", + kernel=InventedCreate.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=tuple( + Owned( + entity, column, token_for_dtype(frame.table(entity)[column].dtype) + ) + for entity in frame.entities + for column in frame.table(entity) + if (entity, column) not in structural + ), + ) + kept = Node( + id="kept", + kernel=InventedKeep.ref, + structural=StructuralDelta.FILTER, + base="create", + inputs=(Slice("person", ("is_detail",)),), + ) + masked = Node( + id="masked", + kernel=InventedMasked.ref, + population="kept", + params={"absent": bool(spec.get("absent_interest"))}, + inputs=(Slice("person", ("is_detail",)),), + outputs=( + Owned( + "person", + "taxable_interest_income", + "float64", + rows="is_detail", + rewrite=True, + ownership=Ownership.ABSENT + if spec.get("absent_interest") + else Ownership.PRODUCED, + ), + ), + ) + compiled = compile_graph( + Graph( + country="us", + sources=(SourceRef("invented", "raw-bytes-v1"),), + nodes=(create, kept, masked), + ) + ) + kernels = KernelRegistry() + kernels.register(InventedCreate()) + kernels.register(InventedKeep()) + kernels.register(InventedMasked()) + store = ContentStore(tmp_path / "store") + observed = {} + manifest = run_graph( + compiled, + sources={"invented": source}, + store=store, + kernels=kernels, + _population_observer=lambda node, population: observed.__setitem__( + node, population + ), + ) + return SimpleNamespace( + population=observed["masked"], + observed=observed, + manifest=manifest, + compiled=compiled, + store=store, + kernels=kernels, + source=source, + ) + + return build + + +def _diagnose(run): + return coverage.diagnose_us_input_coverage( + run.population, compiled=run.compiled, manifest=run.manifest + ) + + +def _input(report, name, entity="person"): + return next( + row for row in report.inputs if (row.name, row.entity) == (name, entity) + ) + + +def test_closed_profiles_match_manifest_without_changing_historical_default(): + path = ( + Path(profile.__file__).parents[1] + / "us/release_input_coverage_manifest.json" + ) + manifest = json.loads(path.read_text()) + expected = tuple( + name for name, row in manifest["columns"].items() if row["status"] == "required" + ) + assert profile.required_us_inputs() == expected + assert ( + len(expected) == 163 + and len(profile.required_us_inputs(profile.USInputProfile.NATIONAL_CD)) == 161 + ) + assert set(expected) - set(profile.NATIONAL_CD_REQUIRED_INPUTS) == { + "block_geoid", + "tract_geoid", + } + assert "employment_income_last_year" not in expected + with pytest.raises(TypeError, match="PROFILE_TYPE"): + profile.required_us_inputs("us_national_cd_161_v1") + + +def test_actual_masked_coverage_separates_origin_clone_grain_and_unknownness(actual): + run = actual() + report = _diagnose(run) + # These are actual executor owners, not a fixture-authored expected map. + assert run.population.version == "kept" + assert run.population.owners["person", "age"] == "kept" + assert run.population.owners["person", "person_id"] == "kept" + assert run.population.owners["person", "taxable_interest_income"] == "masked" + created = coverage.diagnose_us_input_coverage( + run.observed["create"], compiled=run.compiled, manifest=run.manifest + ) + assert created.version == "create" + assert _input(created, "age").owner == "create" + assert report.manifest_key == run.manifest.key + assert report.profile is profile.USInputProfile.NATIONAL_CD + assert report.profile_reference_sha256 == profile.MANIFEST_SHA256 + document = json.loads(report.to_bytes()) + assert document["protocol"] == "microcosm.us.input_coverage_diagnostic.v1" + assert document["release_eligible"] is False + assert {group.entity for group in report.groups} == { + "person", + "household", + "tax_unit", + } + first = report.groups[0] + assert first.ordered_ids == (BIG, BIG + 1) and first.id_dtype == "int64" + interest = _input(report, "taxable_interest_income") + assert interest.owner == "masked" and interest.writer_mask == "is_detail" + arm = { + (report.groups[c.group].origin, report.groups[c.group].clone): c + for c in interest.counts + } + assert (arm["acs", 0].known, arm["acs", 0].unknown, arm["acs", 0].carried_rows) == ( + 1, + 1, + 2, + ) + assert (arm["acs", 1].known, arm["acs", 1].writer_rows) == (2, 2) + assert run.population.frame.person.loc[0, "taxable_interest_income"] == 7.0 + assert pd.isna(run.population.frame.person.loc[1, "taxable_interest_income"]) + assets = _input(report, "bank_account_assets") + assert ( + sum(c.known for c in assets.counts) == 4 + and sum(c.unknown for c in assets.counts) == 2 + ) + assert all( + c.not_applicable == 0 and c.applicability_unresolved > 0 for c in assets.counts + ) + assert ( + _input(report, "first_home_mortgage_balance", None).declaration + == "missing_unresolved_grain" + ) + assert ( + not report.release_eligible + and not report.source_ancestry_verified + and not report.statistical_signal_verified + ) + producer = next(p for p in report.producers if p.node == "create") + assert producer.declared_sources == (("invented", "raw-bytes-v1"),) + assert producer.key == run.manifest.node("create").key + + +def test_required_replay_preserves_coverage_and_manifest_identity(actual): + run = actual() + first = _diagnose(run) + observed = {} + warm = run_graph( + run.compiled, + sources={"invented": run.source}, + store=run.store, + kernels=run.kernels, + resume="require", + _population_observer=lambda node, pop: observed.__setitem__(node, pop), + ) + assert all(row.hit for row in warm.nodes.values()) + second = coverage.diagnose_us_input_coverage( + observed["masked"], compiled=run.compiled, manifest=warm + ) + # Physical nullable backing may normalize at the named store boundary. + assert replace(first, population_storage_sha256="") == replace( + second, population_storage_sha256="" + ) + + +@pytest.mark.parametrize("part", ["owners", "manifest", "ledger", "compiled", "values"]) +def test_inconsistent_population_manifest_or_compiler_refuses(actual, part): + run = actual() + if part == "owners": + owners = dict(run.population.owners) + owners["person", "age"] = "masked" + run.population = replace(run.population, owners=owners) + elif part == "manifest": + run.manifest = replace( + run.manifest, nodes={"create": run.manifest.node("create")} + ) + elif part == "ledger": + run.manifest = replace(run.manifest, mass_ledgers={}) + elif part == "compiled": + run.compiled = replace(run.compiled, owners={}) + else: + tables = { + e: run.population.frame.table(e).copy(deep=True) + for e in run.population.frame.entities + } + tables["person"].loc[0, "age"] = 71.0 + frame = Frame( + tables, + SCHEMA, + {"household": run.population.frame.weights_for("household")}, + run.population.frame.strata, + metadata=run.population.frame.metadata, + mass_log=run.population.frame.mass_log, + ) + run.population = replace(run.population, frame=frame) + with pytest.raises((ValueError, KeyError)): + _diagnose(run) + + +@pytest.mark.parametrize( + "flag,reason", + [ + ("unknown_origin", "ORIGIN"), + ("cross_origin", "MEMBERSHIP_ORIGIN_CLONE"), + ("float_source_ids", "ID_DTYPE"), + ], +) +def test_source_role_and_membership_are_checked_on_actual_rows(actual, flag, reason): + with pytest.raises(ValueError, match=reason): + _diagnose(actual(**{flag: True})) + + +def test_zero_weight_origin_and_zero_observations_remain_known(actual): + run = actual() + report = _diagnose(run) + assert run.population.frame.weights_for("household").values.tolist() == [ + 0.0, + 2.0, + 0.0, + 3.0, + ] + weights = _input(report, "household_weight", "household") + assert weights.declaration == "typed_weight_carrier" + assert sum(c.known for c in weights.counts) == 4 + assert sum(c.known for c in _input(report, "net_worth", "household").counts) == 4 + assert not report.statistical_signal_verified + + +def test_profile_omissions_do_not_omit_assigned_block_diagnostic(actual): + report = _diagnose(actual(missing_block=True)) + assert ( + "block_geoid" not in report.missing_inputs + and "tract_geoid" not in report.missing_inputs + ) + assert report.block_storage_issues == ("missing_assigned_block",) + assert report.assigned_block.name == "census_block_geoid" + + +def test_duplicate_grains_are_rejected_by_the_actual_frame_contract(): + with pytest.raises( + ValueError, match="^Column names must be globally unique across entity tables " + ) as error: + _frame({"ambiguous_age": True}) + assert "'age' on ['person', 'household']" in str(error.value) + + +def test_nonfinite_values_and_missing_grains_remain_unresolved(actual): + report = _diagnose(actual(invalid_numeric=True)) + assert report.ambiguous_grains == () + age = _input(report, "age") + assert sum(c.invalid for c in age.counts) == 1 + assert sum(c.known for c in age.counts) == 3 + assert sum(c.unknown for c in age.counts) == 2 + missing = _input(report, "first_home_mortgage_balance", None) + assert missing.entity is None and missing.counts == () + assert missing.declaration == "missing_unresolved_grain" + + +def test_last_manifest_borrow_mutation_refuses(actual): + run = actual() + fired = False + + def callback(frame, event, arg): + nonlocal fired + if ( + event == "return" + and frame.f_code is type(run.manifest).population.__code__ + and not fired + ): + fired = True + run.population.frame.person["bank_account_assets"].array._data[0] = 123 + + old = sys.getprofile() + try: + sys.setprofile(callback) + with pytest.raises(ValueError): + _diagnose(run) + finally: + sys.setprofile(old) + assert fired + + +def test_declared_absence_is_not_an_applicability_exemption(actual): + report = _diagnose(actual(absent_interest=True)) + interest = _input(report, "taxable_interest_income") + assert interest.declaration == Ownership.ABSENT.value + detail = [c for c in interest.counts if report.groups[c.group].clone == 1] + assert sum(c.known for c in detail) == 0 + assert sum(c.unknown for c in detail) == 3 + assert sum(c.applicability_unresolved for c in detail) == 3 + assert sum(c.not_applicable for c in detail) == 0 + + +@pytest.mark.parametrize( + "entity,column,wrong_owner", + [ + ("person", "age", "create"), + ("person", "person_id", "create"), + ("person", "person_household_id", "create"), + ("household", "household_id", "create"), + ("person", "taxable_interest_income", "kept"), + ], +) +def test_complete_current_ownership_refuses_wrong_carrier_or_writer( + actual, entity, column, wrong_owner +): + run = actual() + assert run.population.owners[entity, column] != wrong_owner + owners = dict(run.population.owners) + owners[entity, column] = wrong_owner + run.population = replace(run.population, owners=owners) + with pytest.raises(ValueError, match="^US_INPUT_COVERAGE_CURRENT_OWNERS$"): + _diagnose(run) + + +@pytest.mark.parametrize("version", ["masked", "missing"]) +def test_population_version_must_be_an_actual_structural_version(actual, version): + run = actual() + run.population = replace(run.population, version=version) + with pytest.raises(ValueError, match="^US_INPUT_COVERAGE_POPULATION_VERSION$"): + _diagnose(run) + + +def test_unclaimed_physical_column_is_not_accepted_as_a_structural_carrier(actual): + run = actual() + original = run.population.frame + tables = { + entity: original.table(entity).copy(deep=True) for entity in original.entities + } + tables["person"]["undeclared_probe"] = np.zeros( + original.n("person"), dtype=np.int64 + ) + frame = Frame( + tables, + original.schema, + {entity: original.weights_for(entity) for entity in original.weighted_entities}, + original.strata, + metadata=original.metadata, + mass_log=original.mass_log, + ) + owners = dict(run.population.owners) + owners["person", "undeclared_probe"] = run.population.version + run.population = replace(run.population, frame=frame, owners=owners) + with pytest.raises(ValueError, match="^US_INPUT_COVERAGE_CURRENT_OWNERS$"): + _diagnose(run) diff --git a/packages/microcosm-build/tests/test_us_survey_calibration.py b/packages/microcosm-build/tests/test_us_survey_calibration.py index f05a3967f..27c1299e8 100644 --- a/packages/microcosm-build/tests/test_us_survey_calibration.py +++ b/packages/microcosm-build/tests/test_us_survey_calibration.py @@ -10,6 +10,9 @@ import test_us_survey_age_artifact as age_fixture from microcosm.build.us_runtime import graph_survey_calibration as stage +from microcosm.build.us_runtime import ( + survey_calibration_diagnostics as diagnostic_check, +) from microcosm.graph import ArtifactValue, KernelContext from microcosm.graph.canonical import canonical_json from microcosm.graph.kernel import Numeric, NumericScope @@ -66,6 +69,32 @@ def context(): ) +def _validated_diagnostics(value, output, payload): + anchors = { + name: output.receipt[name] + for name in ( + "budget_sha256", + "numeric_bounds_sha256", + "counts_sha256", + "accepted_weight_sha256", + "constraint_digest", + "weight_anchor", + "cap_enforcement", + "fixed_zero_rows", + ) + } + return diagnostic_check.validate_survey_calibration_diagnostics( + payload, + counts_payload=value.artifacts["counts"].payload, + bounds_payload=value.artifacts["bounds"].payload, + weights=output.weights.values, + registry=fixture.fixture_registry(), + epochs=value.params["epochs"], + learning_rate=value.params["learning_rate"], + anchors=anchors, + ) + + def test_real_solver_retains_zero_rows_and_returns_weights_only(): value = context() output = stage.SurveyAgeCalibrationKernel().run(value) @@ -80,6 +109,22 @@ def test_real_solver_retains_zero_rows_and_returns_weights_only(): assert diagnostics and output.receipt["fixed_zero_rows"] == 1 assert output.receipt["release_eligible"] is False assert output.receipt["source_admission"] == "required_from_country_runner" + assert { + name: diagnostics["options"][name] + for name in ( + "gate_initialization_supplied", + "budget_basis", + "feasible_draw_pi_hi", + "budget_search", + ) + } == { + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, + } + checked = _validated_diagnostics(value, output, output.artifacts["diagnostics"]) + assert checked["verification"]["optimizer_rerun"] is False @pytest.mark.parametrize( @@ -187,3 +232,26 @@ def test_projection_can_preserve_the_full_signed_id_domain(): assert stage.decode_numeric_survey_bounds( canonical_json(document) ).grouped.household_ids == tuple(document["household_ids"]) + + +@pytest.mark.parametrize( + "field,changed", + [ + ("gate_initialization_supplied", True), + ("budget_basis", "open_probability_mass"), + ("feasible_draw_pi_hi", 0.5), + ("budget_search", {}), + ], +) +def test_survey_diagnostics_recompute_closed_solver_option_values(field, changed): + value = context() + output = stage.SurveyAgeCalibrationKernel().run(value) + raw = output.artifacts["diagnostics"] + _validated_diagnostics(value, output, raw) + document = json.loads(raw) + assert document["options"][field] != changed + document["options"][field] = changed + with pytest.raises( + ValueError, match="^SURVEY_DIAGNOSTICS_RECOMPUTED_VALUES$" + ): + _validated_diagnostics(value, output, canonical_json(document)) diff --git a/packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py b/packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py new file mode 100644 index 000000000..ba7a0c8a4 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py @@ -0,0 +1,255 @@ +"""Immutable value reuse plus actual issuer final-borrow mutation controls. + +Supplied tuples exercise private value helpers only; they are never treated +as authenticated source catalogues. The last controls use real source +issuance over the maintained invented preparation fixture. +""" + +import hashlib +import json +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import survey_population_preparation as owner + + +def _records(): + person = ("1", 37, "1", "observed", "1", "observed", "100", "p.csv", 1) + household = ("2024HU0000001", 1, 1, 100, "h.csv", 1, (person,)) + vacancy = ("2024HU0000002", 1, 0, 0, "h.csv", 2, ()) + return ((household,), (vacancy,)) + + +def _oracle(value): + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _calls(action): + calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is owner._catalogue_digest.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = action() + finally: + sys.setprofile(previous) + return result, len(calls) + + +def test_same_immutable_roots_reuse_exact_digest_without_another_encode(): + records = _records() + expected = _oracle(records) + memo, creation_calls = _calls(lambda: owner._catalogue_memo(records, expected)) + assert creation_calls == 1 + assert type(memo) is tuple + assert memo[0] is records[0] and memo[1] is records[1] + assert memo[2] == expected + result, calls = _calls( + lambda: [ + owner._memoized_catalogue_digest( + (records[0], records[1]), memo, expected_digest=expected + ) + for _ in range(3) + ] + ) + assert result == [expected] * 3 and calls == 0 + + +def test_equal_replacement_roots_miss_without_changing_digest_or_retained_memo(): + records = _records() + expected = _oracle(records) + memo = owner._catalogue_memo(records, expected) + replacement = (tuple(list(records[0])), tuple(list(records[1]))) + assert replacement == records + assert replacement[0] is not records[0] and replacement[1] is not records[1] + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + replacement, memo, expected_digest=expected + ) + ) + assert result == expected and calls == 1 + assert memo[0] is records[0] and memo[1] is records[1] + + +@pytest.mark.parametrize("group", [0, 1]) +def test_changed_immutable_record_root_misses_and_changes_digest(group): + records = _records() + expected = _oracle(records) + memo = owner._catalogue_memo(records, expected) + row = records[group][0] + changed = list(records) + changed[group] = ((row[0] + "different", *row[1:]),) + changed = tuple(changed) + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + changed, memo, expected_digest=expected + ) + ) + assert calls == 1 and result == _oracle(changed) and result != expected + + +@pytest.mark.parametrize( + "kind", + [ + "group_list", + "record_list", + "people_list", + "person_list", + "tuple_subclass", + "str_subclass", + "int_subclass", + "non_ascii", + "long_ascii", + "large_integer", + ], +) +def test_ineligible_shapes_and_subclasses_keep_original_digest_path(kind): + class TupleSubclass(tuple): + pass + + class StrSubclass(str): + pass + + class IntSubclass(int): + pass + + records = _records() + row = records[0][0] + if kind == "group_list": + records = (list(records[0]), records[1]) + elif kind == "record_list": + records = ((list(row),), records[1]) + elif kind == "people_list": + records = (((*row[:6], list(row[6])),), records[1]) + elif kind == "person_list": + records = (((*row[:6], (list(row[6][0]),)),), records[1]) + elif kind == "tuple_subclass": + records = (TupleSubclass(records[0]), records[1]) + elif kind == "str_subclass": + records = (((StrSubclass(row[0]), *row[1:]),), records[1]) + elif kind == "int_subclass": + records = (((row[0], IntSubclass(row[1]), *row[2:]),), records[1]) + elif kind == "non_ascii": + records = ((("é", *row[1:]),), records[1]) + elif kind == "long_ascii": + records = ((("a" * 65_537, *row[1:]),), records[1]) + else: + records = (((row[0], 2**63, *row[2:]),), records[1]) + expected = _oracle(records) + memo = owner._catalogue_memo(records, expected) + assert memo is None + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + records, memo, expected_digest=expected + ) + ) + assert result == expected and calls == 1 + + +def test_nested_mutable_value_is_rehashed_after_in_place_change(): + records = _records() + row = records[0][0] + person = list(row[6][0]) + records = (((*row[:6], (person,)),), records[1]) + expected = _oracle(records) + memo = owner._catalogue_memo(records, expected) + assert memo is None + person[1] += 1 + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + records, memo, expected_digest=expected + ) + ) + assert calls == 1 and result == _oracle(records) and result != expected + + +def test_memo_creation_binds_exact_original_digest(): + records = _records() + assert owner._catalogue_memo(records, "0" * 64) is None + + +def test_swapped_complete_memo_entry_misses(): + records = _records() + expected = _oracle(records) + row = records[0][0] + other = (((row[0] + "other", *row[1:]),), records[1]) + memo = owner._catalogue_memo(other, _oracle(other)) + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + records, memo, expected_digest=expected + ) + ) + assert result == expected and calls == 1 + + +def test_cached_digest_drift_cannot_replace_original_nested_seal(): + records = _records() + expected = _oracle(records) + memo = owner._catalogue_memo(records, expected) + wrong = (*memo[:2], "0" * 64) + result, calls = _calls( + lambda: owner._memoized_catalogue_digest( + records, wrong, expected_digest=expected + ) + ) + assert result == expected and calls == 1 + + +@pytest.fixture(scope="module") +def actual_preparation(tmp_path_factory): + with pytest.MonkeyPatch.context() as monkeypatch: + arguments = fixture(tmp_path_factory.mktemp("catalogue-memo"), monkeypatch) + result = owner.prepare_authenticated_survey_population(**arguments) + state = owner._ISSUED[id(result)][2] + assert type(state.acs_catalogue_memo) is tuple + yield result, state + result.validate() + + +@pytest.mark.parametrize("field", ["records", "vacancies"]) +def test_actual_catalogue_replacement_at_final_producer_return_refuses( + actual_preparation, field +): + result, state = actual_preparation + catalogue = owner.acs_catalogue._lookup(state.catalogues[0]) + original = getattr(catalogue, field) + assert original + row = original[0] + replacement = ((row[0] + "different", *row[1:]), *original[1:]) + fired = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_code is owner._producer.__code__ + and frame.f_back.f_code is owner._validate.__code__ + and not fired + ): + fired.append(True) + object.__setattr__(catalogue, field, replacement) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.SurveyPopulationPreparationError, match="NESTED_EVIDENCE_CHANGED" + ): + result.checked_view() + finally: + sys.setprofile(previous) + object.__setattr__(catalogue, field, original) + assert fired == [True] + assert getattr(catalogue, field) is original From 7bfdb82b6f5b299ffc1131a941de2b4527ea4ee8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:13:01 -0400 Subject: [PATCH 072/255] Add the ASEC v3-to-v4 exact-source restoration helper and its checkpoint tests The restoration rebuilds the Frame with the input's mass log and metadata so a v4 checkpoint keeps its weight history (regression parametrized over an empty log, a declared mass change, and metadata forwarding). The two ASEC source test modules that test_us_asec_coverage_authentication imports were never committed; CI failed on their absence. The helper is classified as bounded I/O with no population treatment in the runtime inventory. --- .../build/us_runtime/asec_raw_stage_v4.py | 194 ++++ .../tests/test_us_asec_checkpoint.py | 87 ++ .../tests/test_us_asec_demographic_source.py | 584 ++++++++++++ .../test_us_asec_household_coverage_fields.py | 846 ++++++++++++++++++ .../tests/test_us_spine_blindness.py | 1 + 5 files changed, 1712 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py create mode 100644 packages/microcosm-build/tests/test_us_asec_demographic_source.py create mode 100644 packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py new file mode 100644 index 000000000..5e60f0a0e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py @@ -0,0 +1,194 @@ +"""Explicit local v3-to-v4 exact-source restoration, never schema relabeling.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import platform +import re +import tempfile +from collections.abc import Mapping +from importlib.metadata import version +from pathlib import Path + +from microcosm.build.frame_checkpoint import write_frame_checkpoint +from microcosm.build.us_runtime import asec_checkpoint, reported_coverage_source +from microcosm.frame import Frame + + +def _file_sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def restore_asec_raw_stage_v4( + checkpoint_path: str | Path, + *, + expected_sha256: str, + coverage_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Authenticate and restore all seven recodes into a new local bundle. + + The input is strictly v3 and operator-untouched. Every pooled income year + must have an explicit local official member or ZIP; this API never fetches. + Existing nonmissing recodes must agree with that source. The output directory + must not exist, so neither historical artifacts nor receipts are overwritten. + + Whole-file hashes authenticate immutable local inputs relative to caller pins; + they do not establish trust in an unknown producer. Structural Frame identity + alone does not bind person values. Source code and serialization versions are + recorded, but no country model runs and no model accuracy is certified here. + """ + source_path = Path(checkpoint_path).expanduser().resolve() + destination = Path(output_dir).expanduser().resolve() + if destination.exists(): + raise FileExistsError(destination) + if not isinstance(expected_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", expected_sha256 + ): + raise ValueError("expected_sha256 must be a lowercase SHA-256 digest") + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 mismatch") + frame, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint(source_path) + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 changed during loading") + years = tuple(sorted({int(year) for year in frame.table("person")["source_year"]})) + receipt_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if set(years) != receipt_years: + raise ValueError("ASEC v3 frame/source receipt year coverage differs") + if set(coverage_paths) != set(years) or any( + type(year) is not int or not isinstance(path, (str, Path)) + for year, path in coverage_paths.items() + ): + raise ValueError( + "explicit local coverage paths must cover exactly the pooled years" + ) + sidecar = reported_coverage_source.load_asec_reported_coverage_sources( + coverage_paths, income_years=years + ) + person = reported_coverage_source.fill_asec_reported_coverage_source( + frame.table("person"), sidecar + ) + restored = Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + binding = copy.deepcopy(metadata) + binding["schema_version"] = asec_checkpoint.ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION + pins = reported_coverage_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + source_pins = [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in years + ] + audit = sidecar.attrs["source_audit"] + for column in reported_coverage_source.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + binding["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": source_pins, + "audit": { + str(year): { + "rows": audit[year]["rows"], + **audit[year]["columns"][column], + } + for year in years + }, + } + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-v4-", dir=destination.parent + ) as temp: + temporary = Path(temp) + output_path = temporary / asec_checkpoint.ASEC_RAW_STAGE_CHECKPOINT_FILENAME + write_frame_checkpoint(output_path, restored, metadata=binding) + # Validate the actual serialization, not only the in-memory construction. + checked, checked_binding = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + output_path + ) + del checked, checked_binding + implementations = { + Path(module.__file__).name: _file_sha256(Path(module.__file__)) + for module in (asec_checkpoint, reported_coverage_source) + } + implementations[Path(__file__).name] = _file_sha256(Path(__file__)) + receipt = { + "schema": "microcosm.asec_raw_stage_restoration.v1", + "input_schema_version": 3, + "output_schema_version": 4, + "input_sha256": expected_sha256, + "output_sha256": _file_sha256(output_path), + "output_file": output_path.name, + "operation": "authenticated_exact_source_join", + "source_pins": source_pins, + "source_years": list(years), + "person_rows": len(person), + "implementation_sha256": implementations, + "runtime_versions": { + "python": platform.python_version(), + **{ + name: version(name) + for name in ("microcosm-build", "pandas", "numpy", "h5py") + }, + }, + "model_execution": False, + "release_certification": False, + "pipeline_lineage_sha256": metadata["pipeline_sha256"], + } + (temporary / "restoration.receipt.json").write_text( + json.dumps(receipt, sort_keys=True, indent=2, allow_nan=False) + "\n" + ) + if destination.exists(): + raise FileExistsError(destination) + os.rename(temporary, destination) + return receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument( + "--coverage", action="append", required=True, metavar="YEAR=LOCAL_PATH" + ) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + paths = {} + for item in args.coverage: + year_text, separator, path = item.partition("=") + if not separator or not year_text.isdecimal() or not path: + parser.error("--coverage must be YEAR=LOCAL_PATH") + year = int(year_text) + if year in paths: + parser.error("--coverage repeats a year") + paths[year] = Path(path) + receipt = restore_asec_raw_stage_v4( + args.input, + expected_sha256=args.expected_sha256, + coverage_paths=paths, + output_dir=args.output_dir, + ) + print(json.dumps(receipt, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index 8ea3569cc..3496eb470 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -118,23 +118,58 @@ def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> Non ) +@pytest.mark.parametrize( + "with_mass_history,forward_metadata", + [(False, False), (True, False), (True, True)], + ids=["empty_log", "declared_mass_change", "metadata_forwarding"], +) def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + with_mass_history: bool, + forward_metadata: bool, ) -> None: import hashlib from dataclasses import replace + from microcosm.build.us_runtime import asec_raw_stage_v4 as restoration from microcosm.build.us_runtime import reported_coverage_source from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 + from microcosm.frame import MassChange legacy = _raw_us_frame() + if with_mass_history: + original = legacy + weights = original.weights_for("household") + legacy = original.with_weights( + "household", + Weights(weights.values * 2.0, weights.kind), + mass=MassChange( + factor=2.0, reason="Invented raw-source design-weight rescaling" + ), + ) + assert original.mass_log == () + assert len(legacy.mass_log) == 1 + record = legacy.mass_log[0] + assert record.entity == "household" + assert record.old_total == float(weights.values.sum()) + assert record.new_total == 2.0 * record.old_total + assert record.declared_factor == 2.0 + assert record.reason == "Invented raw-source design-weight rescaling" + expected_mass_log = legacy.mass_log + assert bool(expected_mass_log) is with_mass_history binding = _raw_binding(legacy) binding["source_receipt"]["sources"].append( dict(binding["source_receipt"]["sources"][0], year=2023) ) input_path = tmp_path / "v3.h5" _write_checkpoint(input_path, legacy, metadata=binding) + loaded_input, _ = checkpoint_module.load_asec_raw_stage_checkpoint(input_path) + assert loaded_input.mass_log == expected_mass_log + np.testing.assert_array_equal( + loaded_input.weights_for("household").values, + legacy.weights_for("household").values, + ) input_sha = hashlib.sha256(input_path.read_bytes()).hexdigest() pins = {} paths = {} @@ -169,6 +204,49 @@ def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( monkeypatch.setattr( reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins ) + forwarded = [] + if forward_metadata: + # The checkpoint codec does not persist Frame.metadata. Supply a + # descriptive value only after the real v3 loader has validated its + # input, then observe the actual converter at the real writer boundary. + supplied_metadata = {"forwarding_probe": {"purpose": "invented observation"}} + real_load = checkpoint_module.load_asec_raw_stage_checkpoint + real_write = restoration.write_frame_checkpoint + + def load_with_descriptive_metadata(path): + loaded, binding = real_load(path) + assert not loaded.metadata + return ( + Frame( + {entity: loaded.table(entity) for entity in loaded.entities}, + loaded.schema, + { + entity: loaded.weights_for(entity) + for entity in loaded.weighted_entities + }, + loaded.strata, + mass_log=loaded.mass_log, + metadata=supplied_metadata, + ), + binding, + ) + + def observe_real_write(path, frame, **kwargs): + assert set(frame.metadata) == {"forwarding_probe"} + assert ( + dict(frame.metadata["forwarding_probe"]) + == supplied_metadata["forwarding_probe"] + ) + assert frame.mass_log == expected_mass_log + forwarded.append(True) + return real_write(path, frame, **kwargs) + + monkeypatch.setattr( + checkpoint_module, + "load_asec_raw_stage_checkpoint", + load_with_descriptive_metadata, + ) + monkeypatch.setattr(restoration, "write_frame_checkpoint", observe_real_write) output_dir = tmp_path / "restored" receipt = restore_asec_raw_stage_v4( input_path, @@ -176,11 +254,20 @@ def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( coverage_paths=paths, output_dir=output_dir, ) + assert forwarded == ([True] if forward_metadata else []) assert receipt["input_sha256"] == input_sha output = output_dir / "asec_raw_stage.checkpoint.h5" assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + assert restored.mass_log == expected_mass_log + # The codec does not reconstruct the separately supplied Frame metadata. + assert not restored.metadata + assert restored.weights_for("household").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + restored.weights_for("household").values, + loaded_input.weights_for("household").values, + ) assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] assert metadata["source_receipt"] == binding["source_receipt"] assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt diff --git a/packages/microcosm-build/tests/test_us_asec_demographic_source.py b/packages/microcosm-build/tests/test_us_asec_demographic_source.py new file mode 100644 index 000000000..ee1ef5549 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_demographic_source.py @@ -0,0 +1,584 @@ +"""Invented CSV cohorts only; no genuine source or population execution.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_current_money as money +from microcosm.build.us_runtime import asec_current_money_source as legacy +from microcosm.build.us_runtime import asec_demographic_source as demographic + + +def _classify( + *, + sex=(1, 2), + flag=(0, 4), + relationship=(1, 5), + sequence=(1, 2), + households=None, + years=None, +): + rows = len(sex) + return demographic.classify_asec_demographic_observations( + income_year=np.array( + years if years is not None else [2022] * rows, dtype=np.int64 + ), + household_id=np.array( + households if households is not None else [1] * rows, dtype=np.int64 + ), + a_sex=np.array(sex, dtype=np.int64), + axsex=np.array(flag, dtype=np.int64), + a_exprrp=np.array(relationship, dtype=np.int64), + p_seq=np.array(sequence, dtype=np.int64), + ) + + +def _member(tmp_path, **changes): + values = { + "PERIDNUM": ["0" * 21 + "1", "0" * 21 + "2"], + "PH_SEQ": [7, 7], + "A_LINENO": [1, 2], + "A_AGE": [55, 14], + "A_SEX": [1, 2], + "AXSEX": [0, 4], + "A_EXPRRP": [1, 5], + "P_SEQ": [1, 2], + # Neither family membership nor line order may supply headship. + "A_FAMREL": [1, 1], + } + values.update(changes) + path = tmp_path / "invented.csv" + pd.DataFrame(values).to_csv(path, index=False) + return path + + +def _read(path): + return demographic.read_demographic_member(path, rows=2, size=path.stat().st_size) + + +def _repoint_published_households(parent, attachment, monkeypatch, native_ids): + """Rewrite the parent roster's published household id, person by person. + + The invented parent publishes native id 7 in every cohort, so this is how + a fixture gives one cohort two published households without touching the + grouping column, whose distinct values the Frame requires to be exactly + the household table's ids. ``source_household_id`` is pooled source + provenance and so is inside the checkpoint's own structural identity: the + identity is rebound to the rewritten Frame rather than defeated. + """ + parent_identity = None + for path in (parent, attachment): + loaded = load_frame_checkpoint(path) + loaded.frame.person["source_household_id"] = np.array( + native_ids, dtype=np.int64 + ) + identity = frame_identity(loaded.frame) + for key in ("identity", "source_construction_identity"): + if key in loaded.metadata: + loaded.metadata[key] = identity.to_payload() + if path == attachment: + digest = money._sha(parent.read_bytes()) + loaded.metadata["parent_checkpoint_sha256"] = digest + receipt = loaded.metadata["household_observations"] + receipt["input_checkpoint_sha256"] = digest + receipt["input_structural_identity_sha256"] = parent_identity.sha256 + else: + parent_identity = identity + write_frame_checkpoint(path, loaded.frame, metadata=loaded.metadata) + monkeypatch.setattr( + legacy, + "_SOURCE_PINS", + ( + money._sha(parent.read_bytes()), + money._sha(attachment.read_bytes()), + legacy._SOURCE_PINS[2], + ), + ) + + +def _sources( + tmp_path, monkeypatch, *, reverse=True, relationship=(1, 5), native_ids=None +): + path = Path(__file__).with_name("test_us_asec_person_income_source.py") + spec = importlib.util.spec_from_file_location("demographic_invented_parent", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + parent, attachment, members = module.invented_sources(tmp_path, monkeypatch) + if native_ids is not None: + _repoint_published_households(parent, attachment, monkeypatch, native_ids) + source = legacy.load_authenticated_current_money_source(parent, attachment) + pins = [] + for year, member, archive, _, _rows, _ in demographic._MEMBER_PINS: + table = pd.read_csv(members[year], dtype=str, keep_default_na=False) + for i, row in table.iterrows(): + position = (int(row.PERIDNUM) - 1) % 2 + for name, values in { + "A_SEX": [1, 2], + "AXSEX": [0, 4], + "A_EXPRRP": relationship, + "P_SEQ": [1, 2], + }.items(): + table.loc[i, name] = str(values[position]) + if native_ids is not None: + table.loc[i, "PH_SEQ"] = str(native_ids[int(row.PERIDNUM) - 1]) + table = table.sort_values("PERIDNUM", ascending=not reverse) + table.to_csv(members[year], index=False) + pins.append( + ( + year, + member, + archive, + demographic._sha(members[year].read_bytes()), + 2, + members[year].stat().st_size, + ) + ) + monkeypatch.setattr(demographic, "_MEMBER_PINS", tuple(pins)) + return source, members + + +def test_exact_readset_and_declared_production_fields(tmp_path): + table = _read(_member(tmp_path)) + assert set(table) == set(demographic.ASEC_DEMOGRAPHIC_SOURCE_COLUMNS) + assert "A_FAMREL" not in table + contract = demographic.contract_document() + declared = contract["production_fields"] + assert { + v["column"] for v in declared["restored_observations"] + declared["derived"] + } == set(demographic.ALIASES + demographic.DERIVED) + assert contract["cohorts"]["2022"]["survey_year"] == 2023 + assert contract["cohorts"]["2024"]["survey_year"] == 2025 + assert not contract["wired_into_prepared_source"] + assert not contract["other_than_two_is_male"] + + +@pytest.mark.parametrize("field", demographic.OBSERVATIONS) +@pytest.mark.parametrize("token", ["", "NA", "1.0", "-1", "+1", " 1", "1 ", "1"]) +def test_missing_or_noncanonical_source_tokens_refuse_without_coercion( + tmp_path, field, token +): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBER_TOKEN_WIDTH" + ): + _read(_member(tmp_path, **{field: [token, "1"]})) + + +@pytest.mark.parametrize("sex", [0, 3, 9]) +def test_unknown_sex_never_becomes_male(sex): + result = _classify(sex=(sex, 2)) + assert result.states["asec_sex_binding_state"].tolist() == [0, 2] + assert result.states["asec_sex_unbound_reason"].tolist() == [1, 0] + + +@pytest.mark.parametrize("flag", [1, 2, 3, 5, 9]) +def test_unnamed_or_invalid_allocation_flag_leaves_sex_unbound(flag): + result = _classify(flag=(flag, 4)) + assert result.states["asec_sex_binding_state"].tolist() == [0, 2] + assert result.states["asec_sex_allocation_state"].tolist() == [0, 2] + + +def test_valid_sex_retains_allocation_provenance(): + result = _classify() + assert result.states["asec_sex_binding_state"].tolist() == [1, 2] + assert result.states["asec_sex_allocation_state"].tolist() == [1, 2] + + +def test_domain_diagnostics_distinguish_unnamed_and_out_of_range_codes(): + result = _classify(sex=(0, 2), flag=(1, 5), relationship=(1, 6), sequence=(1, 99)) + domains = result.diagnostics["by_cohort"]["2022"]["field_domains"] + assert domains["A_SEX"]["rows_outside_declared_range"] == 1 + assert domains["AXSEX"]["rows_in_unnamed_range_codes"] == 1 + assert domains["AXSEX"]["rows_outside_declared_range"] == 1 + assert domains["A_EXPRRP"]["rows_in_unnamed_range_codes"] == 1 + assert domains["P_SEQ"]["rows_outside_declared_range"] == 1 + + +@pytest.mark.parametrize( + "field,token", + [("A_SEX", "01"), ("AXSEX", "04"), ("A_EXPRRP", "001"), ("P_SEQ", "001")], +) +def test_source_tokens_cannot_exceed_printed_width(tmp_path, field, token): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBER_TOKEN_WIDTH" + ): + _read(_member(tmp_path, **{field: [token, "1"]})) + + +@pytest.mark.parametrize( + "years,households", [([2021, 2021], [1, 1]), ([2022, 2022], [0, 0])] +) +def test_unknown_cohort_or_nonpositive_household_refuses(years, households): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="CLASSIFY_COORDINATES" + ): + _classify(years=years, households=households) + + +@pytest.mark.parametrize( + "relationship,reason", + [((5, 13), [2, 2]), ((1, 2), [3, 3]), ((1, 6), [4, 1]), ((1, 0), [4, 1])], +) +def test_missing_duplicate_and_unknown_reference_do_not_use_sequence_fallback( + relationship, reason +): + result = _classify(relationship=relationship) + assert result.states["asec_household_reference_state"].tolist() == [0, 0] + assert result.states["asec_household_reference_unbound_reason"].tolist() == reason + assert ( + result.diagnostics["p_seq_crosscheck"]["unbound_households_with_one_p_seq_one"] + == 1 + ) + + +def test_sequence_disagreement_does_not_override_explicit_reference(): + result = _classify(relationship=(13, 2)) + assert result.states["asec_household_reference_state"].tolist() == [2, 1] + crosscheck = result.diagnostics["p_seq_crosscheck"] + assert crosscheck["p_seq_one_without_reference_code"] == 1 + assert crosscheck["reference_code_without_p_seq_one"] == 1 + assert not crosscheck["used_as_fallback"] + + +def test_sequence_crosscheck_compares_source_codes_even_when_headship_is_unbound(): + crosscheck = _classify(relationship=(1, 2)).diagnostics["p_seq_crosscheck"] + assert crosscheck["agreement_rows"] == 1 + assert crosscheck["p_seq_one_without_reference_code"] == 0 + assert crosscheck["reference_code_without_p_seq_one"] == 1 + + +def test_subfamily_role_and_row_order_do_not_supply_household_headship(): + inputs = dict( + sex=(1, 2, 1, 2), + flag=(0, 4, 0, 4), + relationship=(1, 5, 10, 7), + sequence=(3, 1, 2, 4), + ) + result = _classify(**inputs) + assert result.states["asec_household_reference_state"].tolist() == [1, 2, 2, 2] + reordered = _classify(**{k: tuple(reversed(v)) for k, v in inputs.items()}) + for name, values in result.states.items(): + np.testing.assert_array_equal(values[::-1], reordered.states[name]) + assert result.diagnostics == reordered.diagnostics + assert not result.diagnostics["family_relationship_field_read"] + + +@pytest.mark.parametrize( + "codes,expected", + [ + ([20, 21, 37, 38], [1, 2, 3, 3]), + ([20, 37, 37, 38], [0, 0, 3, 3]), + ([20, 99, 37, 38], [0, 0, 3, 3]), + ([21, 22, 37, 38], [0, 0, 3, 3]), + ([20, 20, 37, 38], [0, 0, 3, 3]), + ], +) +def test_acs_housing_units_and_both_group_quarters_codes_stay_distinct(codes, expected): + relshipp = np.array(codes, dtype=np.int64) + before = relshipp.copy() + states, diagnostics = demographic.acs_household_reference_states( + household_id=np.array([1, 1, 2, 3], dtype=np.int64), relshipp=relshipp + ) + assert states.tolist() == expected + np.testing.assert_array_equal(relshipp, before) + assert diagnostics["group_quarters_only_households"] == 2 + assert not diagnostics["manufactured_group_quarters_reference_person"] + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_three_cohort_restoration_joins_native_keys_and_preserves_parent( + tmp_path, monkeypatch, reverse +): + source, members = _sources(tmp_path, monkeypatch, reverse=reverse) + before = legacy._frame_signature(source.frame) + observations = demographic.load_authenticated_asec_demographic_source( + source, member_paths=members + ) + assert observations.array("asec_A_SEX").tolist() == [1, 2] * 3 + assert observations.array("asec_AXSEX").tolist() == [0, 4] * 3 + assert observations.array("asec_A_EXPRRP").tolist() == [1, 5] * 3 + assert [r["joined_rows"] for r in observations.receipt["sources"]] == [2] * 3 + assert set(observations.receipt["diagnostics"]["by_cohort"]) == { + "2022", + "2023", + "2024", + } + attached = demographic.attach_asec_demographic_source(source, observations) + attached.validate() + for entity in source.frame.entities: + actual = attached.frame.table(entity) + if entity == "person": + actual = actual.drop( + columns=list(demographic.ALIASES + demographic.DERIVED) + ) + pd.testing.assert_frame_equal( + actual, source.frame.table(entity), check_exact=True + ) + assert legacy._frame_signature(source.frame) == before + assert not observations.array("asec_A_SEX").flags.writeable + assert "is_female" not in attached.frame.person + assert "is_household_head" not in attached.frame.person + + +def test_restored_reference_can_disagree_with_unverified_incumbent_recode( + tmp_path, monkeypatch +): + source, members = _sources(tmp_path, monkeypatch, relationship=(13, 2)) + observations = demographic.load_authenticated_asec_demographic_source( + source, member_paths=members + ) + assert observations.array("asec_household_reference_state").tolist() == [2, 1] * 3 + assert [ + r["incumbent_relationship_crosscheck"]["mismatch_rows"] + for r in observations.receipt["sources"] + ] == [2] * 3 + + +def test_candidate_bundle_is_reconstructed_and_mutation_is_refused( + tmp_path, monkeypatch +): + source, members = _sources(tmp_path, monkeypatch) + destination = tmp_path / "demographic-bundle" + receipt = demographic.write_asec_demographic_source( + source, member_paths=members, output_dir=destination + ) + candidate = destination / demographic.FILENAME + observed = demographic.load_authenticated_asec_demographic_source( + source, member_paths=members, candidate_path=candidate + ) + assert observed.content_sha256 == receipt["content_sha256"] + payload = bytearray(candidate.read_bytes()) + payload[-1] ^= 1 + candidate.write_bytes(payload) + with pytest.raises( + demographic.DemographicSourceRefusalError, match="DEMOGRAPHIC_CANONICAL_BYTES" + ): + demographic.load_authenticated_asec_demographic_source( + source, member_paths=members, candidate_path=candidate + ) + + +def _verify(years, natives, groups): + return demographic.verify_household_membership( + income_year=np.array(years, dtype=np.int64), + source_household_id=np.array(natives, dtype=np.int64), + household_id=np.array(groups, dtype=np.int64), + ) + + +#: Three cohorts, each publishing one household, all under the same native id. +_REUSED_YEARS = [2022, 2022, 2023, 2023, 2024, 2024] +_REUSED_NATIVE_IDS = [7, 7, 7, 7, 7, 7] +_REUSED_GROUPS = [1, 1, 2, 2, 3, 3] + + +def test_native_household_id_reuse_across_cohorts_is_admitted_and_counted(): + result = _verify(_REUSED_YEARS, _REUSED_NATIVE_IDS, _REUSED_GROUPS) + assert result["bijective"] + assert result["grouping_households"] == result["published_households"] == 3 + assert result["membership_edges"] == 3 + assert result["native_household_ids_reused_across_cohorts"] == 1 + assert result["native_household_id_reuse_across_cohorts_is_admitted"] + assert result["published_household_key"] == ["income_year", "source_household_id"] + assert result["by_cohort"]["2023"] == { + "income_year": 2023, + "survey_year": 2024, + "rows": 2, + "grouping_households": 1, + "published_households": 1, + "native_household_ids": 1, + } + assert result["counters_zero_in_every_successful_receipt"] == [ + "grouping_households_spanning_multiple_cohorts", + "grouping_households_spanning_multiple_published_households", + "published_households_spanning_multiple_grouping_households", + ] + assert all(result[name] == 0 for name in demographic.MEMBERSHIP_REFUSAL_COUNTERS) + + +def test_membership_verification_does_not_depend_on_row_order(): + forward = _verify(_REUSED_YEARS, _REUSED_NATIVE_IDS, _REUSED_GROUPS) + order = [5, 0, 3, 2, 4, 1] + shuffled = _verify( + [_REUSED_YEARS[i] for i in order], + [_REUSED_NATIVE_IDS[i] for i in order], + [_REUSED_GROUPS[i] for i in order], + ) + assert forward == shuffled + + +@pytest.mark.parametrize( + "years,natives,groups,reason", + [ + # Two published households of one cohort collapsed into one grouping + # household: its single reference person would otherwise be read as + # the head of both, and the other published household would silently + # lose its own. + ([2022, 2022], [7, 8], [1, 1], "MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS"), + ( + [2022, 2022, 2022], + [7, 7, 8], + [1, 1, 1], + "MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS", + ), + # One published household split across two grouping households: each + # part is classified on a subset of the roster the reference-person + # cardinality is defined over. + ([2022, 2022], [7, 7], [1, 2], "MEMBERSHIP_SPLIT_NATIVE_HOUSEHOLDS"), + ([2022] * 3, [7, 7, 7], [1, 1, 2], "MEMBERSHIP_SPLIT_NATIVE_HOUSEHOLDS"), + # A grouping household spanning cohorts, with the native id reused and + # with two different native ids. + ([2022, 2023], [7, 7], [1, 1], "MEMBERSHIP_COHORT_COLLISION"), + ([2022, 2023], [7, 8], [1, 1], "MEMBERSHIP_COHORT_COLLISION"), + ], +) +def test_unequal_household_partitions_refuse_without_naming_a_household( + years, natives, groups, reason +): + with pytest.raises(demographic.DemographicSourceRefusalError) as error: + _verify(years, natives, groups) + assert str(error.value) == f"person_household_id: {reason}" + + +@pytest.mark.parametrize( + "years,natives,groups,message", + [ + ([2021, 2021], [7, 7], [1, 1], "income_year: MEMBERSHIP_COHORT"), + ([2022, 2022], [0, 7], [1, 1], "source_household_id: MEMBERSHIP_COORDINATE"), + ( + [2022, 2022], + [7, 7], + [0, 1], + "person_household_id: MEMBERSHIP_COORDINATE", + ), + ], +) +def test_membership_coordinates_must_be_declared_and_positive( + years, natives, groups, message +): + with pytest.raises(demographic.DemographicSourceRefusalError) as error: + _verify(years, natives, groups) + assert str(error.value) == message + + +@pytest.mark.parametrize("name", ["income_year", "source_household_id", "household_id"]) +def test_membership_refuses_non_integer_coordinate_arrays(name): + arrays = { + "income_year": np.array([2022, 2022], dtype=np.int64), + "source_household_id": np.array([7, 7], dtype=np.int64), + "household_id": np.array([1, 1], dtype=np.int64), + } + arrays[name] = arrays[name].astype(np.float64) + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBERSHIP_INPUT" + ): + demographic.verify_household_membership(**arrays) + + +def test_classifier_refuses_a_grouping_household_spanning_two_cohorts(): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="CLASSIFY_HOUSEHOLD_COHORT" + ): + _classify(years=[2022, 2023], households=[1, 1]) + + +@pytest.mark.parametrize("field", ["PH_SEQ", "A_LINENO", "A_AGE"]) +@pytest.mark.parametrize("token", ["", " ", " 1", "1.0", "-1", "+1", "1 ", "NA"]) +def test_blank_or_noncanonical_coordinate_tokens_refuse(tmp_path, field, token): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBER_INTEGER" + ): + _read(_member(tmp_path, **{field: [token, "1"]})) + + +def test_blank_person_key_token_refuses(tmp_path): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBER_PERSON_KEY" + ): + _read(_member(tmp_path, PERIDNUM=["", "0" * 21 + "2"])) + + +def test_nonpositive_published_household_coordinate_refuses(tmp_path): + with pytest.raises( + demographic.DemographicSourceRefusalError, match="MEMBER_COORDINATES" + ): + _read(_member(tmp_path, PH_SEQ=["0", "7"])) + + +def test_restoration_records_the_verified_household_membership(tmp_path, monkeypatch): + source, members = _sources(tmp_path, monkeypatch) + observations = demographic.load_authenticated_asec_demographic_source( + source, member_paths=members + ) + membership = observations.receipt["household_membership"] + assert observations.receipt["household_grouping_column"] == "person_household_id" + assert membership["grouping_column"] == "person_household_id" + assert membership["equivalence"] == "bidirectional_partition_bijection" + assert membership["verified_before_classification"] + assert membership["bijective"] + assert membership["rows"] == 6 + assert membership["grouping_households"] == membership["published_households"] == 3 + assert membership["membership_edges"] == 3 + # The parent publishes the same native household id in all three cohorts. + assert membership["native_household_ids_reused_across_cohorts"] == 1 + assert set(membership["by_cohort"]) == {"2022", "2023", "2024"} + assert all( + membership[name] == 0 for name in demographic.MEMBERSHIP_REFUSAL_COUNTERS + ) + assert ( + demographic.contract_document()["household_membership_verification"][ + "native_household_id_reuse_across_cohorts" + ] + == "admitted_and_counted" + ) + assert observations.array("asec_household_reference_state").tolist() == [1, 2] * 3 + + +def test_merged_published_households_refuse_before_the_demographic_reader( + tmp_path, monkeypatch +): + """Pin the layering the demographic fence restates. + + The 2022 cohort now publishes households 7 and 8 while the parent roster + still groups both persons together. The parent's own authentication + refuses that roster first: ``asec_household_observations._household_keys`` + requires the same equivalence, and it is reached from ``_scope``, so from + every ``AuthenticatedCurrentMoneySource.validate()`` the demographic + reader performs. The demographic module refuses the same shape on its own + evidence, so relaxing the parent fence would not silently admit a merged + household into a headship verdict. + """ + with pytest.raises(money.MoneyRefusalError, match="SOURCE_CONTRACT_REFUSAL"): + _sources(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + with pytest.raises(demographic.DemographicSourceRefusalError) as error: + _verify(_REUSED_YEARS, [7, 8, 7, 7, 7, 7], _REUSED_GROUPS) + assert str(error.value) == ( + "person_household_id: MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS" + ) + + +def test_restoration_receipt_names_its_structurally_zero_counters( + tmp_path, monkeypatch +): + source, members = _sources(tmp_path, monkeypatch) + observations = demographic.load_authenticated_asec_demographic_source( + source, member_paths=members + ) + for entry in observations.receipt["sources"]: + assert entry["counters_zero_in_every_successful_receipt"] == [ + "accepted_source_missing_tokens", + "incumbent_conflicts", + "native_key_or_age_conflicts", + "unreferenced_source_rows", + ] + assert entry["zero_counter_semantics"] == demographic.ZERO_COUNTER_SEMANTICS + assert entry["unreferenced_source_rows"] == 0 + assert entry["incumbent_conflicts"] == 0 + assert entry["native_key_or_age_conflicts"] == 0 diff --git a/packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py b/packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py new file mode 100644 index 000000000..1dfbb8c4b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_household_coverage_fields.py @@ -0,0 +1,846 @@ +"""Invented hhpub members for source-only household coverage observations.""" + +import _csv as native_csv +import copy +import csv +import gc +import hashlib +import importlib +import inspect +import io +import os +import pickle +import sys +import weakref +from collections.abc import Mapping +from contextlib import contextmanager +from dataclasses import FrozenInstanceError, replace +from types import SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import asec_household_coverage_fields as coverage +from microcosm.build.us_runtime import asec_original_household_weights as shared +from microcosm.build.us_runtime import native_household_origin as origin + + +def _csv( + rows, header=("H_SEQ", "H_HHTYPE", "HRHTYPE", "H_LIVQRT", "H_NUMPER", "EXTRA") +): + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerow(header) + writer.writerows(rows) + return output.getvalue().encode() + + +@pytest.fixture +def members(tmp_path, monkeypatch): + def make(rows=None, *, payload=None, years=(2024,), expected_rows=None): + rows = rows if rows is not None else [("00001", "1", "09", "08", "03", "")] + payload = payload if payload is not None else _csv(rows) + paths, pins = {}, [] + for year in years: + path = tmp_path / f"invented-{year}.csv" + path.write_bytes(payload) + paths[year] = path + pins.append( + origin.AsecNativeMemberPin( + income_year=year, + survey_year=year + 1, + canonical_member_id=f"invented/{year + 1}/household", + member_name=f"hhpub{str(year + 1)[-2:]}.csv", + archive_sha256=origin.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[ + year + ].zip_sha256, + member_sha256=hashlib.sha256(payload).hexdigest(), + size_bytes=len(payload), + rows=len(rows) if expected_rows is None else expected_rows, + ) + ) + monkeypatch.setattr(origin, "_ASEC_MEMBER_PINS", tuple(pins)) + return paths + + return make + + +def test_whole_cohort_lookup_preserves_literal_codes_and_reported_count(members): + source = coverage.load_authenticated_asec_household_coverage_fields( + members(years=(2022, 2023, 2024)) + ) + doc = source.document + assert [row["income_year"] for row in doc["records"]] == [2022, 2023, 2024] + assert len({row["origin_key"] for row in doc["records"]}) == 3 + row = source.households_for([(2024, 1)])[0] + assert [row[name] for name in coverage.COLUMNS] == ["00001", "1", "09", "08", "03"] + assert ( + row["field_states"]["HRHTYPE"]["label"] + == "Group quarters with actual families (This is new in 1994)" + ) + assert row["reported_person_count"] == 3 + assert row["coverage_interpretation"] == "unresolved" + assert doc["source_authenticated"] is True + assert doc["population_binding_authenticated"] is False + assert doc["release_eligible"] is False + assert "weights" not in doc + + +@pytest.mark.parametrize( + "name,token,status", + [ + ("H_HHTYPE", "1", "valid"), + ("H_HHTYPE", "01", "malformed"), + ("HRHTYPE", "9", "valid"), + ("HRHTYPE", "09", "valid"), + ("HRHTYPE", "11", "unlabelled"), + ("H_LIVQRT", "00", "unlabelled"), + ("H_NUMPER", "17", "unlabelled"), + ("H_NUMPER", "", "missing"), + ("H_NUMPER", "NA", "malformed"), + ("H_NUMPER", "3.0", "malformed"), + ("H_NUMPER", " 3", "malformed"), + ("H_NUMPER", "3\x00", "malformed"), + ("H_NUMPER", "-1", "malformed"), + ("HRHTYPE", "9", "malformed"), + ], +) +def test_literal_valid_missing_and_unresolved_states(members, name, token, status): + row = dict(zip(coverage.COLUMNS, ["1", "1", "09", "08", "3"], strict=True)) + row[name] = token + source = coverage.load_authenticated_asec_household_coverage_fields( + members([(*row.values(), "")]) + ) + actual = source.document["records"][0] + assert actual[name] == token + assert actual["field_states"][name]["code_status"] == status + if name == "H_NUMPER" and status != "valid": + assert actual["reported_person_count"] is None + + +@pytest.mark.parametrize( + "hh,hr,count,diagnostics", + [ + ( + "1", + "00", + "0", + [ + "interview_with_noninterview_household_type", + "interview_with_noninterview_person_count", + ], + ), + ( + "2", + "09", + "03", + [ + "noninterview_with_interview_household_type", + "noninterview_with_positive_person_count", + ], + ), + ("3", "00", "00", []), + ("", "09", "3", []), + ], +) +def test_interview_and_noninterview_contradictions_remain_diagnostics( + members, hh, hr, count, diagnostics +): + source = coverage.load_authenticated_asec_household_coverage_fields( + members([("1", hh, hr, "08", count, "")]) + ) + row = source.document["records"][0] + assert row["diagnostics"] == diagnostics + assert row["field_states"]["HRHTYPE"]["universe_status"] == ( + "in" if hh == "1" else "outside" if hh in ("2", "3") else "unresolved" + ) + if hh != "1" or count in ("0", "00"): + assert row["reported_person_count"] is None + + +def test_student_quarters_retains_dictionary_code_without_population_inference(members): + source = coverage.load_authenticated_asec_household_coverage_fields( + members([("1", "1", "10", "11", "1", "")]) + ) + row = source.document["records"][0] + assert ( + row["field_states"]["H_LIVQRT"]["label"] + == "Student quarters in college dormitory" + ) + assert row["diagnostics"] == ["student_quarters_dictionary_methodology_unresolved"] + assert row["coverage_interpretation"] == "unresolved" + assert ( + source.document["interpretation_notes"]["student_quarters"][ + "observed_presence_asserted" + ] + is False + ) + assert ( + source.document["interpretation_notes"]["student_quarters"]["methodology_url"] + == "https://www.census.gov/topics/income-poverty/guidance/group-quarters.html" + ) + + +@pytest.mark.parametrize("hr,quarters", [("01", "12"), ("09", "01"), ("10", "07")]) +def test_household_and_living_quarter_codes_do_not_assign_shared_or_exclusive_frame( + members, hr, quarters +): + row = coverage.load_authenticated_asec_household_coverage_fields( + members([("1", "1", hr, quarters, "2", "")]) + ).document["records"][0] + assert row["diagnostics"] == [] + assert row["coverage_interpretation"] == "unresolved" + assert "domain" not in row + + +def test_definition_pages_and_vintage_positions_are_exact_and_defensive(members): + source = coverage.load_authenticated_asec_household_coverage_fields(members()) + field = source.document["fields"]["H_NUMPER"] + assert field["position_by_survey_year"] == {"2023": 82, "2024": 82, "2025": 84} + assert field["pdf_page_1based_by_survey_year"] == {"2023": 3, "2024": 3, "2025": 10} + field["labels"]["0"] = "invented mutation" + assert ( + source.document["fields"]["H_NUMPER"]["labels"]["0"] == "Noninterview household" + ) + + +def test_exact_candidate_reconstruction_and_rehashed_mutation_refusal(members): + paths = members() + source = coverage.load_authenticated_asec_household_coverage_fields(paths) + assert ( + coverage.load_authenticated_asec_household_coverage_fields( + paths, candidate=source.payload + ).payload + == source.payload + ) + altered = source.document + altered["records"][0]["HRHTYPE"] = "01" + altered["projection_sha256"] = shared._sha(shared._encode(altered["records"])) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="CANDIDATE_MISMATCH" + ): + coverage.load_authenticated_asec_household_coverage_fields( + paths, candidate=shared._encode(altered) + ) + paths[2024].write_bytes(paths[2024].read_bytes().replace(b",09,", b",01,")) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="MEMBER_SHA256"): + coverage.load_authenticated_asec_household_coverage_fields( + paths, candidate=source.payload + ) + + +def test_constructor_and_defensive_views_cannot_mint_authority(members): + source = coverage.load_authenticated_asec_household_coverage_fields(members()) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="SOURCE_CONSTRUCTOR" + ): + coverage.AuthenticatedAsecHouseholdCoverageFields(source.payload) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="SOURCE_CONSTRUCTOR" + ): + replace(source, payload=source.payload) + with pytest.raises(FrozenInstanceError): + source.payload = b"{}" + source.households_for([(2024, 1)])[0]["H_NUMPER"] = "16" + assert source.households_for([(2024, 1)])[0]["H_NUMPER"] == "03" + object.__setattr__(source, "payload", source.payload + b" ") + with pytest.raises(coverage.HouseholdCoverageSourceError, match="SOURCE_MUTATION"): + _ = source.document + + +@pytest.mark.parametrize( + "roster", + [ + [], + [(2024, 2)], + [(2024, 1), (2024, 1)], + [(2023, 1)], + [(2024, True)], + [(2024, "1")], + ], +) +def test_native_lookup_refuses_unknown_duplicate_or_coerced_keys(members, roster): + source = coverage.load_authenticated_asec_household_coverage_fields(members()) + with pytest.raises(coverage.HouseholdCoverageSourceError): + source.households_for(roster) + + +@pytest.mark.parametrize( + "payload,expected", + [ + (b"H_SEQ,H_HHTYPE\n1,1\n", 1), + (b"H_SEQ,H_HHTYPE,HRHTYPE,H_LIVQRT,H_NUMPER,H_SEQ\n1,1,9,8,3,1\n", 1), + (_csv([("1", "1", "9", "8", "3")]), 1), + (_csv([("1", "1", "9", "8", "3", "", "extra")]), 1), + (_csv([("1", "1", "9", "8", "3", ""), ("01", "1", "9", "8", "3", "")]), 2), + (_csv([("", "1", "9", "8", "3", "")]), 1), + (_csv([("1", "1", "9", "8", "3", "")]), 2), + (b"H_SEQ,H_HHTYPE,HRHTYPE,H_LIVQRT,H_NUMPER\n1,1,9,8,\xff\n", 1), + ], +) +def test_complete_cohort_structure_and_encoding_refuse(members, payload, expected): + with pytest.raises(coverage.HouseholdCoverageSourceError): + coverage.load_authenticated_asec_household_coverage_fields( + members(payload=payload, expected_rows=expected) + ) + + +@pytest.mark.parametrize("mutation", ["field", "helper", "registry"]) +def test_live_contract_dependency_or_registry_mutation_refuses( + members, monkeypatch, mutation +): + source = coverage.load_authenticated_asec_household_coverage_fields(members()) + if mutation == "field": + monkeypatch.setattr(coverage, "_FIELDS", {"changed": True}) + elif mutation == "helper": + original = shared._implementation + monkeypatch.setattr( + shared, "_implementation", lambda: {**original(), "changed": True} + ) + else: + monkeypatch.setattr( + origin, "_ASEC_MEMBER_PINS", (replace(origin._ASEC_MEMBER_PINS[0], rows=2),) + ) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="PRODUCER_CHANGED"): + _ = source.document + + +def test_record_and_projection_token_limits_are_checked(members, monkeypatch): + paths = members([("1", "1", "9", "8", "3", "x" * 150)]) + monkeypatch.setattr(shared, "_MAX_RECORD_CHARS", 90) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="MEMBER_RECORD_SIZE" + ): + coverage.load_authenticated_asec_household_coverage_fields(paths) + monkeypatch.setattr(shared, "_MAX_RECORD_CHARS", 65536) + paths = members([("1", "1", "9", "8", "3" * 129, "")]) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="MEMBER_FIELD_SIZE" + ): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +@pytest.mark.parametrize("kind", ["directory", "fifo", "grown"]) +def test_regular_exact_source_capture_is_required(members, kind): + paths = members() + path = paths[2024] + if kind == "grown": + path.write_bytes(path.read_bytes() + b"extra") + else: + path.unlink() + path.mkdir() if kind == "directory" else os.mkfifo(path) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="MEMBER_CAPTURE"): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +@pytest.mark.parametrize("mutation", ["clear", "replace"]) +def test_caller_path_mapping_is_snapshotted_before_fingerprinting( + members, monkeypatch, mutation +): + paths = members() + original = coverage._producer + + def changing_producer(): + producer = original() + if mutation == "clear": + paths.clear() + else: + paths[2024] = paths[2024].with_name("must-not-be-opened.csv") + return producer + + monkeypatch.setattr(coverage, "_producer", changing_producer) + source = coverage.load_authenticated_asec_household_coverage_fields(paths) + assert len(source.document["records"]) == 1 + assert len(source.document["members"]) == 1 + + +def test_selected_pins_must_match_fingerprinted_registry(members, monkeypatch): + paths = members() + original = shared._registry + first = True + + def changing_registry(): + nonlocal first + pins = original() + if first: + first = False + monkeypatch.setattr( + origin, "_ASEC_MEMBER_PINS", (replace(pins[0], rows=2),) + ) + return pins + + monkeypatch.setattr(shared, "_registry", changing_registry) + with pytest.raises( + coverage.HouseholdCoverageSourceError, match="REGISTRY_TRANSITION" + ): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +@pytest.mark.parametrize( + "filename", + [ + "asec_household_coverage_fields.py", + "asec_original_household_weights.py", + "asec_student_controls.py", + ], +) +def test_actual_owner_and_helper_code_bytes_bind_issued_capsule( + members, monkeypatch, filename +): + source = coverage.load_authenticated_asec_household_coverage_fields(members()) + actual_files = coverage.resources.files + + def changed_files(package): + package_path = actual_files(package) + + def child(name): + path = package_path.joinpath(name) + if name == filename: + return SimpleNamespace( + read_bytes=lambda: path.read_bytes() + b"\n# changed\n" + ) + return path + + return SimpleNamespace(joinpath=child) + + monkeypatch.setattr(coverage.resources, "files", changed_files) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="PRODUCER_CHANGED"): + _ = source.document + + +def test_payload_bound_prevents_encoding_and_oversize_candidate(members, monkeypatch): + paths = members() + monkeypatch.setattr(shared, "_MAX_PAYLOAD_BYTES", 100) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="CANDIDATE_SIZE"): + coverage.load_authenticated_asec_household_coverage_fields( + paths, candidate=b"x" * 101 + ) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="ENCODING_SIZE"): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +def test_private_capture_rewrite_refuses_even_with_original_digest( + members, monkeypatch +): + paths = members() + original = shared._capture_owner._snapshot + + def changed(path, destination, *, size): + digest = original(path, destination, size=size) + destination.write_bytes(destination.read_bytes().replace(b",09,", b",01,")) + return digest + + monkeypatch.setattr(shared._capture_owner, "_snapshot", changed) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="CAPTURE_CHANGED"): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +def test_capture_growth_refuses_below_text_decoder(members, monkeypatch): + paths = members() + original = shared._DigestReader + + class GrowingReader(original): + def __init__(self, raw, size): + super().__init__(raw, size) + with open(raw.name, "ab") as output: + output.write(b"\nextra") + + monkeypatch.setattr(shared, "_DigestReader", GrowingReader) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="CAPTURE_SIZE"): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +@pytest.mark.parametrize( + "invalid", [{}, {True: "invented.csv"}, {2025: "invented.csv"}, {2024: object()}] +) +def test_source_requests_require_a_nonempty_closed_cohort_subset(members, invalid): + members() + with pytest.raises(coverage.HouseholdCoverageSourceError, match="MEMBER_PATHS"): + coverage.load_authenticated_asec_household_coverage_fields(invalid) + + +def test_combined_request_rows_are_bounded_before_any_source_capture( + members, monkeypatch +): + paths = members(years=(2023, 2024)) + monkeypatch.setattr(shared, "_MAX_ROWS", 1) + + def forbidden_capture(*args, **kwargs): + raise AssertionError("Must refuse aggregate allocation before source access") + + monkeypatch.setattr(shared._capture_owner, "_snapshot", forbidden_capture) + with pytest.raises(coverage.HouseholdCoverageSourceError, match="REQUEST_ROWS"): + coverage.load_authenticated_asec_household_coverage_fields(paths) + + +@pytest.fixture(params=["coverage", "weights"]) +def capsule_owner(request, members): + """Both real owners load the same single invented household member.""" + paths = members( + payload=_csv( + [("00001", "1", "09", "08", "03", "255212")], + header=(*coverage.COLUMNS, "HSUP_WGT"), + ) + ) + if request.param == "coverage": + return SimpleNamespace( + module=coverage, + load=coverage.load_authenticated_asec_household_coverage_fields, + cls=coverage.AuthenticatedAsecHouseholdCoverageFields, + error=coverage.HouseholdCoverageSourceError, + paths=paths, + ) + return SimpleNamespace( + module=shared, + load=shared.load_authenticated_asec_household_weights, + cls=shared.AuthenticatedAsecHouseholdWeights, + error=shared.HouseholdWeightSourceError, + paths=paths, + ) + + +def _rewritten_payload(source): + document = source.document + row = document["records"][0] + if "H_NUMPER" in row: + assert row["reported_person_count"] == 3 + row.update(H_NUMPER="16", reported_person_count=16) + else: + assert row["weight_integer_units"] == 255212 + row.update(HSUP_WGT="1", weight_integer_units=1) + document["projection_sha256"] = shared._sha(shared._encode(document["records"])) + return shared._encode(document) + + +def test_recomputed_self_checksum_cannot_authorize_rewritten_source(capsule_owner): + owner = capsule_owner + source = owner.load(owner.paths) + payload = _rewritten_payload(source) + object.__setattr__(source, "payload", payload) + object.__setattr__(source, "_issued_sha256", shared._sha(payload)) + with pytest.raises(owner.error, match="^SOURCE_MUTATION$"): + _ = source.document + + +@pytest.mark.parametrize( + "kind", ["new", "forged", "empty", "copy", "deepcopy", "pickle"] +) +def test_new_or_copied_value_does_not_inherit_issuance(capsule_owner, kind): + owner = capsule_owner + source = owner.load(owner.paths) + if kind in ("new", "forged", "empty"): + unissued = object.__new__(owner.cls) + if kind != "empty": + payload = _rewritten_payload(source) if kind == "forged" else source.payload + object.__setattr__(unissued, "payload", payload) + object.__setattr__(unissued, "_issued_sha256", shared._sha(payload)) + elif kind == "pickle": + unissued = pickle.loads(pickle.dumps(source)) + else: + unissued = getattr(copy, kind)(source) + assert unissued is not source + with pytest.raises(owner.error, match="^SOURCE_MUTATION$"): + _ = unissued.document + assert source.document["source_authenticated"] is True + + +def test_independent_equal_reissue_retains_its_own_weak_issuance(capsule_owner): + owner = capsule_owner + source = owner.load(owner.paths) + replay = owner.load(owner.paths, candidate=source.payload) + assert replay == source and replay is not source + assert replay.payload == source.payload == shared._encode(source.document) + source_id, replay_id = id(source), id(replay) + source_ref = weakref.ref(source) + del source + gc.collect() + assert source_ref() is None + assert source_id not in shared._CAPSULE_ISSUANCE + assert replay_id in shared._CAPSULE_ISSUANCE + assert replay.document["source_authenticated"] is True + del replay + gc.collect() + assert replay_id not in shared._CAPSULE_ISSUANCE + + +def test_verification_decodes_the_checked_payload_snapshot(capsule_owner, monkeypatch): + owner = capsule_owner + source = owner.load(owner.paths) + original_payload = source.payload + changed_payload = _rewritten_payload(source) + original_sha = shared._sha + + def mutate_after_hash(payload): + result = original_sha(payload) + if payload is original_payload: + object.__setattr__(source, "payload", changed_payload) + object.__setattr__(source, "_issued_sha256", original_sha(changed_payload)) + return result + + # Deterministic interleaving of a public object write after checksum work. + monkeypatch.setattr(shared, "_sha", mutate_after_hash) + assert shared._encode(source.document) == original_payload + with pytest.raises(owner.error, match="^SOURCE_MUTATION$"): + _ = source.document + + +def test_stale_weak_callback_cannot_remove_reused_identity(capsule_owner, monkeypatch): + owner = capsule_owner + first = owner.load(owner.paths) + second = owner.load(owner.paths) + first_reference = shared._CAPSULE_ISSUANCE[id(first)][0] + replacement_entry = shared._CAPSULE_ISSUANCE[id(second)] + # Model reuse deterministically instead of relying on allocator-specific ids. + with monkeypatch.context() as patch: + patch.setitem(shared._CAPSULE_ISSUANCE, id(first), replacement_entry) + first_reference.__callback__(first_reference) + assert shared._CAPSULE_ISSUANCE[id(first)] is replacement_entry + assert first.document == second.document + + +@pytest.mark.parametrize("payload", [None, bytearray(b"{}"), b"12345"]) +def test_private_constructor_bounds_precede_issuance( + capsule_owner, monkeypatch, payload +): + owner = capsule_owner + monkeypatch.setattr(shared, "_MAX_PAYLOAD_BYTES", 4) + with pytest.raises(owner.error, match="^SOURCE_BYTES$"): + owner.cls(payload, _token=owner.module._TOKEN) + + +@pytest.mark.parametrize( + "keys,reported_size,lookups", + [ + ([2022, 2023, 2024, 2025], 4, []), + ([2025], 1, []), + ([True], 1, []), + (["2024"], 1, []), + ([2024, 2024], 2, [2024]), + ([2022, 2023, 2024, 2025], 3, [2022, 2023, 2024]), + ([2024, 2023], 1, [2024]), + ([2024], 2, [2024]), + ], +) +def test_mapping_bounds_and_keys_precede_value_lookup( + capsule_owner, members, keys, reported_size, lookups +): + owner = capsule_owner + paths = members(years=(2022, 2023, 2024)) + + class HostileMapping(Mapping): + fetched = [] + yielded = 0 + + def __len__(self): + return reported_size + + def __iter__(self): + for key in keys: + self.yielded += 1 + yield key + + def __getitem__(self, key): + self.fetched.append(key) + return paths.get(key, "must-not-be-opened.csv") + + mapping = HostileMapping() + with pytest.raises(owner.error, match="^MEMBER_PATHS$"): + owner.load(mapping) + assert mapping.fetched == lookups + assert mapping.yielded <= 4 + + +def test_mapping_size_transition_refuses_before_capture(capsule_owner, monkeypatch): + owner = capsule_owner + + class ChangingMapping(Mapping): + size = 1 + + def __len__(self): + return self.size + + def __iter__(self): + yield 2024 + + def __getitem__(self, key): + self.size = 2 + return owner.paths[key] + + def forbidden_capture(*args, **kwargs): + raise AssertionError("Mapping transition must refuse before capture") + + monkeypatch.setattr(shared._capture_owner, "_snapshot", forbidden_capture) + with pytest.raises(owner.error, match="^MEMBER_PATHS$"): + owner.load(ChangingMapping()) + + +@pytest.mark.parametrize("aliases", ["csv", "native", "both", "owner_module"]) +def test_live_csv_reader_drift_refuses_before_capture( + capsule_owner, monkeypatch, aliases +): + owner = capsule_owner + issued = owner.load(owner.paths) + assert owner.load(owner.paths, candidate=issued.payload) == issued + original_reader = csv.reader + + def changed_reader(*args, **kwargs): + reader = original_reader(*args, **kwargs) + header = next(reader) + yield header + for original in reader: + row = list(original) + for name, value in (("HSUP_WGT", "1"), ("H_NUMPER", "16")): + if name in header: + row[header.index(name)] = value + yield row + + def forbidden_capture(*args, **kwargs): + raise AssertionError("Reader drift must refuse before source capture") + + with monkeypatch.context() as patch: + patch.setattr(shared._capture_owner, "_snapshot", forbidden_capture) + if aliases in ("csv", "both"): + patch.setattr(csv, "reader", changed_reader) + if aliases in ("native", "both"): + patch.setattr(native_csv, "reader", changed_reader) + if aliases == "owner_module": + patch.setattr( + owner.module, + "csv", + SimpleNamespace(**{**vars(csv), "reader": changed_reader}), + ) + with pytest.raises(owner.error, match="^SOURCE_CSV_READER_CHANGED$"): + owner.load(owner.paths) + with pytest.raises(owner.error, match="^SOURCE_CSV_READER_CHANGED$"): + _ = issued.document + assert owner.load(owner.paths, candidate=issued.payload) == issued + + +@pytest.mark.parametrize("replacement", ["python_wrapper", "wrong_builtin"]) +def test_csv_binding_refuses_replaced_aliases_before_helper_import( + monkeypatch, replacement +): + name = "microcosm.build.us_runtime.source_csv_builtin" + + def wrapper(*args, **kwargs): + raise AssertionError("Spoofed reader must never execute") + + wrapper.__module__ = "_csv" + wrapper.__name__ = "reader" + wrapper.__self__ = native_csv + fake = wrapper if replacement == "python_wrapper" else native_csv.writer + with monkeypatch.context() as patch: + patch.delitem(sys.modules, name, raising=False) + patch.setattr(csv, "reader", fake) + patch.setattr(native_csv, "reader", fake) + with pytest.raises(ValueError, match="^SOURCE_CSV_READER_CHANGED$"): + importlib.import_module(name) + + +def test_csv_binding_helper_bytes_bind_each_capsule(capsule_owner, monkeypatch): + owner = capsule_owner + issued = owner.load(owner.paths) + real_files = shared.resources.files + + def changed_files(package): + package_path = real_files(package) + + def child(name): + path = package_path.joinpath(name) + if name == "source_csv_builtin.py": + return SimpleNamespace( + read_bytes=lambda: path.read_bytes() + b"\n# drift\n" + ) + return path + + return SimpleNamespace(joinpath=child) + + monkeypatch.setattr(shared.resources, "files", changed_files) + with pytest.raises(owner.error, match="^PRODUCER_CHANGED$"): + _ = issued.document + + +@contextmanager +def transient_csv_rebinding(target, aliases, moment): + """Time only public alias mutation; never replace a production function.""" + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "strict=True" in line + ) + real_csv, real_native = csv.reader, native_csv.reader + fired, calls = [], [] + + def restore(): + csv.reader, native_csv.reader = real_csv, real_native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + reader = real_csv(*args, **kwargs) + header = next(reader) + yield header + for original in reader: + row = list(original) + for name, value in ( + ("HSUP_WGT", "1"), + ("H_NUMPER", "16"), + ("PRPERTYP", "2"), + ): + if name in header: + row[header.index(name)] = value + yield row + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + csv.reader = replacement + if aliases == "both": + native_csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + yield fired, calls + finally: + sys.settrace(previous) + restore() + + +@pytest.mark.parametrize("aliases", ["csv", "both"]) +@pytest.mark.parametrize("moment", ["entry", "before_call"]) +def test_transient_csv_rebinding_never_supplies_source_cells( + capsule_owner, aliases, moment +): + owner = capsule_owner + control = owner.load(owner.paths) + assert owner.load(owner.paths, candidate=control.payload) == control + before = ( + owner.module._producer() + if owner.module is coverage + else shared._implementation() + ) + with transient_csv_rebinding(owner.module._read_capture, aliases, moment) as ( + fired, + calls, + ): + try: + result = owner.load(owner.paths) + except owner.error as exc: + assert str(exc) == "SOURCE_CSV_READER_CHANGED" + else: + assert result == control and result.document == control.document + assert fired and not calls + assert before == ( + owner.module._producer() + if owner.module is coverage + else shared._implementation() + ) + assert owner.load(owner.paths, candidate=control.payload) == control diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index de6f713fc..6354b5ab3 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -226,6 +226,7 @@ "acs_sources.py", "acs_transfer_bank.py", # Bounded checkpoint I/O; no population treatment. "asec_checkpoint.py", # Bounded checkpoint I/O; no population treatment. + "asec_raw_stage_v4.py", # Authenticated source restoration; no treatment. "asec_pool.py", "base_pool.py", "block_ladder_sources.py", From cabcad42a9fabb3dcc02329c0661513fae2e5ff1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:13:01 -0400 Subject: [PATCH 073/255] Repair two stale test expectations after the post-clone geography change The strict-leaf audit stub now mirrors PolicyEngineUSVariableMetadataIndex's keyword-only surface (include_consumers), and the PUF55 recipient graph is 19 run nodes plus the projection and matrix nodes since geography moved after the initial clone. --- .../tests/test_us_acs_transfer.py | 24 ++++++++++--------- .../test_us_graph_puf55_survey_recipients.py | 6 +++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index e9e615bec..42d03205a 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -1467,9 +1467,8 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( ) person = result.frame.person - eligible = ( - person["is_female"].astype(bool) - & person["age"].between(15, 44, inclusive="both") + eligible = person["is_female"].astype(bool) & person["age"].between( + 15, 44, inclusive="both" ) assert person.loc[eligible, "is_pregnant"].all() assert not person.loc[~eligible, "is_pregnant"].any() @@ -1490,7 +1489,9 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( .first() .sum() ) - record = next(item for item in result.imputed_inputs if item.column == "is_pregnant") + record = next( + item for item in result.imputed_inputs if item.column == "is_pregnant" + ) receipt = record.structural_receipt assert receipt is not None assert sum(pattern.recipient_rows for pattern in record.patterns) == ( @@ -1576,12 +1577,8 @@ def test_pregnancy_partial_clone_fanout_receipt_categories_are_disjoint( record = result.imputed_inputs[0] receipt = record.structural_receipt assert receipt is not None - assert receipt["preexisting_value_fanout_rows"] == int( - (missing & eligible).sum() - ) - assert receipt["ineligible_rows_assigned_false"] == int( - (missing & ~eligible).sum() - ) + assert receipt["preexisting_value_fanout_rows"] == int((missing & eligible).sum()) + assert receipt["ineligible_rows_assigned_false"] == int((missing & ~eligible).sum()) assert ( receipt["preexisting_value_fanout_rows"] + receipt["ineligible_rows_assigned_false"] @@ -2468,7 +2465,12 @@ def test_strict_leaf_audit_reports_missing_us_extra( from microcosm.frame.adapters import policyengine_us as adapter_module class _MissingMetadataIndex: - def __init__(self) -> None: + # Mirrors PolicyEngineUSVariableMetadataIndex's keyword-only surface. + # The audit path constructs it twice: puf_support._formula_owned_engine + # passes include_consumers=False before the strict-leaf branch + # constructs it bare, and both must raise the absent-extra ImportError + # rather than a signature TypeError. + def __init__(self, *, include_consumers: bool = True) -> None: raise ImportError("policyengine-us is absent") monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py b/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py index 30a75a559..2a2c795ce 100644 --- a/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py +++ b/packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py @@ -90,11 +90,13 @@ def _verified_artifacts(case, manifest): return payloads[graph.PROJECTION_NODE, "projection"], matrices -def test_actual_twenty_two_node_cold_required_replay_keeps_complete_population( +def test_actual_twenty_one_node_cold_required_replay_keeps_complete_population( recipient_graph, ): case = recipient_graph - assert len(case.compiled.order) == 22 + # The financial run graph (19 nodes since geography moved after the + # initial clone) plus exactly the projection and matrix nodes below. + assert len(case.compiled.order) == 19 + 2 assert case.compiled.graph.sources == case.run.compiled.graph.sources assert set(case.run.sources) == { graph.source_graph.SOURCE_NAME, From fe9a7bd6aa64c213bd37866b1d0ff96fa6a50fc1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:13:01 -0400 Subject: [PATCH 074/255] Install the engine-free US extra into the wheels gate after the boundary check The US source runtimes require microunit (microcosm-frame's us extra); an installed constellation without it fails closed instead of skipping, which is a false packaging verdict. The extra is requested against the built wheel so the gate proves the wheel carries resolvable extra metadata; policyengine-us stays absent. --- .github/workflows/test.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bcc4bc65..86d472a21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -400,6 +400,43 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk + - name: Add the engine-free US extra the source runtimes require + # microunit is microcosm-frame's `us` extra: the sanctioned tax-unit + # constructor, and the only optional dependency the US source runtimes + # need without a country engine. Every ACS/CREATE stage names it in + # us_runtime.graph_implementation.STAGE_DEPENDENCIES, so an installed + # constellation without it cannot hash those stages: the suite then + # fails closed (SOURCE_PREPARATION_REFUSED / CATALOGUE_ISSUANCE_REFUSED) + # instead of skipping, which is a false packaging verdict. + # + # This runs after the boundary check above, so that check still proves + # the base wheels import with no optional dependency installed at all. + # Requesting the extra against the built wheel — rather than naming + # `microunit` here — keeps the extra declared only in + # packages/microcosm-frame/pyproject.toml and makes this gate prove the + # wheel carries resolvable extra metadata. policyengine-us stays absent: + # it is declared by microcosm-build[us] and microcosm-frame[policyengine], + # and neither is requested. + run: | + frame_wheel="$(ls dist/microcosm_frame-*.whl)" + uv pip install --python /tmp/wheels-venv/bin/python \ + --constraint /tmp/wheel-constraints.txt \ + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" + env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' + from importlib import metadata, util + + assert util.find_spec("microunit") is not None, ( + "microcosm-frame[us] must install microunit into the wheel venv" + ) + assert util.find_spec("policyengine_us") is None + try: + metadata.distribution("policyengine-us") + except metadata.PackageNotFoundError: + pass + else: + raise AssertionError("policyengine-us must be absent from the base wheel gate") + print("microunit", metadata.version("microunit")) + PY - name: Run the suite against the installed wheels run: | /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \ From b979797a101dd44a3c845338a5db109665bad2b6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:13:01 -0400 Subject: [PATCH 075/255] Record the incomplete native post-clone financial pilot observation --- ...stclone-financial-incomplete-20260910.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 experiments/us-native-postclone-financial-incomplete-20260910.json diff --git a/experiments/us-native-postclone-financial-incomplete-20260910.json b/experiments/us-native-postclone-financial-incomplete-20260910.json new file mode 100644 index 000000000..aeecacf9c --- /dev/null +++ b/experiments/us-native-postclone-financial-incomplete-20260910.json @@ -0,0 +1,22 @@ +{ + "automatic_retry": false, + "cpu_limit_seconds": 7200, + "exact_exit_signal_available": false, + "final_guard_receipt_present": false, + "guard_sha256": "f46e5dc061cbb50f7941891e2db8f7159ebe034f736280279b330605a40ad07b", + "interpretation": "The process is absent and its last sampled CPU exceeded the reviewed limit. Final verification did not complete; stored intermediate artifacts are not an accepted output. Retained progress samples identify repeated source/catalogue checks, but do not measure an isolated performance cause.", + "last_observed_cpu_seconds": 7216.063893, + "last_observed_wall_seconds": 7315.9110773339635, + "native_acceptance": false, + "next_step": "Review and test a semantics-preserving verification optimization before proposing another native run.", + "peak_rss_bytes": 11515363328, + "postcheck_native_payloads_reopened": false, + "published_financial_manifest_present": false, + "release_acceptance": false, + "resource_progress_sha256": "eaa7f4dc175f055d7b3e6997f9c47b2a9a3573e041412338a487ccba06fc7d7c", + "root_incomplete_observation_sha256": "b4c704e61cec4f00cc4ce0340cb8504eb592e4773a10b473c5e9cbcf497eb2b7", + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "One cold 1/1000 native postclone nineteen-node financial pilot", + "status": "incomplete_without_final_guard_receipt", + "wall_limit_seconds": 10800 +} From 4ffa5233ca1dc7351e6f97e050a6490ff038dd10 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:22:20 -0400 Subject: [PATCH 076/255] Classify the launch integration's runtime modules in the spine-blindness inventory The consolidation added 113 graph-native source, catalogue, composition, clone, geography, predictor, PUF and calibration modules that the reviewed inventory did not list, so the classification tests failed closed. Each is recorded from its module docstring (reviewed 2026-09-11) in a separate set; none is a migrated population-treatment registry operator, and the inventory does not exempt any of them from the source-spine scan. The pool build tool's import graph is re-pinned from 70 to 73 modules: it now also reaches _person_signal_summary, operator_column_contracts and reported_coverage_source, which join the required set so a silent drop fails. --- .../tests/test_us_spine_blindness.py | 287 +++++++++++++++++- 1 file changed, 283 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 6354b5ab3..db3c60587 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -71,9 +71,12 @@ _SPINE_BLIND_BUILD_TOOLS = (_REPOSITORY_ROOT / "tools" / "build_us_multispine_pool.py",) _REQUIRED_POOL_RUNTIME_MODULES = frozenset( { + "_person_signal_summary.py", "late_producer_dag.py", "multispine_pool.py", + "operator_column_contracts.py", "puf_support.py", + "reported_coverage_source.py", "spine_agreement.py", "spine_assembly.py", "us_late_overlap_ownership.py", @@ -311,8 +314,276 @@ "worker_identity.py", # Portable primary-QRF worker identity; no population treatment. } ) -_CLASSIFIED_US_RUNTIME_MODULES = frozenset(_SPINE_BLIND_OPERATOR_MODULES).union( - _OTHER_US_RUNTIME_MODULES +# The US launch integration (PR #893) adds the graph-native source, catalogue, +# composition, clone, geography, predictor, PUF and calibration modules below. +# None is a migrated population-treatment registry operator; each is classified +# here from its module docstring (reviewed 2026-09-11) so an added module +# still fails until reviewed. This inventory does not exempt any of them from +# the source-spine scan below; modules that inspect provenance by charter are +# listed separately in _SOURCE_SPINE_PROVENANCE_OWNERS with their reason. +_US_LAUNCH_GRAPH_RUNTIME_MODULES = frozenset( + { + # Bounded, checksummed codec preserving target-only money source authority + "_asec_current_money_codec.py", + # Validate the measured person-signal summary shape before gate decisions + "_person_signal_summary.py", + # Pure, unbound ACS housing-unit semantics; no source authentication + "acs_housing_universe.py", + # Closed ACS source observations, with a same-snapshot native Frame bridge + "acs_housing_universe_source.py", + # Closed real ACS preparation plus literal coverage issuance, without + # admission + "acs_native_coverage_binding.py", + # Closed ACS coverage source issuance; native consistency grants no identity + "acs_person_coverage_authentication.py", + # Additive, literal ACS person coverage fields for an exact native roster + "acs_person_coverage_columns.py", + # Closed raw ACS catalogue before native population or unit construction + "acs_population_catalogue.py", + # Issue selected 2024 ASEC observations with original household DESIGN + # anchors + "asec_2024_native_population.py", + # Closed, source-only PRPERTYP evidence for an exact current-money parent + "asec_coverage_authentication.py", + # Pure ASEC current-money decoding and price restatement + "asec_current_money.py", + # Pinned additive consumer status, separate from the immutable money recipe + "asec_current_money_graph_resources.py", + # Explicit packaged-resource and installed microunit byte verification + "asec_current_money_resources.py", + # Typed subset of an authenticated ASEC current-money body + "asec_current_money_selection.py", + # Authenticate the reviewed full ASEC v4 plus household-observation + # attachment + "asec_current_money_source.py", + # Actual tax-only reconstruction from verified target-current ASEC money + "asec_current_money_units.py", + # Explicit ASEC demographic source contract: sex, its allocation flag, and + # the + "asec_demographic_source.py", + # Closure-admitted PolicyEngine-US evaluation of the corrected money leaves + "asec_engine_evaluation.py", + # Closed original ASEC household observations, without coverage inference + "asec_household_coverage_fields.py", + # Attach authenticated raw ASEC household observations by exact original + # keys + "asec_household_observations.py", + # Immutable interview-time ASEC housing observations and publisher routing + "asec_housing_status.py", + # Fixed-cohort numeric source authentication and tax-result composition + "asec_housing_status_source.py", + # Source-specific occupied-HU evidence; never a population or release + # verdict + "asec_housing_universe.py", + # Authenticate carried ASEC HU evidence and preserve the chosen parent chain + "asec_housing_universe_source.py", + # Closed PAW source observations, independent of the sealed 33-field money + # body + "asec_income_observations.py", + # Authenticate literal original ASEC household weights without pooling them + "asec_original_household_weights.py", + # Bounded, non-authoritative original-CSV ASEC PRPERTYP projection + "asec_person_coverage_source.py", + # Restore an observed person total from closed, authenticated Census members + "asec_person_income_source.py", + # Authenticate the complete 2024 ASEC catalogue before population + # construction + "asec_population_catalogue.py", + # Full-source ASEC preparation for the current-money graph slice + "asec_prepared_source.py", + # Closed Census interview-week controls, separate from the T1 money source + "asec_student_controls.py", + # Pinned population-only Census API responses to atomic support, without I/O + "atomic_block_api_sources.py", + # Pinned Census source bytes to normalized atomic-block support, without I/O + "atomic_block_sources.py", + # Normalize US block mappings for shared geography operators; no new kernel + "atomic_block_support.py", + # Public CD references and uncertainty, without target or candidate + # activation + "cd_reference.py", + # Pinned public ACS acquisition; credentials never enter stored provenance + "cd_reference_sources.py", + # Pure export comparison against a supplied common parent; no source issuer + "common_frame_export_contract.py", + # Corrected CPS monetary leaves derived from a typed selected-money subset + "cps_carried_current.py", + # Current survey source projection for ASEC sex and household state + "current_asec_demographics.py", + # Current survey Social Security totals, reason literals and unknown shares + "current_social_security_source.py", + # Qualified observed geography and stable household draw keys for both + # surveys + "current_survey_geography.py", + # Source-qualified current ASEC to ACS financial predictor preparation + "current_survey_predictors.py", + # National demographic calibration with registry-bound graph diagnostics + "demographic_calibration_graph.py", + # Complete canonical PUF enrichment boundaries; no source admission is + # issued + "full_puf_enrichment.py", + # Typed ACS source evidence and exact whole-household graph selection + "graph_acs_housing_universe.py", + # Typed PAW transport and reported-income accounting, never source authority + "graph_asec_income.py", + # The five-node ASEC slice: source, selection, leaves, engine and reported + # income + "graph_asec_prepared.py", + # Assign shared geography after the complete combined-survey support clone + "graph_atomic_survey_clone.py", + # Checked atomic survey geography plus current financial development output + "graph_atomic_survey_financial.py", + # Complete initial support clones before assigning qualified survey + # geography + "graph_atomic_survey_population.py", + # The combined-survey PUF-support clone, as a real graph ``EXPAND`` stage + "graph_combined_clone.py", + # Bind the carried prepared-ASEC evidence to the ASEC rows of a composed + # population + "graph_composed_asec_binding.py", + # Corrected ASEC monetary and demographic measurements over composed ASEC + # rows + "graph_composed_asec_measures.py", + # Source-free declarations shared by composed producers and their consumers + "graph_composed_contracts.py", + # Compose the prepared ASEC population with the native ACS population + "graph_composed_population.py", + # Identity-bound metadata for real US operators in the population graph + "graph_context.py", + # Project authenticated survey geography into declared household cells + "graph_current_survey_geography.py", + # Graph training, draw and attachment of current survey financial predictors + "graph_current_survey_predictors.py", + # Current survey-first E00900 diagnostic placement, not donor admission + "graph_current_survey_puf_transfer.py", + # Bounded PUF profile attachment to an independently retained Population + "graph_full_puf_enrichment.py", + # Declared lookup import, household assignment, and geography validation + "graph_geography.py", + # Non-authoritative HU transport bound to CREATE and selected graph ancestry + "graph_housing_universe.py", + # Reviewed, path-independent implementation identities for the US source + # graph + "graph_implementation.py", + # Disjoint household age counts, and the composition that calibrates them + "graph_national_age_counts.py", + # Graph source producers and population-grain native household lineage + "graph_native_household_origin.py", + # Closed additional identity for native-origin nodes; upstream bytes stay + # fixed + "graph_native_origin_implementation.py", + # One source-backed canonical PUF55 donor CREATE, shared by both routes + "graph_puf55_canonical_donor.py", + # Typed PUF55 route artifacts after current financial leaves + "graph_puf55_survey_recipients.py", + # A fixture-only price → donor → matrix → raw draw → masked placement graph + "graph_puf_detail_transfer.py", + # Source-qualified development diagnostic, preserving the fixture-only path + "graph_puf_diagnostic_consumer.py", + # Real, declared US source construction for the population graph + "graph_sources.py", + # Unweighted age counts as an ordered artifact, without population columns + "graph_survey_age_artifact.py", + # Transport source-qualified budget values through their actual graph edges + "graph_survey_budget.py", + # Source-blind age calibration over typed numeric budget and count artifacts + "graph_survey_calibration.py", + # Graph declarations for a source-authenticated, selected survey population + "graph_survey_population.py", + # Explicit input-name profiles; neither a release gate nor source admission + "input_coverage_profile.py", + # Explicit activation of the direct-national ACS 2024 published age counts + "national_age_activation.py", + # Private native household origins, authenticated from original source + # members + "native_household_origin.py", + # Source-visible operator column declarations, without operation imports + "operator_column_contracts.py", + # Describe required inputs on one actual graph Population and attached + # manifest + "population_input_coverage.py", + # Project a bound canonical59 artifact into the survey-SS PUF55 donor + "puf55_canonical_donor.py", + # Validate and merge the two PUF55 raw chains before one finalization + "puf55_route_finalization.py", + # Disjoint PUF55 recipient values from an actual, authenticated financial + # run + "puf55_survey_recipients.py", + # Filer/joint-spouse report sums for conditioning, never beneficiary amounts + "puf55_survey_ss_measurement.py", + # Return-grain canonical PUF59 with explicit model/growth lineage + "puf59_canonical.py", + # Bounded deterministic private canonical59 array envelope; no source + # admission + "puf59_canonical_artifact.py", + # Fixture-only PUF price-result import and combined-host feature placement + "puf_detail_transfer.py", + # Qualified transport for one development-only PUF conditional diagnostic + "puf_diagnostic_consumer.py", + # Additive full-return PUF source projection, before canonical modeling + "puf_full_source.py", + # Graph owner of the additive complete PUF return-source artifact + "puf_full_source_graph.py", + # Pure per-field growth contract and transform for a raw-source PUF + # successor + "puf_growth.py", + # Graph nodes that run the pure PUF growth transform inside a real + # population + "puf_growth_graph.py", + # The thirteen-field 2015 PUF monetary source projection: the twelve + # accepted + "puf_monetary_agi_projection.py", + # The first monetary amounts typed out of the 2015 PUF delivery bytes + "puf_monetary_source.py", + # The developmental CPI-U price-restatement baseline for the 2015 PUF + "puf_price_baseline.py", + # Explicit canonical PUF QBI data model, adapted from the archived PUF owner + "puf_qbi_model.py", + # The first typed artifact read straight from the 2015 PUF delivery bytes + "puf_raw_source.py", + # Source-specific, explicit PUF2015 to target2024 monetary transport + "puf_target2024_growth.py", + # Measured ASEC at-interview coverage recodes omitted by frozen H5 inputs + "reported_coverage_source.py", + # Bind source CSV readers to the real stdlib builtin, including caller + # aliases + "source_csv_builtin.py", + # Explicit S0101-only activation for the selected ACS/ASEC development path + "survey_age_activation.py", + # Country admission for survey age profiles with an explicit source prefix + "survey_age_calibration.py", + # Two explicit S0101-only source captures; no network or target activation + "survey_age_sources.py", + # Reconstruct observed constraints, initial clones and geography from live + # sources + "survey_atomic_geography.py", + # Recompute supported age diagnostics without rerunning an optimizer + "survey_calibration_diagnostics.py", + # Plan one draw over complete supplied survey records, before native Frames + "survey_catalogue_selection.py", + # Admit a checked financial graph result under its original sampling budget + "survey_financial_successor.py", + # Pure observed-age alias; no source, population or release authority + "survey_observed_age.py", + # Source-qualified, sampling-aware budgets for the complete first survey + # clone + "survey_origin_budget.py", + # Pure US development-domain declarations over complete, supplied literal + # rows + "survey_population_domains.py", + # Authenticate complete catalogues, select once, and stack original anchors + "survey_population_preparation.py", + # Directional ContentStore replay checks, never source or population + # authority + "survey_population_replay.py", + # Explicit source-report Social Security component basis + "survey_social_security.py", + } +) +_CLASSIFIED_US_RUNTIME_MODULES = ( + frozenset(_SPINE_BLIND_OPERATOR_MODULES) + .union(_OTHER_US_RUNTIME_MODULES) + .union(_US_LAUNCH_GRAPH_RUNTIME_MODULES) ) @@ -3263,6 +3534,10 @@ def test_us_runtime_module_classification_is_complete() -> None: actual = {path.name for path in _US_RUNTIME.glob("*.py")} overlap = sorted( set(_SPINE_BLIND_OPERATOR_MODULES).intersection(_OTHER_US_RUNTIME_MODULES) + | set(_SPINE_BLIND_OPERATOR_MODULES).intersection( + _US_LAUNCH_GRAPH_RUNTIME_MODULES + ) + | set(_OTHER_US_RUNTIME_MODULES).intersection(_US_LAUNCH_GRAPH_RUNTIME_MODULES) ) assert not overlap, f"runtime module classifications overlap: {overlap}" assert not _unclassified_runtime_modules(actual), ( @@ -3404,8 +3679,12 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - assert len(runtime_graph) == 70, ( - f"{tool.name} must reach the pinned 70-module runtime graph; " + # 73 since the US launch integration: the pool tool now also reaches + # _person_signal_summary.py, operator_column_contracts.py and + # reported_coverage_source.py (reviewed 2026-09-11; all three are + # listed in _REQUIRED_POOL_RUNTIME_MODULES so a silent drop fails). + assert len(runtime_graph) == 73, ( + f"{tool.name} must reach the pinned 73-module runtime graph; " f"reached {len(runtime_graph)}" ) assert not missing_modules, ( From 1734b9e90d30edc2eab5cdd01c26ed4c49618722 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:22:31 -0400 Subject: [PATCH 077/255] Re-pin the seed protocol and compiled seed map digests for the launch integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three attested kernel modules changed on this branch — acs_transfer (its derived-transfer column contract moved to operator_column_contracts), housing_inputs (ACS rent donor/recipient preparation and finalization) and microcosm.fit.qrf (predict_from_uniforms draw path) — so the seed protocol's content digest and the compiled seed stream map digest legitimately moved. Main is fully merged (6f7571e1); spec_engine and the US spec are unchanged. The committed F0 coverage report is regenerated from the re-pinned inventory and tools/spec_engine_coverage.py --check passes. --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 45e73dcbb..5c8e67132 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "failures": [], "observed": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 89171055c..309b2f4de 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "seed_protocol": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "seed_map": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "seed_protocol": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", From a90965f2c83d7ad03b10b76bffc0a064614c3ac5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:29:14 -0400 Subject: [PATCH 078/255] Review the launch integration's source-spine accesses in the blindness guard The composed two-survey population is source-qualified by charter: ACS and ASEC rows are composed, cloned and bound to their original records, and the PUF55 routes condition on source-owned Social Security totals (known for ASEC reporters, unknown otherwise). Seventeen launch modules that read the support-channel and spine-source-id columns to select or validate origin rows join the reviewed provenance owners with their reason; none routes a PUF-detail imputation by spine. Twenty-one further modules trip only the scanner's fail-closed dynamic selector class. Every site was read: they index tuples and state records, node/kernel maps keyed by node id, byte payloads in codecs, PUF donor tables by profile-declared names, or numpy masks -- never a provenance column. A separate reviewed set accepts that class for exactly those modules while accessor calls and contraband source columns keep failing, with a regression that proves the acceptance never hides either. --- .../tests/test_us_spine_blindness.py | 112 +++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index db3c60587..ba756a99e 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -119,6 +119,31 @@ "us_late_overlap_ownership.py", "us_late_producer_registry.py", "warm_start_selection.py", # Provenance reporting and recovery. + # US launch integration (PR #893), reviewed 2026-09-11. The composed + # two-survey population is source-qualified by charter: ACS and ASEC + # rows are composed, cloned and bound to their original records, and + # the PUF55 routes condition on source-owned Social Security totals + # (known for ASEC reporters, unknown otherwise; nine versus eight + # predictors). These modules read support_channel/spine_source_id to + # select or validate origin rows and to bind original identities; + # none routes a PUF-detail imputation by spine. + "current_asec_demographics.py", # ASEC rows -> sex/household state. + "current_social_security_source.py", # Source-owned SS totals per row. + "current_survey_geography.py", # Origin roster and draw keys. + "current_survey_predictors.py", # Source-qualified predictor prep. + "graph_combined_clone.py", # Clone EXPAND keeps the channel labels. + "graph_composed_asec_binding.py", # Binds prepared ASEC to its rows. + "graph_composed_population.py", # Composes ACS + ASEC populations. + "graph_current_survey_predictors.py", # Trains/draws per source. + "graph_sources.py", # Declares the channel slices it loads. + "graph_survey_population.py", # Declares/validates origin columns. + "native_household_origin.py", # Original household lineage. + "population_input_coverage.py", # Reports coverage by source group. + "puf55_survey_recipients.py", # ASEC reporter ids for SS routes. + "puf55_survey_ss_measurement.py", # Reported SS sums by reporter. + "puf_detail_transfer.py", # Validates channels at placement. + "puf_diagnostic_consumer.py", # Development diagnostic by origin. + "survey_population_preparation.py", # Stacks original anchors. } ) @@ -3365,6 +3390,46 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: ) +# US launch integration (PR #893), reviewed 2026-09-11. The fail-closed +# scanner records every dynamic subscript, getattr or .get() on a value it +# cannot resolve. In these modules every such site was read: they index +# tuples and state records (entry[2], state[0]), node/kernel maps keyed by +# node id, byte payloads in codecs (payload[offset:...]), PUF donor tables +# by profile-declared predictor/target names, or numpy masks. None selects a +# provenance column. The scanner keeps running on them: accessor calls and +# contraband source columns still fail; only the dynamic-selector class is +# accepted, and only for the listed modules. +_REVIEWED_DYNAMIC_SELECTOR_MODULES = frozenset( + { + "acs_native_coverage_binding.py", + "acs_person_coverage_columns.py", + "acs_population_catalogue.py", + "asec_2024_native_population.py", + "atomic_block_api_sources.py", + "atomic_block_sources.py", + "full_puf_enrichment.py", + "graph_atomic_survey_financial.py", + "graph_atomic_survey_population.py", + "graph_current_survey_puf_transfer.py", + "graph_full_puf_enrichment.py", + "graph_puf55_canonical_donor.py", + "graph_puf_detail_transfer.py", + "puf55_route_finalization.py", + "puf59_canonical_artifact.py", + "puf_full_source.py", + "puf_price_baseline.py", + "puf_qbi_model.py", + "puf_target2024_growth.py", + "survey_atomic_geography.py", + "survey_social_security.py", + } +) +_DYNAMIC_SELECTOR_FINDING_MARKERS = ( + "unresolvable dynamic", + "hidden or expanded arguments", +) + + def _non_owner_source_spine_accesses( module_name: str, source: str, @@ -3373,7 +3438,52 @@ def _non_owner_source_spine_accesses( if module_name in _SOURCE_SPINE_PROVENANCE_OWNERS: return () - return _source_spine_accesses(source) + accesses = _source_spine_accesses(source) + if module_name in _REVIEWED_DYNAMIC_SELECTOR_MODULES: + accesses = tuple( + access + for access in accesses + if not any(marker in access for marker in _DYNAMIC_SELECTOR_FINDING_MARKERS) + ) + return accesses + + +def test_reviewed_dynamic_selector_modules_still_fail_on_provenance_reads() -> None: + """The dynamic-selector acceptance never hides an accessor or a column.""" + + reviewed = next(iter(sorted(_REVIEWED_DYNAMIC_SELECTOR_MODULES))) + dynamic_only = """ +def pick(table, name): + return table[name] +""" + assert _non_owner_source_spine_accesses("unlisted_module.py", dynamic_only) + assert _non_owner_source_spine_accesses(reviewed, dynamic_only) == () + with_accessor = """ +from microcosm.build.us_runtime.support_provenance import support_channel_column + +def pick(table, name): + channel = table[support_channel_column("person")] + return table[name], channel +""" + remaining = _non_owner_source_spine_accesses(reviewed, with_accessor) + assert remaining and all( + "support_channel_column" in access for access in remaining + ), remaining + with_column = """ +def pick(table): + return table["person_spine_source_id"] +""" + assert _non_owner_source_spine_accesses(reviewed, with_column) + overlap = sorted( + _REVIEWED_DYNAMIC_SELECTOR_MODULES.intersection(_SOURCE_SPINE_PROVENANCE_OWNERS) + ) + assert not overlap, overlap + missing = sorted( + name + for name in _REVIEWED_DYNAMIC_SELECTOR_MODULES + if not (_US_RUNTIME / name).is_file() + ) + assert not missing, missing def _called_function_names(source: str) -> set[str]: From 931ee20d39b931de38cd0db85cc9610b7d164da7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:29:14 -0400 Subject: [PATCH 079/255] Let the seed-identity diagnostic import the engine under its boundary The derive phase imports policyengine-us, which reads eight public parameter CSVs at module import; the boundary refused them as data files, and the shared engine lane's diagnostic failed on every run of this branch. The eight files are now pinned by relative path, size and RECORD sha256 (verified against 1.819.0), hashed once before the hook is armed, and only their resolved paths may be opened; any other data-suffixed open, a pin whose bytes or RECORD differ, or a pinned path outside the distribution still refuses. The report records the policy and pins. Two import-time side effects of the same chain are recognised narrowly: psutil reads /proc/stat for its CPU-times layout (metadata, like /proc/cpuinfo), and urllib3 probes IPv6 by creating one socket and binding it to loopback from _has_ipv6 -- only socket.__new__ and a loopback bind issued from that function are accepted; connect, resolve, send and every other socket event keep the NETWORK_OR_CHILD refusal. --- ...ec_seed_identity_engine_parameter_files.py | 185 ++++++++++++++++++ ...est_spec_seed_identity_per_code_context.py | 4 +- tools/spec_seed_identity_diagnostics.py | 157 ++++++++++++++- 3 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py new file mode 100644 index 000000000..28dce08eb --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py @@ -0,0 +1,185 @@ +"""Pinned engine parameter files and the urllib3 probe: invented hooks only. + +The derive phase imports policyengine-us, which reads eight public parameter +CSVs at module import. The boundary may open exactly those files, only when +their bytes hash to the pins verified before the hook is armed; every other +data-suffixed open still refuses. urllib3's import-time IPv6 probe creates one +socket and binds it to loopback from ``_has_ipv6``; nothing else on a socket is +accepted. +""" + +import base64 +import hashlib +import importlib.util +import os +import types +from importlib import metadata +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("engine_parameter_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _boundary(diagnostic, monkeypatch, allowed): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + allowed_data_files=allowed, + ) + assert len(hooks) == 1 + return hooks[0], distinct + + +def test_pins_match_the_installed_distribution_record(diagnostic): + """Every pin is a regular file in the distribution whose RECORD digest agrees.""" + + dist = metadata.distribution(diagnostic.ENGINE_PUBLIC_PARAMETER_DISTRIBUTION) + root = Path(dist.locate_file("")).resolve() + record = {} + for entry in dist.files or (): + if entry.hash is not None and entry.hash.mode == "sha256": + padded = entry.hash.value + "=" * (-len(entry.hash.value) % 4) + record[str(entry)] = base64.urlsafe_b64decode(padded).hex() + assert len(diagnostic.ENGINE_PUBLIC_PARAMETER_FILES) == 8 + for relative, expected, size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES: + path = root / relative + assert path.is_file() and not path.is_symlink(), relative + raw = path.read_bytes() + assert len(raw) == size and hashlib.sha256(raw).hexdigest() == expected + assert record[relative] == expected + report, allowed = diagnostic.pinned_engine_parameter_files() + assert report["policy"] == "pinned_engine_public_parameter_files_v1" + assert report["count"] == 8 and report["version"] == dist.version + assert allowed == frozenset( + (root / relative).resolve() + for relative, _digest, _size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES + ) + + +def test_a_pin_whose_bytes_differ_refuses_before_arming(diagnostic, tmp_path): + """Verification fails closed on bytes, size, RECORD, or a missing file.""" + + relative = "invented_engine/parameters/table.csv" + root = tmp_path / "site-packages" + (root / "invented_engine/parameters").mkdir(parents=True) + raw = b"invented,public,parameter\n1,2,3\n" + (root / relative).write_bytes(raw) + good = hashlib.sha256(raw).hexdigest() + encoded = base64.urlsafe_b64encode(hashlib.sha256(raw).digest()).rstrip(b"=") + + class _Hash: + mode = "sha256" + + def __init__(self, value: str) -> None: + self.value = value + + class _Entry(str): + hash = _Hash(encoded.decode()) + + class _Distribution: + version = "0.0.0-invented" + files = [_Entry(relative)] + + def locate_file(self, _name): + return root + + def distribution(name): + assert name == "invented-engine" + return _Distribution() + + real = metadata.distribution + try: + metadata.distribution = distribution + report, allowed = diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + assert report["count"] == 1 and allowed == {(root / relative).resolve()} + for pins in ( + ((relative, "0" * 64, len(raw)),), + ((relative, good, len(raw) + 1),), + (("invented_engine/parameters/absent.csv", good, len(raw)),), + (("../outside.csv", good, len(raw)),), + ): + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=pins + ) + _Entry.hash = _Hash("AAAA") # RECORD disagrees with the file bytes. + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + finally: + metadata.distribution = real + + +def test_hook_opens_only_the_allowed_data_files(diagnostic, monkeypatch): + allowed = frozenset({Path("/invented/site-packages/engine/parameters/pinned.csv")}) + hook, distinct = _boundary(diagnostic, monkeypatch, allowed) + hook("open", (str(next(iter(allowed))), "r", os.O_RDONLY)) + assert distinct == [] + for path in ( + "/invented/site-packages/engine/parameters/other.csv", + "/invented/site-packages/engine/parameters/pinned.csv.bak.csv", + "/invented/repo/pinned.csv", + ): + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + hook("open", (path, "r", os.O_RDONLY)) + assert distinct == ["DATA_FILE"] + + +def test_proc_stat_is_readable_metadata(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + hook("open", ("/proc/stat", "r", os.O_RDONLY)) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$"): + hook("open", ("/proc/version", "r", os.O_RDONLY)) + + +def _call_as_has_ipv6(diagnostic, event, args): + """Invoke the hook from a frame named like urllib3's probe function.""" + + code = compile( + "def _has_ipv6(hook, event, args):\n return hook(event, args)\n", + "/invented/site-packages/urllib3/util/connection.py", + "exec", + ) + namespace: dict[str, object] = {} + exec(code, namespace) + return namespace["_has_ipv6"](diagnostic, event, args) + + +def test_urllib3_ipv6_probe_is_the_only_accepted_socket_use(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + sock = types.SimpleNamespace() + _call_as_has_ipv6(hook, "socket.__new__", (sock, 10, 1, 0)) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("::1", 0))) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("127.0.0.1", 0))) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.bind", (sock, ("0.0.0.0", 0))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.connect", (sock, ("::1", 80))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.__new__", (sock, 10, 1, 0)) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.bind", (sock, ("::1", 0))) + assert distinct == ["NETWORK_OR_CHILD"] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py index 4d889c763..0ee55f9a9 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py @@ -42,7 +42,7 @@ def test_caught_middle_child_denial_keeps_own_context(diagnostic, audited): terminal = None secret = "invented-secret-never-retain" for event, args, code in ( - ("open", ("/proc/stat", "r", os.O_RDONLY), "READ_SCOPE"), + ("open", ("/proc/version", "r", os.O_RDONLY), "READ_SCOPE"), ( "subprocess.Popen", (secret, [secret], secret, {secret: secret}), @@ -112,7 +112,7 @@ def test_eight_record_capacity_never_discards_or_grows(diagnostic, audited): contexts.update({code: diagnostic.encoded({"code": code}) for code in codes}) retained = tuple(contexts.items()) with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$") as error: - hook("open", ("/proc/stat", "r", os.O_RDONLY)) + hook("open", ("/proc/version", "r", os.O_RDONLY)) assert tuple(contexts.items()) == retained and len(contexts) == 8 assert refusals == distinct == ["READ_SCOPE"] assert first == [error.value.boundary_context] diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 7b1616c3a..1c9d87dbe 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import base64 import contextlib import hashlib import importlib.metadata @@ -164,6 +165,111 @@ ) +#: Public parameter data files the pinned policyengine-us distribution reads +#: at import time (pandas.read_csv at module import: HUD fair market rents, +#: income limits, payment standards and utility allowances; HHS rating areas +#: and second-lowest silver plan costs). The derive phase must import the +#: engine, so these exact files -- and nothing else with a data suffix -- may +#: be opened, and only when their bytes hash to the pin. The pins are the +#: distribution's RECORD digests, verified against policyengine-us 1.819.0 on +#: 2026-09-11. A distribution whose files differ fails closed before the +#: boundary is armed (ENGINE_PARAMETER_PIN). +ENGINE_PUBLIC_PARAMETER_FILES_POLICY = "pinned_engine_public_parameter_files_v1" +ENGINE_PUBLIC_PARAMETER_DISTRIBUTION = "policyengine-us" +ENGINE_PUBLIC_PARAMETER_FILES = ( + ( + "policyengine_us/parameters/gov/hhs/medicaid/geography/aca_rating_areas.csv", + "a953284013a051956e1951f26e31252fcdf2275010953d09fd16546cb3df508e", + 68571, + ), + ( + "policyengine_us/parameters/gov/hhs/medicaid/geography/medicaid_rating_areas.csv", + "98bd49798e56026262002fb82699b7940d6bfb26589934909682f355ee4e086e", + 59225, + ), + ( + "policyengine_us/parameters/gov/hhs/medicaid/geography/second_lowest_silver_plan_cost.csv", + "b2e415151d5ecc662b62e8b05ea3e1878dcad9e63384c255a314ea3254e47c55", + 293454, + ), + ( + "policyengine_us/parameters/gov/hud/fmr/fair_market_rents.csv", + "aaaa3fe7935e553c5333da51722640704351040092dbf8102c2b19b86a373183", + 1314027, + ), + ( + "policyengine_us/parameters/gov/hud/fmr/small_area_fair_market_rents.csv", + "3ae1edbe6491bafd0b4c4d66401d914e9edf37304f8d93eb43a786a3e4d98ec1", + 246698, + ), + ( + "policyengine_us/parameters/gov/hud/income_limits/section8_income_limits.csv", + "2a8de86c81a8806e75eb15b278a9708e855cd241ba4cc564871c94a565f46442", + 1570746, + ), + ( + "policyengine_us/parameters/gov/hud/payment_standards/zip_code_payment_standards.csv", + "66179687d3e0d9d6c99a58a528aad672ee6a3af8820189209eac6be1ba02d097", + 84054, + ), + ( + "policyengine_us/parameters/gov/hud/utility_allowance/county_utility_allowances.csv", + "0642a75980473ffa33be12d51a3c58ed735472598109b0998b3634d918563809", + 3913, + ), +) + + +def pinned_engine_parameter_files( + *, + distribution: str = ENGINE_PUBLIC_PARAMETER_DISTRIBUTION, + pins: tuple[tuple[str, str, int], ...] = ENGINE_PUBLIC_PARAMETER_FILES, +) -> tuple[dict, frozenset[Path]]: + """Verify the pinned engine parameter files before the boundary is armed. + + Locates the installed distribution through its metadata (the engine is + not imported), requires every pinned file to be a regular file inside the + distribution root whose bytes hash to the pin and whose RECORD entry + carries that digest, and returns the bounded report plus the resolved + paths the audit hook may allow. Hashing happens once here; the hook only + compares resolved paths. + """ + from importlib import metadata + + try: + dist = metadata.distribution(distribution) + except metadata.PackageNotFoundError: + raise RefusalError("ENGINE_PARAMETER_PIN") from None + root = Path(dist.locate_file("")).resolve() + record = {} + for entry in dist.files or (): + if entry.hash is not None and entry.hash.mode == "sha256": + padded = entry.hash.value + "=" * (-len(entry.hash.value) % 4) + record[str(entry)] = base64.urlsafe_b64decode(padded).hex() + allowed: set[Path] = set() + files = [] + for relative, expected, size in pins: + path = (root / relative).resolve() + require(path.is_relative_to(root), "ENGINE_PARAMETER_PIN") + require(path.is_file() and not path.is_symlink(), "ENGINE_PARAMETER_PIN") + raw = path.read_bytes() + actual = digest(raw) + require(len(raw) == size and actual == expected, "ENGINE_PARAMETER_PIN") + require(record.get(relative) == expected, "ENGINE_PARAMETER_PIN") + allowed.add(path) + files.append({"path": relative, "sha256": expected, "size": size}) + return ( + { + "policy": ENGINE_PUBLIC_PARAMETER_FILES_POLICY, + "distribution": distribution, + "version": dist.version, + "files": files, + "count": len(files), + }, + frozenset(allowed), + ) + + class RefusalError(RuntimeError): """A fixed-code refusal with optional immutable, bounded code metadata.""" @@ -323,6 +429,39 @@ def read_refusal_context(code, event, path, requested, roots) -> bytes: return payload +_LOOPBACK_PROBE_HOSTS = frozenset({"::1", "127.0.0.1"}) + + +def _is_urllib3_ipv6_probe(event: str, args: tuple[object, ...]) -> bool: + """Recognise urllib3's import-time IPv6 capability probe and nothing else. + + ``urllib3.util.connection._has_ipv6`` creates one socket and binds it to + the loopback address on an ephemeral port; it never connects, sends or + resolves a name. Only ``socket.__new__`` and a loopback ``socket.bind`` + issued from that exact function are accepted; every other socket event + keeps the NETWORK_OR_CHILD refusal. + """ + if event == "socket.bind": + address = args[1] if len(args) > 1 else None + if not ( + isinstance(address, tuple) + and address + and address[0] in _LOOPBACK_PROBE_HOSTS + ): + return False + elif event != "socket.__new__": + return False + frame = sys._getframe(1) + while frame is not None: + code = frame.f_code + if code.co_name == "_has_ipv6" and code.co_filename.replace("\\", "/").endswith( + "urllib3/util/connection.py" + ): + return True + frame = frame.f_back + return False + + def install_boundary( root: Path, owned: Path, @@ -331,6 +470,7 @@ def install_boundary( first_refusal: list[bytes] | None = None, distinct_refusals: list[str] | None = None, code_contexts: dict[str, bytes] | None = None, + allowed_data_files: frozenset[Path] = frozenset(), ) -> list[str]: refusals: list[str] = [] blocked_files = ( @@ -368,6 +508,12 @@ def install_boundary( Path("/proc/self/maps"), # operation_path resolves /proc/self to this process's numeric PID. Path(f"/proc/{os.getpid()}/maps"), + # psutil reads the CPU-times field layout from /proc/stat when it is + # imported (joblib's loky backend imports it during the derive phase); + # metadata only, like /proc/cpuinfo above. + Path("/proc/stat"), + # The pinned engine parameter files verified before the hook was armed. + *allowed_data_files, } context_roots = ( @@ -440,7 +586,11 @@ def owned_mutation( def audit(event: str, args: tuple[object, ...]) -> None: # gethostname reads a local OS label; it opens no network connection. if ( - (event.startswith("socket.") and event != "socket.gethostname") + ( + event.startswith("socket.") + and event != "socket.gethostname" + and not _is_urllib3_ipv6_probe(event, args) + ) or event.startswith("subprocess.") or event in { @@ -485,7 +635,7 @@ def audit(event: str, args: tuple[object, ...]) -> None: if event == "open" and args: path = operation_path(args[0]) name = str(path) - if name.lower().endswith(blocked_files): + if name.lower().endswith(blocked_files) and path not in allowed_data_files: refuse("DATA_FILE", event="open", path=path, requested=args[0]) flags = args[2] if len(args) > 2 else 0 writing = isinstance(flags, int) and bool( @@ -959,6 +1109,7 @@ def main() -> int: first_refusal: list[bytes] = [] distinct_refusals: list[str] = [] code_contexts: dict[str, bytes] = {} + engine_parameter_files, allowed_data_files = pinned_engine_parameter_files() refusals = install_boundary( root, owned, @@ -966,6 +1117,7 @@ def main() -> int: first_refusal=first_refusal, distinct_refusals=distinct_refusals, code_contexts=code_contexts, + allowed_data_files=allowed_data_files, ) # Dependency-created temporaries must stay inside this invocation's scope. tempfile.tempdir = str(owned) @@ -994,6 +1146,7 @@ def main() -> int: "contents": "invented_spec_fixture_and_dependency_temporaries", "uploaded": False, }, + "engine_public_parameter_files": engine_parameter_files, } try: bootstrap["absent_stdlib_zip_omitted"] = omit_absent_stdlib_zip() From 123650f6f8d4782950f72ae01e4bb68a28345074 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 09:36:37 -0400 Subject: [PATCH 080/255] Complete the seed-identity diagnostic's kernel-module roster The pinned roster omitted microcosm.calibrate.gates and microcosm.calibrate.initialization, which the seed protocol's direct-kernel attestation has listed since before this branch (spec_engine/seeds.py is unchanged against main), so the derive phase refused SEED_MODULE_ROSTER on every run once the boundary let the engine import through. Both modules and their source files join the roster and the stamped source paths; the loaded US bundle's 44-module roster now matches the pin. --- tools/spec_seed_identity_diagnostics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 1c9d87dbe..3b3fb618c 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -114,6 +114,8 @@ "packages/microcosm-build/src/microcosm/build/us_runtime/workers_compensation.py", "packages/microcosm-build/tests/test_spec_engine_loader.py", "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/initialization.py", "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py", "packages/microcosm-fit/src/microcosm/fit/qrf.py", "packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py", @@ -160,6 +162,8 @@ "microcosm.build.us_runtime.wic_claim", "microcosm.build.us_runtime.workers_compensation", "microcosm.calibrate.exact_k", + "microcosm.calibrate.gates", + "microcosm.calibrate.initialization", "microcosm.calibrate.solve", "microcosm.fit.qrf", ) From 1007c0e696adf7fe0ab08ae39053614df44277fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 10:36:51 -0400 Subject: [PATCH 081/255] Re-pin the AM/BE/UK bundle digests and the loader's golden semantic hash The seed protocol's content digest moved on this branch (three attested kernel modules changed: acs_transfer, housing_inputs, fit.qrf) and every country bundle's spec_sha256 and the loader's golden semantic-hash vector embed it, so these four pins move with the seed re-pin already recorded. Main is fully merged; spec_engine and the country specs are unchanged. The shared-spec CI group's remaining local failures (release-target parity regeneration) need the 131 MB local facts feed that CI does not have and are recorded separately. --- .../tests/test_spec_engine_country_bundles.py | 6 +++--- packages/microcosm-build/tests/test_spec_engine_loader.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 1169547c3..13270e3d1 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "d983a5e6bb2f91f9abd44669fe5b1c795d1a7597c4b6acbe9282eaa956bb326d" +AM_SPEC_SHA256 = "e2b275861eee20ee8184d030aa22a6d7fad2bc2fd9922588b3ca8566f81c5878" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "2a7d83483c90abe31108f8c3a77c053b4e6f2fd50d3cf3290a85e63ea3b9fbac", + "938e842e4e72f9319b06edfe4055ef2c0190ae942c8dcb801a2effe1fd6f4b98", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "2269bc281ada99dce077730657e3868972e025b48be5f9db8c11b3cc63285bac", + "3130d6bee3e2e06ec514ffc452bc6343d8cf3394fb0f16c055d263722254e9e3", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 0b583de0a..f4c01d8a6 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "57026e5896dd52a382746fe7641f615dcc4c55965a1f50bea5971a8f92710d71" + "a5048106b31c856308aaf4dda1792a443e20254124785cdb4eb784adc456a1d5" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") From c2c6b37df6fd1e5ecac0afe6f13cab94940da83a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:11:09 -0400 Subject: [PATCH 082/255] Reuse the ASEC catalogue records identity over unchanged immutable leaves Every checked borrow of the ASEC source catalogue re-encoded and re-hashed all household and unrepresented records twice, and the survey population preparation's nested seal did it a third time (~2 s per pass on the real catalogue; the native pilot spent ~20% of its CPU there). The catalogue now binds a leaf snapshot to the identity in the same streaming pass at issuance, and later borrows reuse that identity only when the issued root tuples are the same objects, the memo identity equals the owner's retained one, and every projected leaf (literals, keys, person rosters, members' fields) is the very object seen while streaming. Frozen dataclasses stay writable through object.__setattr__, so the leaves carry the proof; any subclass leaf, replaced container, or foreign type keeps the full canonical encoding and its refusals. Inventory scope review for asec_population_catalogue.py: no new imports or resource accesses; the only new unbound uses are the survey_population_domains references inside _eligible_key and _eligible_leaves (exact record/key/source type checks), so the module's unbound_uses digest is re-pinned. Co-Authored-By: Claude Fable 5 --- .../us_runtime/asec_population_catalogue.py | 179 ++++++++++- .../graph_implementation_inventory.json | 2 +- .../survey_population_preparation.py | 4 +- .../test_us_asec_catalogue_records_memo.py | 298 ++++++++++++++++++ 4 files changed, 472 insertions(+), 11 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py index 09d971b68..c7db08401 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -10,10 +10,12 @@ import hashlib import json +import operator import sys import weakref from collections.abc import Mapping from dataclasses import asdict, dataclass +from itertools import chain from pathlib import Path from typing import NamedTuple @@ -144,9 +146,27 @@ def _record(row): return result -def _records_identity(households, ledger): +class _RecordsMemo(NamedTuple): + """The issued record roots, their projected leaves, and the identity streamed.""" + + households: tuple + ledger: tuple + household_leaves: tuple + ledger_leaves: tuple + identity: tuple + + +def _records_identity(households, ledger, *, memo=False): + """Stream every canonical record; optionally bind a leaf snapshot memo. + + With ``memo=True`` the result is ``(identity, memo)``. The memo's leaves are + projected from the exact rows encoded in this same pass, so it can only + stand for this identity. It is ``None`` when any record carries a leaf + that is not an exact ``str``/``int`` or the ASEC source member. + """ _require(type(households) is tuple and type(ledger) is tuple, "RECORD_TYPE") digest, count, size, persons = hashlib.sha256(), 0, 0, 0 + household_leaves, ledger_leaves, eligible = [], [], memo for kind, rows in (("household", households), ("unrepresented", ledger)): for row in rows: data = _encode([kind, _record(row)]) + b"\n" @@ -156,7 +176,141 @@ def _records_identity(households, ledger): count += 1 if kind == "household": persons += len(row.persons) - return (digest.hexdigest(), size, count, persons) + if eligible: + projected = _eligible_leaves(row, kind) + if projected is None: + eligible = False + elif kind == "household": + household_leaves.append(projected) + else: + ledger_leaves.append(projected) + identity = (digest.hexdigest(), size, count, persons) + if not memo: + return identity + if not eligible: + return identity, None + return identity, _RecordsMemo( + households, ledger, tuple(household_leaves), tuple(ledger_leaves), identity + ) + + +_KEY_LEAVES = operator.attrgetter("source", "source_year", "survey_year", "native_id") +_HOUSEHOLD_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt" +) +_HOUSEHOLD_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "persons" +) +_PERSON_LITERALS = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state" +) +_PERSON_LEAVES = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state", "household_key" +) +_LEDGER_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) +_LEDGER_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) + + +def _household_leaves(row): + key, *literals, persons = _HOUSEHOLD_LEAVES(row) + return ( + key, + *_KEY_LEAVES(key), + *literals, + persons, + *chain.from_iterable( + (*_PERSON_LEAVES(person), *_KEY_LEAVES(person.household_key)) + for person in persons + ), + ) + + +def _ledger_leaves(row): + key, *literals = _LEDGER_LEAVES(row) + return (key, *_KEY_LEAVES(key), *literals) + + +def _eligible_key(key): + return ( + type(key) is domains.HouseholdKey + and key.source is domains.Source.ASEC + and type(key.source_year) is int + and type(key.survey_year) is int + and type(key.native_id) is str + ) + + +def _eligible_leaves(row, kind): + """Project one record's leaves when every one is an exact immutable value.""" + if not _eligible_key(row.key): + return None + if kind == "household": + if type(row) is not domains.AsecHousehold or type(row.persons) is not tuple: + return None + literals = list(_HOUSEHOLD_LITERALS(row)) + for person in row.persons: + if type(person) is not domains.AsecPerson or not _eligible_key( + person.household_key + ): + return None + literals.extend(_PERSON_LITERALS(person)) + leaves = _household_leaves(row) + else: + if type(row) is not UnrepresentedAsecHousehold: + return None + literals = _LEDGER_LITERALS(row) + leaves = _ledger_leaves(row) + if any(type(item) is not str for item in literals): + return None + return leaves + + +def _leaves_unchanged(rows, snapshot, project): + if len(rows) != len(snapshot): + return False + for row, expected in zip(rows, snapshot, strict=True): + try: + current = project(row) + except (AttributeError, TypeError): + return False + if len(current) != len(expected) or not all( + map(operator.is_, current, expected) + ): + return False + return True + + +def _memoized_records_identity( + households, ledger, memo=None, *, expected_identity=None +): + """Reuse the issued records identity only over the same unchanged leaves. + + A hit needs the exact issued household and ledger tuples, a memo identity + equal to the owner's retained one, and every projected leaf (literals, + keys, person rosters and their members' fields) to be the very object seen + while the identity streamed. Frozen dataclasses stay writable through + ``object.__setattr__``, so the leaves carry the proof, not the records. + Anything else falls back to the full canonical encoding and its refusals. + """ + if ( + type(households) is tuple + and type(ledger) is tuple + and type(memo) is _RecordsMemo + and households is memo.households + and ledger is memo.ledger + and type(memo.household_leaves) is tuple + and type(memo.ledger_leaves) is tuple + and type(memo.identity) is tuple + and memo.identity == expected_identity + and _leaves_unchanged(households, memo.household_leaves, _household_leaves) + and _leaves_unchanged(ledger, memo.ledger_leaves, _ledger_leaves) + ): + return memo.identity + return _records_identity(households, ledger) def _bound_records_identity(rows, roster): @@ -234,6 +388,16 @@ class _State(NamedTuple): households: tuple ledger: tuple records_identity: tuple + records_memo: _RecordsMemo | None = None + + +def _current_records_identity(state): + return _memoized_records_identity( + state.households, + state.ledger, + state.records_memo, + expected_identity=state.records_identity, + ) def _validate_state(state): @@ -245,8 +409,7 @@ def _validate_state(state): "PARENT_CHANGED", ) _require( - _records_identity(state.households, state.ledger) == state.records_identity, - "RECORDS_CHANGED", + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" ) state.parent.validate() native.coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) @@ -261,8 +424,7 @@ def _validate_state(state): "PARENT_CHANGED", ) _require( - _records_identity(state.households, state.ledger) == state.records_identity, - "RECORDS_CHANGED", + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" ) # Pure final seals after source/resource I/O; no recursively repeated reads. current = native._attached_evidence( @@ -389,7 +551,9 @@ def issue_asec_source_catalogue( ) _, _, rows, roster = native._roster(parent, coverage, anchors, fields, None) households, ledger = _rows(rows, roster, anchors, fields) - records_identity = _records_identity(households, ledger) + records_identity, records_memo = _records_identity( + households, ledger, memo=True + ) identity = json.loads(parent.source.identity) files = [ ( @@ -440,6 +604,7 @@ def issue_asec_source_catalogue( households, ledger, records_identity, + records_memo, ) payload = _encode( { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index ebd5b5066..2bc25474c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -429,7 +429,7 @@ "microcosm.build.us_runtime" ], "resource_accesses_sha256": "2622beb3c86b2a9e1cea2db1c6b34a1ec2d2cf29824c3d3e276ea41baeb4f93b", - "unbound_uses_sha256": "05ea2b2c3d774e57d76e89616228798fa8ef05c1f6c8524b08816b5ad3844a81" + "unbound_uses_sha256": "138ac0967786b481d52704c91d5d5cefc4e39f92d7cab4154215bc77a981ef6e" }, "microcosm.build/us_runtime/asec_prepared_source.py": { "imports": [ diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 3bb7d8603..72428c0d1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -1076,9 +1076,7 @@ def _nested_seals( ) ) if module is asec_catalogue: - values.append( - asec_catalogue._records_identity(state.households, state.ledger) - ) + values.append(asec_catalogue._current_records_identity(state)) return tuple(values) diff --git a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py new file mode 100644 index 000000000..5fdb14e8f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py @@ -0,0 +1,298 @@ +"""Immutable-leaf reuse of the ASEC records identity plus actual issuer controls. + +Supplied records exercise private value helpers only; they are never treated +as an authenticated source catalogue. The last controls use real issuance over +the maintained invented preparation fixture. +""" + +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_population_catalogue as owner +from microcosm.build.us_runtime import survey_population_domains as domains +from microcosm.build.us_runtime import survey_population_preparation as preparation + + +def _key(native_id="1"): + return domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, native_id) + + +def _records(): + key = _key() + person = domains.AsecPerson("2024000000000000000001", "1", "37", "1", "1", key) + household = domains.AsecHousehold(key, "1", "1", "1", "1", "1234567", (person,)) + ledger = owner.UnrepresentedAsecHousehold(_key("2"), "2", "9", "1", "0", "7654321") + return (household,), (ledger,) + + +def _calls(action): + calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is owner._records_identity.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = action() + finally: + sys.setprofile(previous) + return result, len(calls) + + +def _issued(households, ledger): + (identity, memo), calls = _calls( + lambda: owner._records_identity(households, ledger, memo=True) + ) + assert calls == 1 + return identity, memo + + +def _memoized(households, ledger, memo, identity): + return _calls( + lambda: owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + ) + + +def test_same_issued_leaves_reuse_exact_identity_without_another_encode(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + assert identity == owner._records_identity(households, ledger) + assert type(memo) is owner._RecordsMemo + assert memo.households is households and memo.ledger is ledger + assert memo.identity == identity + assert memo.household_leaves == (owner._household_leaves(households[0]),) + assert memo.ledger_leaves == (owner._ledger_leaves(ledger[0]),) + result, calls = _calls( + lambda: [ + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + for _ in range(3) + ] + ) + assert result == [identity] * 3 and calls == 0 + + +def test_equal_replacement_roots_miss_without_changing_identity_or_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + replacement = (tuple(list(households)), tuple(list(ledger))) + assert replacement == (households, ledger) + assert replacement[0] is not households and replacement[1] is not ledger + result, calls = _memoized(*replacement, memo, identity) + assert result == identity and calls == 1 + assert memo.households is households and memo.ledger is ledger + + +@pytest.mark.parametrize( + "target", + ["household", "person", "household_key", "person_key", "ledger", "reason"], +) +def test_in_place_leaf_change_misses_and_changes_identity(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household, person = households[0], households[0].persons[0] + if target == "household": + object.__setattr__(household, "hsup_wgt", "0000000") + elif target == "person": + object.__setattr__(person, "age", "38") + elif target == "household_key": + object.__setattr__(household.key, "native_id", "9") + elif target == "person_key": + object.__setattr__(person, "household_key", _key("9")) + elif target == "ledger": + object.__setattr__(ledger[0], "h_numper", "1") + else: + object.__setattr__(ledger[0], "reason", "other") + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + + +@pytest.mark.parametrize("target", ["persons", "key"]) +def test_replaced_container_misses_even_when_equal(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household = households[0] + original = getattr(household, target) + replacement = ( + tuple(list(original)) if target == "persons" else _key(original.native_id) + ) + assert replacement == original and replacement is not original + object.__setattr__(household, target, replacement) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_persons_replaced_by_a_list_takes_the_refusing_path(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__setattr__(households[0], "persons", list(households[0].persons)) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +def test_deleted_leaf_misses_instead_of_raising_inside_the_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__delattr__(households[0].persons[0], "age") + with pytest.raises(AttributeError): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +@pytest.mark.parametrize( + "kind", + [ + "str_subclass_literal", + "str_subclass_person", + "str_subclass_reason", + "int_subclass_year", + "str_subclass_native_id", + ], +) +def test_subclass_leaves_yield_no_memo_and_keep_the_full_identity_path(kind): + class StrSubclass(str): + pass + + class IntSubclass(int): + pass + + households, ledger = _records() + household, person = households[0], households[0].persons[0] + if kind == "str_subclass_literal": + object.__setattr__(household, "h_hhtype", StrSubclass("1")) + elif kind == "str_subclass_person": + object.__setattr__(person, "prpertyp", StrSubclass("1")) + elif kind == "str_subclass_reason": + object.__setattr__(ledger[0], "reason", StrSubclass("x")) + elif kind == "int_subclass_year": + object.__setattr__(household.key, "source_year", IntSubclass(2024)) + else: + object.__setattr__(ledger[0].key, "native_id", StrSubclass("2")) + identity, memo = _issued(households, ledger) + assert memo is None + assert identity == owner._records_identity(households, ledger) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_memo_requires_agreement_with_the_owner_identity(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + wrong = (*identity[:3], identity[3] + 1) + result, calls = _memoized(households, ledger, memo, wrong) + assert result == identity and calls == 1 + drifted = memo._replace(identity=wrong) + result, calls = _memoized(households, ledger, drifted, identity) + assert result == identity and calls == 1 + plain = tuple(memo) + assert plain == memo + result, calls = _memoized(households, ledger, plain, identity) + assert result == identity and calls == 1 + + +def test_swapped_memo_entry_misses(): + households, ledger = _records() + identity, _ = _issued(households, ledger) + other_households, other_ledger = _records() + object.__setattr__(other_households[0], "hsup_wgt", "0000001") + other_identity, other_memo = _issued(other_households, other_ledger) + assert other_identity != identity + result, calls = _memoized(households, ledger, other_memo, identity) + assert result == identity and calls == 1 + result, calls = _memoized(households, ledger, other_memo, other_identity) + assert result == identity and calls == 1 + + +def test_crafted_memo_over_lists_still_refuses(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + rows = list(households) + crafted = owner._RecordsMemo( + rows, ledger, memo.household_leaves, memo.ledger_leaves, identity + ) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + rows, ledger, crafted, expected_identity=identity + ) + + +def test_wrong_source_member_or_foreign_types_yield_no_memo(): + households, ledger = _records() + foreign = domains.HouseholdKey(domains.Source.ACS, 2024, 2025, "1") + object.__setattr__(ledger[0], "key", foreign) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_SOURCE$"): + owner._records_identity(households, ledger, memo=True) + assert owner._eligible_leaves(ledger[0], "unrepresented") is None + assert owner._eligible_leaves(households[0], "unrepresented") is None + assert owner._eligible_leaves(ledger[0], "household") is None + + +@pytest.fixture(scope="module") +def actual_preparation(tmp_path_factory): + with pytest.MonkeyPatch.context() as monkeypatch: + arguments = fixture(tmp_path_factory.mktemp("asec-records-memo"), monkeypatch) + result = preparation.prepare_authenticated_survey_population(**arguments) + state = preparation._ISSUED[id(result)][2] + catalogue = state.catalogues[1] + catalogue_state = owner._ISSUED[id(catalogue)][2] + assert type(catalogue_state.records_memo) is owner._RecordsMemo + assert catalogue_state.records_memo.identity == catalogue_state.records_identity + yield result, catalogue, catalogue_state + result.validate() + catalogue.validate() + + +def test_actual_borrows_reuse_the_issued_identity(actual_preparation): + result, catalogue, _ = actual_preparation + _, calls = _calls(catalogue.validate) + assert calls == 0 + _, calls = _calls(result.checked_view) + assert calls == 0 + + +def test_actual_household_leaf_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + row = catalogue_state.households[0] + original = row.hsup_wgt + object.__setattr__(row, "hsup_wgt", original + "0") + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(row, "hsup_wgt", original) + _, calls = _calls(catalogue.validate) + assert calls == 0 + + +def test_actual_person_key_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + person = catalogue_state.households[0].persons[0] + original = person.household_key + object.__setattr__(person, "household_key", _key(original.native_id + "0")) + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(person, "household_key", original) + result.validate() From d872960b92fb0b803948f6cd305747d780849b92 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:06:30 -0400 Subject: [PATCH 083/255] Carry a calibration hierarchy on every national demographic count target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's diagnostics schema 8 (#855) requires a complete hierarchy for every registry-backed target, so the `national.demographic_calibration` node failed on the merge ref: its S0101 registry carried none. Each activated cell now declares the only hierarchy it may have — provider `census_acs`, category `census_acs.population_by_age` (S0101) or `census_acs.population_by_sex_and_age` (B01001), both labels added to the reviewed US variable-label catalog, the activation's own national publisher row (0100000US, "United States", level country), one `publisher_cell` dimension whose value is the cell and whose label is the published one the band binds, and the cell itself as the target so its id equals the spec name. Both S0101 spec builders attach it; the registry declaration round- trips it; the registry validator refuses a spec without exactly the hierarchy rebuilt from its table, name, label and geography; the survey diagnostics verifier's defensive pin moves from schema 6 to 8. Controls: the invented seven-node graph's diagnostics are schema 8 with a hierarchy on every row; the registry JSON round-trips every hierarchy; missing, foreign-cell, foreign-geography and foreign-category hierarchies are refused; the label catalog pin lists the two new rows. Co-Authored-By: Claude Fable 5 --- .../demographic_calibration_graph.py | 27 ++++++- .../us_runtime/national_age_activation.py | 73 ++++++++++++++++++ .../build/us_runtime/survey_age_activation.py | 3 + .../survey_calibration_diagnostics.py | 9 ++- .../tests/test_us_national_age_counts.py | 77 +++++++++++++++++++ .../microcosm/calibrate/variable_labels.py | 2 + .../tests/test_variable_labels.py | 2 + 7 files changed, 186 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py index 85d39944f..6a174053b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/demographic_calibration_graph.py @@ -23,6 +23,7 @@ from pandas.api.types import is_complex_dtype, is_numeric_dtype import microcosm.build.cd_benchmark.protocol as protocol_module +import microcosm.build.us_runtime.national_age_activation as age_module import microcosm.calibrate._target_loss_attribution as attribution_module import microcosm.calibrate.diagnostics as diagnostics_module import microcosm.calibrate.kernels as kernels_module @@ -31,6 +32,7 @@ import microcosm.calibrate.solve as solve_module from microcosm.build.cd_benchmark.protocol import RESERVED_FAMILIES from microcosm.calibrate import calibrate, diagnostics_payload +from microcosm.calibrate.hierarchy import CalibrationHierarchy from microcosm.calibrate.registry import TargetRegistry, TargetSpec from microcosm.frame import WeightKind from microcosm.graph import ( @@ -121,6 +123,18 @@ def _validate_registry(registry: TargetRegistry) -> None: raise ValueError( "Demographic target scope must be explicit national population counts for 2024." ) + if spec.hierarchy is None or spec.hierarchy != ( + age_module.expected_demographic_hierarchy( + table, + variable=spec.name, + label=spec.hierarchy.target.label, + geography=metadata["geography"], + ) + ): + raise ValueError( + "Each demographic target must carry exactly its own national " + "Census ACS calibration hierarchy." + ) scopes.add(metadata["evidence_scope"]) if len(scopes) != 1: raise ValueError( @@ -134,9 +148,15 @@ def _registry_from_json(text: str) -> TargetRegistry: document = json.loads(text) if not isinstance(document, dict) or set(document) != {"country", "specs"}: raise ValueError("Unsupported registry declaration shape.") - registry = TargetRegistry( - [TargetSpec(**raw) for raw in document["specs"]], country=document["country"] - ) + specs = [] + for raw in document["specs"]: + if not isinstance(raw, dict): + raise ValueError("Unsupported registry declaration shape.") + hierarchy = raw.get("hierarchy") + if hierarchy is not None: + raw = {**raw, "hierarchy": CalibrationHierarchy.from_dict(hierarchy)} + specs.append(TargetSpec(**raw)) + registry = TargetRegistry(specs, country=document["country"]) if text != _registry_json(registry): raise ValueError( "Registry declaration must have its canonical typed representation." @@ -240,6 +260,7 @@ def implementation_hash(self) -> str: { "adapter": source_hash( sys.modules[__name__], + age_module, registry_module, diagnostics_module, attribution_module, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py index b0ba5fac6..bf1a03535 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py @@ -42,7 +42,16 @@ from dataclasses import asdict, dataclass from pathlib import Path +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyDimension, + HierarchyGeography, + HierarchyNode, +) +from microcosm.calibrate.provider_labels import calibration_provider_label from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.calibrate.variable_labels import calibration_variable_label from ..cd_benchmark.protocol import RESERVED_FAMILIES from .cd_reference import AUTHORITIES, classify_acs_value @@ -131,6 +140,67 @@ def __post_init__(self) -> None: ) +#: Diagnostics schema 8 hierarchy vocabulary for activated national age cells. +#: The provider and category labels come from the reviewed US label catalogs; +#: the geography is the activation's own national publisher row; the single +#: dimension is the publisher cell whose published label the band binds. +HIERARCHY_PROVIDER = "census_acs" +HIERARCHY_DIMENSION_ID = "publisher_cell" +HIERARCHY_DIMENSION_LABEL = "Publisher cell" +NATIONAL_GEOGRAPHY_ID = "0100000US" +NATIONAL_GEOGRAPHY_LABEL = "United States" +NATIONAL_GEOGRAPHY_LEVEL = "country" + + +def _hierarchy_category_id(table: str) -> str: + if table == "S0101": + return "population_by_age" + if table == "B01001": + return "population_by_sex_and_age" + raise ActivationError(f"UNSUPPORTED_TABLE:{table}") + + +def expected_demographic_hierarchy( + table: str, *, variable: str, label: str, geography: str +) -> CalibrationHierarchy: + """The only hierarchy a national demographic count cell may carry. + + Everything but the published cell label is fixed by the table and the + national geography, so a registry validator can rebuild it from a spec and + refuse any other provider, category, geography, dimension, or target. + """ + _require(geography == NATIONAL_GEOGRAPHY_ID, f"UNSUPPORTED_GEOGRAPHY:{geography}") + category_id = _hierarchy_category_id(table) + provider_label = calibration_provider_label("us", HIERARCHY_PROVIDER) + category_label = calibration_variable_label("us", HIERARCHY_PROVIDER, category_id) + _require(bool(provider_label) and bool(category_label), "HIERARCHY_LABELS") + return CalibrationHierarchy( + provider=HierarchyNode(HIERARCHY_PROVIDER, provider_label), + category=HierarchyCategory( + f"{HIERARCHY_PROVIDER}.{category_id}", category_label, HIERARCHY_PROVIDER + ), + geography=HierarchyGeography( + NATIONAL_GEOGRAPHY_ID, NATIONAL_GEOGRAPHY_LABEL, NATIONAL_GEOGRAPHY_LEVEL + ), + dimensions=( + HierarchyDimension( + HIERARCHY_DIMENSION_ID, HIERARCHY_DIMENSION_LABEL, variable, label + ), + ), + target=HierarchyNode(variable, label), + ) + + +def demographic_target_hierarchy( + table: str, band: AgeBand, *, geography: str +) -> CalibrationHierarchy: + """The complete schema 8 hierarchy for one activated band's target spec.""" + _require(type(band) is AgeBand, "AGE_BAND_TYPE") + return expected_demographic_hierarchy( + table, variable=band.variable, label=band.label, geography=geography + ) + + def _bands() -> tuple[AgeBand, ...]: quinary = [(low, low + 4) for low in range(0, 85, 5)] bands = [] @@ -617,6 +687,9 @@ def derived_cell(variable: str) -> dict: ), source=f"{data_entry['url']} ({declaration.table} {variable}E)", family=f"acs.{declaration.table}", + hierarchy=demographic_target_hierarchy( + declaration.table, band, geography=declaration.geography + ), notes=( f"activation={digest}; band={band.low}-" f"{'+' if band.high is None else band.high}; " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py index 5ac898f64..deffa57ee 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py @@ -178,6 +178,9 @@ def cell(variable): se=age._standard_error(derived, band.variable, confidence_level=0.9), source=f"{data_url} (S0101 {band.variable}E)", family="acs.S0101", + hierarchy=age.demographic_target_hierarchy( + "S0101", band, geography="0100000US" + ), notes=f"activation={digest}; label={band.label}; age_convention={_CONVENTION}; published 90% MOE divided by {age._MOE_90_TO_SE}, not consumed by current loss", metadata={ "table": "S0101", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py index c0d6cd4dd..033322cc3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -72,11 +72,12 @@ def validate_survey_calibration_diagnostics( anchors, activation_binding=None, ): - """Return defensive schema-6 values plus an explicit verification annotation. + """Return defensive schema-8 values plus an explicit verification annotation. Every field except the four named history fields is reconstructed using score_targets and the package's diagnostics encoder. Only warning-free - schema 6 with target-loss attribution is supported in this first profile. + schema 8 (every registry-backed target carrying its calibration hierarchy) + with target-loss attribution is supported in this first profile. No convergence, optimality, target eligibility or release claim is granted. """ _require( @@ -185,14 +186,14 @@ def validate_survey_calibration_diagnostics( ) expected = diagnostics_payload(combined, target_registry=registry, build=anchors) _require( - expected.get("schema_version") == 6 + expected.get("schema_version") == 8 and expected.get("diagnostic_warnings") == [] and "target_loss_basis" in expected, "UNSUPPORTED_DIAGNOSTIC_SCHEMA", ) _require(canonical_json(expected) == payload, "RECOMPUTED_VALUES") # Make the distinction visible to downstream callers without altering the - # stored schema-6 evidence or its digest. + # stored schema-8 evidence or its digest. document["verification"] = { "protocol": "microcosm.us.survey-age-diagnostics-verification.v1", "artifact_sha256": hashlib.sha256(payload).hexdigest(), diff --git a/packages/microcosm-build/tests/test_us_national_age_counts.py b/packages/microcosm-build/tests/test_us_national_age_counts.py index 527de13b8..10b4934b0 100644 --- a/packages/microcosm-build/tests/test_us_national_age_counts.py +++ b/packages/microcosm-build/tests/test_us_national_age_counts.py @@ -19,6 +19,7 @@ import pandas as pd import pytest +from microcosm.build.us_runtime import demographic_calibration_graph as demographic from microcosm.build.us_runtime import graph_national_age_counts from microcosm.build.us_runtime.demographic_calibration_graph import ( DemographicCalibrationKernel, @@ -33,8 +34,10 @@ from microcosm.build.us_runtime.national_age_activation import ( NATIONAL_AGE_ACTIVATION, band_columns, + demographic_target_hierarchy, ) from microcosm.calibrate import calibrate, diagnostics_payload +from microcosm.calibrate.hierarchy import HierarchyCategory, HierarchyGeography from microcosm.calibrate.registry import TargetRegistry, TargetSpec from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.graph import ( @@ -139,6 +142,9 @@ EPOCHS, LEARNING_RATE, CAP = 240, 0.05, 4.0 +FIXTURE_BANDS = {band.variable: band for band in NATIONAL_AGE_ACTIVATION.bands} + + def fixture_registry() -> TargetRegistry: """Fictional values and identities; these are not Census observations.""" @@ -152,6 +158,9 @@ def fixture_registry() -> TargetRegistry: period="2024", source="Invented fixture; no published Census value", family="acs.S0101", + hierarchy=demographic_target_hierarchy( + "S0101", FIXTURE_BANDS[name], geography="0100000US" + ), metadata={ "table": "S0101", "reference_sha256": "a" * 64, @@ -881,6 +890,21 @@ def forbidden(context): def test_the_real_graph_counts_and_calibrates_the_invented_population(tmp_path): cold = graph_fixture(tmp_path) + diagnostics = cold["diagnostics"] + assert diagnostics["schema_version"] == 8 + rows = diagnostics["targets"] + assert [row["target_name"] for row in rows] == list(FIXTURE_TARGETS) + for row in rows: + hierarchy = row["hierarchy"] + assert hierarchy["target"]["id"] == row["target_name"] + assert hierarchy["provider"]["id"] == "census_acs" + assert hierarchy["category"]["id"] == "census_acs.population_by_age" + assert hierarchy["geography"] == { + "id": "0100000US", + "label": "United States", + "level": "country", + } + assert [d["value_id"] for d in hierarchy["dimensions"]] == [row["target_name"]] assert cold["hits"] == [False, False, False, False] assert cold["household_ids"] == list(HOUSEHOLD_IDS) assert cold["count_dtypes"] == ["int64"] @@ -954,3 +978,56 @@ def test_a_fresh_process_replays_warm_with_no_kernel_runs(tmp_path): assert {k: v for k, v in warm.items() if k != "hits"} == { k: v for k, v in cold.items() if k != "hits" } + + +def test_the_registry_json_round_trips_every_calibration_hierarchy(): + registry = fixture_registry() + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + assert [spec.hierarchy for spec in frozen] == [spec.hierarchy for spec in registry] + assert demographic._registry_json(frozen) == demographic._registry_json(registry) + for spec in frozen: + hierarchy = spec.hierarchy + assert hierarchy.provider.id == "census_acs" + assert hierarchy.category == HierarchyCategory( + "census_acs.population_by_age", "Population by age", "census_acs" + ) + assert hierarchy.geography == HierarchyGeography( + "0100000US", "United States", "country" + ) + (dimension,) = hierarchy.dimensions + assert dimension.id == "publisher_cell" + assert dimension.value_id == hierarchy.target.id == spec.name + assert dimension.value_label == hierarchy.target.label + assert hierarchy.target.label == FIXTURE_BANDS[spec.name].label + + +@pytest.mark.parametrize( + "kind", ["missing", "foreign_cell", "foreign_geography", "foreign_category"] +) +def test_a_spec_without_exactly_its_own_hierarchy_is_refused(kind): + registry = fixture_registry() + first, *rest = list(registry) + hierarchy = first.hierarchy + if kind == "missing": + hierarchy = None + elif kind == "foreign_cell": + (dimension,) = hierarchy.dimensions + hierarchy = replace( + hierarchy, dimensions=(replace(dimension, value_id="S0101_C01_019"),) + ) + elif kind == "foreign_geography": + hierarchy = replace( + hierarchy, + geography=HierarchyGeography("0400000US06", "California", "state"), + ) + else: + hierarchy = replace( + hierarchy, + category=HierarchyCategory( + "census_acs.resident_population", "Resident population", "census_acs" + ), + ) + changed = TargetRegistry([replace(first, hierarchy=hierarchy), *rest], country="us") + with pytest.raises(ValueError, match="calibration hierarchy"): + demographic._validate_registry(changed) + demographic._validate_registry(registry) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/variable_labels.py b/packages/microcosm-calibrate/src/microcosm/calibrate/variable_labels.py index 32a1425bd..8685c6fce 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/variable_labels.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/variable_labels.py @@ -52,6 +52,8 @@ def _immutable_registry( }, "census_acs": { "resident_population": "Resident population", + "population_by_age": "Population by age", + "population_by_sex_and_age": "Population by sex and age", }, "census_pep": { "resident_population": "Resident population", diff --git a/packages/microcosm-calibrate/tests/test_variable_labels.py b/packages/microcosm-calibrate/tests/test_variable_labels.py index 41654f183..b905bb8eb 100644 --- a/packages/microcosm-calibrate/tests/test_variable_labels.py +++ b/packages/microcosm-calibrate/tests/test_variable_labels.py @@ -27,6 +27,8 @@ def _pairs(value: str) -> set[tuple[str, str]]: cbo|qualified_dividend_income_projection cbo|wages_and_salaries_projection census_acs|resident_population +census_acs|population_by_age +census_acs|population_by_sex_and_age census_pep|resident_population census_stc|individual_income_tax_collections cms_aca|cms_aca From 72fbe7b5ed5bc4a47ec6547a40d4e8f3748ad027 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:06:30 -0400 Subject: [PATCH 084/255] Inject the late manifest mutation through the stage alias, not the sealed module `survey_origin_budget` seals its own functions in its producer check, so a monkeypatch of `verify_survey_weight_only_successor` on that module made `freeze_survey_origin_budget` refuse with PRODUCER_CHANGED before the seven-node graph ever ran, and the three terminal-manifest mutation controls could never reach the MANIFEST_NODE_STATE refusal they exist to prove. The seam is now a delegating stand-in for the stage's `budgets` alias, so the sealed module is untouched and the mutation lands exactly where it did before: after successor verification, before the final manifest-state check. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_survey_age_calibration_run.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py b/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py index 31adbd17b..11cdae554 100644 --- a/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py +++ b/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py @@ -338,8 +338,19 @@ def late(value): assert getattr(row, field) == value changed.append(True) + class _LateBudgets: + """The budgets module seals its own functions in its producer check, so + the late seam is injected through the stage's alias, never by editing + the sealed module.""" + + verify_survey_weight_only_successor = staticmethod(late) + + def __getattr__(self, name): + return getattr(actual_budgets, name) + + actual_budgets = stage.budgets monkeypatch.setattr(stage, "run_graph", capture) - monkeypatch.setattr(stage.budgets, "verify_survey_weight_only_successor", late) + monkeypatch.setattr(stage, "budgets", _LateBudgets()) arguments = authenticated_arguments(tmp_path, monkeypatch) arguments["seed_value"] = arguments.pop("seed") try: From 2ca11c85a0f710b8fa495e363243b940f7eabae2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:06:30 -0400 Subject: [PATCH 085/255] Project record classes and the source member's value into the memo leaves An independent read of c2c6b37df found two in-process changes that alter the canonical encoding without touching a projected leaf: reassigning the `Source.ASEC` member's `_value_` (the encoder writes `source.value`, the memo pinned only member identity), and reassigning `__class__` on a record, key or person to a layout-compatible subclass (the encoder refuses anything but the exact types on every full pass). Both now miss: each key projects `type(key)`, the member and its value object; each household, person and ledger row projects its class and the persons container's class. Two regressions exercise both vectors and the full path's refusal. Co-Authored-By: Claude Fable 5 --- .../us_runtime/asec_population_catalogue.py | 18 +++++-- .../test_us_asec_catalogue_records_memo.py | 54 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py index c7db08401..03c83d4d7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -215,15 +215,26 @@ def _records_identity(households, ledger, *, memo=False): ) +def _key_leaves(key): + # The encoder writes ``source.value``; an enum member is a mutable object, + # so its value object is a leaf in its own right. Classes are leaves too: + # ``__class__`` can be reassigned in place, and the encoder refuses + # anything but the exact record types on every full pass. + source, *rest = _KEY_LEAVES(key) + return (type(key), source, source.value, *rest) + + def _household_leaves(row): key, *literals, persons = _HOUSEHOLD_LEAVES(row) return ( + type(row), key, - *_KEY_LEAVES(key), + *_key_leaves(key), *literals, + type(persons), persons, *chain.from_iterable( - (*_PERSON_LEAVES(person), *_KEY_LEAVES(person.household_key)) + (type(person), *_PERSON_LEAVES(person), *_key_leaves(person.household_key)) for person in persons ), ) @@ -231,13 +242,14 @@ def _household_leaves(row): def _ledger_leaves(row): key, *literals = _LEDGER_LEAVES(row) - return (key, *_KEY_LEAVES(key), *literals) + return (type(row), key, *_key_leaves(key), *literals) def _eligible_key(key): return ( type(key) is domains.HouseholdKey and key.source is domains.Source.ASEC + and type(key.source.value) is str and type(key.source_year) is int and type(key.survey_year) is int and type(key.native_id) is str diff --git a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py index 5fdb14e8f..d22eed814 100644 --- a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py +++ b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py @@ -238,6 +238,60 @@ def test_wrong_source_member_or_foreign_types_yield_no_memo(): assert owner._eligible_leaves(ledger[0], "household") is None +def test_source_member_value_tampering_misses_and_the_full_path_reflects_it(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + member = domains.Source.ASEC + original = member._value_ + object.__setattr__(member, "_value_", "tampered") + try: + assert member.value == "tampered" + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + finally: + object.__setattr__(member, "_value_", original) + assert member.value == original + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.mark.parametrize("target", ["household", "person", "key", "ledger"]) +def test_class_reassignment_misses_and_the_full_path_refuses(target): + class SwappedHousehold(domains.AsecHousehold): + __slots__ = () + + class SwappedPerson(domains.AsecPerson): + __slots__ = () + + class SwappedKey(domains.HouseholdKey): + __slots__ = () + + class SwappedLedger(owner.UnrepresentedAsecHousehold): + __slots__ = () + + households, ledger = _records() + identity, memo = _issued(households, ledger) + victim, swapped = { + "household": (households[0], SwappedHousehold), + "person": (households[0].persons[0], SwappedPerson), + "key": (households[0].key, SwappedKey), + "ledger": (ledger[0], SwappedLedger), + }[target] + original = type(victim) + object.__setattr__(victim, "__class__", swapped) + try: + assert type(victim) is swapped + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + finally: + object.__setattr__(victim, "__class__", original) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + @pytest.fixture(scope="module") def actual_preparation(tmp_path_factory): with pytest.MonkeyPatch.context() as monkeypatch: From 77c9e7840f8b7f3ab234e512f2c8a66314e950f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 15:05:31 -0400 Subject: [PATCH 086/255] Re-pin the merged UK bundle and the pool tool's country-spec digest; mark the stamp as in-process Two attested identities moved with the merge ref, both re-pinned to the values the merged tree computes: the UK country bundle (main's #903 added the frs_relationships spine stage on top of this branch's attested kernel changes, so neither side's pin survives the merge), and the multispine pool tool's constants-adapter country_spec digest, which had already moved to the branch's attested value recorded in docs/evidence/spec-engine/ us-f0-coverage.json when the seed identities were re-pinned but was still asserted at main's value in test_us_multispine_pool_tool.py. survey_atomic_geography._population_stamp now says it is an in-process seal that must never be persisted or compared across reconstructions (microcosm#907 / #909), naming what a cross-call pin has to carry instead. Co-Authored-By: Claude Fable 5 --- .../build/us_runtime/survey_atomic_geography.py | 9 ++++++++- .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py index d6fe50b0d..63e6efcea 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -228,7 +228,14 @@ def _copy_population(population): def _population_stamp(population): - """Pure in-process seal, including storage beneath nullable masks.""" + """Pure in-process seal, including storage beneath nullable masks. + + Never persist it or compare it across reconstructions: it folds physical + storage parts (masked ``_data`` under nulls, and before microcosm#907 + object-dtype pointer bytes), so equal content rebuilt elsewhere need not + match. A cross-call pin carries ``source._frame_identity`` plus the + version, owners, weight kinds, mass ledger and design weights instead. + """ _require(type(population) is Population, "POPULATION_TYPE") digest = hashlib.sha256() frame = population.frame diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index d23984962..4c6df020a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -55,7 +55,7 @@ ), ( "uk", - "c396ee51ab029ed780ec25b69d34b128f53b08c673dc87a7f600f74264e755e3", + "9f9ee63ddfa95b91c025779fc60b6438500eb0f2c9872b10e5b68d5270dc1ec3", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index fc076a070..0dee0adbc 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2514,7 +2514,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6", + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed", }, } From 2c5c64f4826a96e6400500d438ef0bf58e89f015 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 18:27:57 -0400 Subject: [PATCH 087/255] Mark the engine parameter-file pin control as requiring the US engine The control reads the installed policyengine-us distribution's RECORD; the no-engine fast and wheels lanes have no such distribution and were failing with PackageNotFoundError instead of skipping through the requires_us marker. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_spec_seed_identity_engine_parameter_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py index 28dce08eb..2771d8988 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py @@ -48,6 +48,7 @@ def _boundary(diagnostic, monkeypatch, allowed): return hooks[0], distinct +@pytest.mark.requires_us def test_pins_match_the_installed_distribution_record(diagnostic): """Every pin is a regular file in the distribution whose RECORD digest agrees.""" From 069d5ed9a717909fc505a4c0c8e6404c231ea591 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 19:45:40 -0400 Subject: [PATCH 088/255] Restore the engine resolver by monkeypatch instead of reloading puf_support The no-engine fallback control reloaded puf_support in its finally block. A reload rebinds every function in the module to a new object, and the survey-origin budget seals puf_support's live functions in its producer check, so every freeze after that test in the same process refused with PRODUCER_CHANGED: 42 refusals across the fast and wheels lanes, found by a per-test seal probe. Restoring the original attribute keeps the module's objects identical and exercises the same fallback. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_us_puf_support.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_puf_support.py b/packages/microcosm-build/tests/test_us_puf_support.py index 75e510b6e..41e1925e9 100644 --- a/packages/microcosm-build/tests/test_us_puf_support.py +++ b/packages/microcosm-build/tests/test_us_puf_support.py @@ -1,6 +1,5 @@ """US PUF support-channel expansion tests.""" -import importlib from collections.abc import Sequence import numpy as np @@ -960,17 +959,20 @@ def test_assert_formula_owned_blocklist_current_flags_stale_entries() -> None: assert_formula_owned_blocklist_current(drifted_engine) -def test_resolve_formula_owned_outputs_engine_none_falls_back_to_static() -> None: +def test_resolve_formula_owned_outputs_engine_none_falls_back_to_static( + monkeypatch, +) -> None: # With no engine passed and metadata unavailable, resolution falls back to # the static seed. The fallback is exercised by monkeypatching the lazy # engine resolver to report no engine, so the test is deterministic even - # where policyengine_us is installed. - puf_support_module._formula_owned_engine = lambda: None - try: - requested = {"interest_deduction", "employment_income_before_lsr"} - assert resolve_formula_owned_outputs(requested) == {"interest_deduction"} - finally: - importlib.reload(puf_support_module) + # where policyengine_us is installed. Restoring the original attribute + # (never reloading the module) keeps every function object in + # puf_support identical afterwards: the survey-origin budget seals this + # module's live functions, and a reload would refuse every later freeze + # in the same process with PRODUCER_CHANGED. + monkeypatch.setattr(puf_support_module, "_formula_owned_engine", lambda: None) + requested = {"interest_deduction", "employment_income_before_lsr"} + assert resolve_formula_owned_outputs(requested) == {"interest_deduction"} class _ImportErrorEngine: @@ -1291,16 +1293,12 @@ def test_policyengine_broadcasts_annual_reported_enrollment_to_each_month( "spm_units": { "unit_100": { "members": ["person_1", "person_2"], - "receives_tanf": { - "2024": bool(spm_flags.loc[100, "receives_tanf"]) - }, + "receives_tanf": {"2024": bool(spm_flags.loc[100, "receives_tanf"])}, "receives_snap": {"2024": bool(spm_flags.loc[100, "receives_snap"])}, }, "unit_200": { "members": ["person_3"], - "receives_tanf": { - "2024": bool(spm_flags.loc[200, "receives_tanf"]) - }, + "receives_tanf": {"2024": bool(spm_flags.loc[200, "receives_tanf"])}, "receives_snap": {"2024": bool(spm_flags.loc[200, "receives_snap"])}, }, }, From 3ad1b1aaeb2fe84522210e0c284a12476640d924 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:02:27 -0400 Subject: [PATCH 089/255] Open the #893 reconciliation lane journal Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/PROGRESS.md b/PROGRESS.md index 78fd642ad..9d785a2e0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,3 +1,36 @@ +# #893 reconciliation to main's amended graph interface (amendments 19 and 20) + +Lane: `microcosm-us-launch-verified-lanes-20260910` (the live integration +worktree for PR #893, branch `microcosm-us-launch-integration-20260909`). +Started 2026-09-12 at `069d5ed9a`. Everything below the `---` rule at the end +of this section is prior-lane history; see "Root journals are history, not +state" in `CLAUDE.md`. + +## State + +In progress. Baseline before any change: `packages/microcosm-graph/tests` +5 failed / 362 passed (exit 1) — exactly the five the brief names. + +## Done + +- Read `CLAUDE.md`, the charter's "Interface freeze" at the amendment-20 head + (`23ba24770`), PR #893's body, and the Amendment 19 lane's "Scope" list on + main. Diffed the graph package against `23ba24770` file by file and located + every consumer of each not-yet-amended piece outside the graph package. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. + +## Next + +- `git merge origin/main`, then `git merge origin/amend-keyed-seed-and-uniform-draws` + (`23ba24770`), resolving every graph/fit/charter file to main's bytes. +- Re-apply, as separate named commits, only the not-yet-amended pieces the US + runtime on this branch consumes; drop the rest and say so in `out.md`. +- Run the graph and fit packages, the spine-blindness file, the spec-engine + check, `ci_test_groups --verify`, ruff, then the touched US suites. +- Write the layer map and the draft amendment entries to `out.md`. + +--- + # F1 portable worker identity — CI crawl fix ## State From dc621c14c59e699a53553b9630cb6e75af6e7d0a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:10:49 -0400 Subject: [PATCH 090/255] Re-pin the seed protocol and compiled seed map digests on the merged tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both digests hash the exact source bytes of the attested kernel modules (spec_engine/seeds.py). The merged tree carries the branch's acs_transfer and housing_inputs changes and, through amendment 20, the same fit.qrf bytes the branch already carried, so the actual digests are the values the branch pinned in 1734b9e90 — not the amendment-20 head's, which were computed on a tree without the two US kernel changes. The F0 coverage report is regenerated from the re-pinned inventory; tools/spec_engine_coverage.py --check exits 0. Co-Authored-By: Claude Fable 5.1 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 5120a883a..5c8e67132 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "373786d5ba5e115c3317c42662b3998cdef8061441cf0ba48c36828bd7d55e69", - "protocol_sha256": "d8f595a05dd71d7eccce5976d640baf97541e3ca159fd6179e7b1ab1a1286d8e" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "failures": [], "observed": { - "map_sha256": "373786d5ba5e115c3317c42662b3998cdef8061441cf0ba48c36828bd7d55e69", - "protocol_sha256": "d8f595a05dd71d7eccce5976d640baf97541e3ca159fd6179e7b1ab1a1286d8e" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "d8f595a05dd71d7eccce5976d640baf97541e3ca159fd6179e7b1ab1a1286d8e", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "d8f595a05dd71d7eccce5976d640baf97541e3ca159fd6179e7b1ab1a1286d8e", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c72fb9e1bcb13e0288670b94261162de464e46d75c7c70c1742a34bf7a95bf87" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c72fb9e1bcb13e0288670b94261162de464e46d75c7c70c1742a34bf7a95bf87" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 42eee01d5..309b2f4de 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "373786d5ba5e115c3317c42662b3998cdef8061441cf0ba48c36828bd7d55e69", - "seed_protocol": "d8f595a05dd71d7eccce5976d640baf97541e3ca159fd6179e7b1ab1a1286d8e", + "seed_map": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "seed_protocol": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", From cff8fbf324f2bf4f664e5646d1ad0fe3d38d9d22 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:13:02 -0400 Subject: [PATCH 091/255] Re-pin the calibrate and simulate H1 parity cases the branch's kernels move On the merged tree six graph tests were red (test_acceptance_h_parity, test_graph_parity_pins x4, test_graph_serialize), all on the calibrate and simulate cases' stale pins. Both moves are the branch's own, not the merge's: CalibrateKernel.implementation_hash binds microcosm.calibrate's solve and diagnostics modules, which the branch's grouped-bound solver path changes (layer 05), and SimulateRulesKernel.implementation_hash binds microcosm.frame.bundle, to which the branch adds Frame.__reduce__ (layer 01). Re-recorded with tools/graph_parity_repin.py calibrate / simulate, which keeps every pinned platform (both cases pin only arm64/darwin/py3.14) and leaves direct.csv untouched. The four parity test files: 27 passed, exit 0. Co-Authored-By: Claude Fable 5.1 --- .../tests/fixtures/parity/kernels/calibrate/pins.json | 2 +- .../tests/fixtures/parity/kernels/simulate/pins.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index a9610f6b8..1b196a434 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"5f5f74d9bce03175a37006d5c3e5e4dcefdec1118982351fd2ca66e3793dc48b","kernel":"calibrate.adam@1","node":"calibrate","node_key":"184ccd0a25e1d00cc4393b2880ff13a601ac04207bc37b0160b7ae60596cb3d7","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"184ccd0a25e1d00cc4393b2880ff13a601ac04207bc37b0160b7ae60596cb3d7"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74","kernel":"calibrate.adam@1","node":"calibrate","node_key":"faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1"}},"seed":0} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json index 15f5d879b..57ec48a0f 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json @@ -1 +1 @@ -{"dependencies":{},"implementation_hash":"eed54eaf27b53faf446068aa833ec16ab398840faf870445dbdcd6a4604c566c","kernel":"simulate.rules@1","node":"simulate","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123"}},"seed":null} +{"dependencies":{},"implementation_hash":"9a6937b103dfdc5b1e8d0f69d4a27792400b1ee3c0bbea85f63e4d8c02458384","kernel":"simulate.rules@1","node":"simulate","node_key":"85c622d86b4fc358e94472187144c627c5ff3ff6a44614c48a38c7b92e112cd9","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"85c622d86b4fc358e94472187144c627c5ff3ff6a44614c48a38c7b92e112cd9"}},"seed":null} From 752ab840f1c52676c024b518ca35ed039b22db2b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:14:08 -0400 Subject: [PATCH 092/255] Re-apply the raw-byte source codec on main's graph package The not-yet-amended codecs piece from the integration branch, re-applied on main's codecs.py byte-for-byte: SourceCodecRegistry gains an explicit raw-byte mode (register_bytes / load_bytes / bytes_names / as_bytes_mapping) that a name may hold in place of Frame mode but never both, the shipped raw-bytes-v1 codec reads one bounded regular file and interprets nothing, and load_source_bytes / SourceBytesCodec / RAW_BYTES_MAX_BYTES are exported. The executor's availability check (SOURCE_CODECS.get) already resolves both modes, so no executor change rides with it. Consumers on this branch outside the graph package: graph_atomic_geography, graph_geography, graph_sources, graph_atomic_survey_population, graph_puf55_canonical_donor, puf_raw_source, puf_monetary_source, puf_monetary_agi_projection, puf_full_source_graph, atomic_block_sources, atomic_block_api_sources, survey_atomic_geography, puf55_survey_recipients. The branch carried no graph-level tests for it; four are added here (the shipped codec's contract, the regular-file refusals incl. a FIFO opened non-blocking, mode exclusivity and name reservation, and a graph whose CREATE source is raw bytes). test_graph_codecs.py: 8 passed, exit 0. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/graph/__init__.py | 4 + .../src/microcosm/graph/codecs.py | 254 ++++++++++++++++-- .../tests/test_graph_codecs.py | 196 +++++++++++++- 3 files changed, 435 insertions(+), 19 deletions(-) diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index ef69b3fcc..fdc4d6be5 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -104,6 +104,7 @@ "NumericScope", "Tolerance", "Slice", + "SourceBytesCodec", "SourceCodec", "SourceCodecRegistry", "SourceRef", @@ -123,6 +124,7 @@ "graph_to_json", "keyed_uniform", "load_source", + "load_source_bytes", "run_graph", "source_hash", ] @@ -146,9 +148,11 @@ def _check_frame_version() -> None: from .codecs import ( # noqa: E402 - check dependency series before runtime import SOURCE_CODECS, + SourceBytesCodec, SourceCodec, SourceCodecRegistry, load_source, + load_source_bytes, ) from .executor import NodeRejected, run_graph # noqa: E402 from .explain import explain_html # noqa: E402 diff --git a/packages/microcosm-graph/src/microcosm/graph/codecs.py b/packages/microcosm-graph/src/microcosm/graph/codecs.py index 4cf4bd3e4..3ebc9d8e9 100644 --- a/packages/microcosm-graph/src/microcosm/graph/codecs.py +++ b/packages/microcosm-graph/src/microcosm/graph/codecs.py @@ -1,12 +1,32 @@ -"""Source codecs: the sole boundary from source bytes to :class:`Frame`. - -Two codecs ship with the graph runtime: - -``frame-store`` +"""Source codecs: the boundary from source bytes to a decoded input. + +A codec is registered in exactly one of two explicit modes, and a name +belongs to at most one mode: + +*Frame mode* (:data:`SourceCodec`, :meth:`SourceCodecRegistry.register`, +:meth:`SourceCodecRegistry.load`) decodes a source into a population +:class:`Frame`. A ``CREATE`` node's kernel calls it to build a population, +and so does any other node whose source really is one — a held-out reference +frame read for scoring, say. + +*Raw-byte mode* (:data:`SourceBytesCodec`, +:meth:`SourceCodecRegistry.register_bytes`, +:meth:`SourceCodecRegistry.load_bytes`) decodes a source into immutable +``bytes``. A lookup table (an NPZ of ratios, a CSV crosswalk) is not a +population, and an import kernel that turns one into a typed +:class:`~microcosm.graph.decl.ArtifactOutput` must be able to read its real +bytes without a Frame codec registered as a pretence. Neither mode can be +loaded through the other: the mismatch is a :class:`TypeError` naming the +mode the codec actually has, so no caller receives bytes where it declared a +Frame. + +Three codecs ship with the graph runtime: + +``frame-store`` (Frame) Loads a content-verified Frame from the path of a ``ContentStore`` frame object directory. -``csv-tables`` +``csv-tables`` (Frame) Loads one CSV per entity using ``schema.json``. The schema may give a ``tables`` mapping (otherwise ``.csv`` is used), global or per-entity dtype mappings, a ``strata_column``, and either (a) a weight @@ -14,11 +34,19 @@ A JSON weight entry is ``{"kind": "design", "values": [...]}`` or ``{"kind": "design", "column": "household_weight"}``; entries are keyed by entity, or may carry their own ``entity`` field. + +``raw-bytes-v1`` (raw bytes) + Reads one regular file, bounded at :data:`RAW_BYTES_MAX_BYTES`. It + interprets nothing: the consuming kernel owns the payload's format and + validates it. Identity, the pre-run content key, and the post-run + mutation check stay where they already are, in the executor. """ from __future__ import annotations import json +import os +import stat from collections.abc import Callable, Mapping from pathlib import Path from types import MappingProxyType @@ -32,39 +60,97 @@ from .store import ContentStore, StoreUnavailable __all__ = [ + "RAW_BYTES_MAX_BYTES", "SOURCE_CODECS", + "SourceBytesCodec", "SourceCodec", "SourceCodecRegistry", "load_csv_tables", "load_frame_store", + "load_raw_bytes", "load_source", + "load_source_bytes", ] type SourceCodec = Callable[..., Frame] +type SourceBytesCodec = Callable[..., bytes] + +RAW_BYTES_MAX_BYTES = 64 * 1024 * 1024 +"""The most ``raw-bytes-v1`` will read from one source file (64 MiB).""" class SourceCodecRegistry: - """Named source-to-Frame loaders.""" + """Named source loaders in two explicit modes: Frame and raw bytes. + + A name is registered in one mode only. :meth:`get` answers the + availability question the executor asks before any kernel runs, in + either mode; :meth:`load` and :meth:`load_bytes` are what hold a codec + to the mode it was registered in. + """ def __init__(self) -> None: self._loaders: dict[str, SourceCodec] = {} + self._byte_loaders: dict[str, SourceBytesCodec] = {} - def register(self, name: str, loader: SourceCodec) -> SourceCodec: - """Register and return ``loader`` under a non-empty codec name.""" - + @staticmethod + def _check_declaration(name: str, loader: object) -> None: if not isinstance(name, str) or not name: raise ValueError("Source codec names must be non-empty strings.") if not callable(loader): raise TypeError("Source codec loaders must be callable.") + + def register(self, name: str, loader: SourceCodec) -> SourceCodec: + """Register and return a Frame ``loader`` under a non-empty codec name.""" + + self._check_declaration(name, loader) + if name in self._byte_loaders: + raise ValueError( + f"Source codec {name!r} is already registered as a raw-bytes codec." + ) incumbent = self._loaders.get(name) if incumbent is not None and incumbent is not loader: raise ValueError(f"Source codec {name!r} is already registered.") self._loaders[name] = loader return loader - def get(self, name: str) -> SourceCodec: - """Resolve ``name`` or raise fatal :class:`StoreUnavailable`.""" + def register_bytes(self, name: str, loader: SourceBytesCodec) -> SourceBytesCodec: + """Register and return a raw-byte ``loader`` under a non-empty codec name. + + A raw-byte codec returns the source's bytes and claims nothing about + their meaning; it never stands in for a population. + """ + + self._check_declaration(name, loader) + if name in self._loaders: + raise ValueError( + f"Source codec {name!r} is already registered as a Frame codec." + ) + incumbent = self._byte_loaders.get(name) + if incumbent is not None and incumbent is not loader: + raise ValueError(f"Source codec {name!r} is already registered.") + self._byte_loaders[name] = loader + return loader + + def get(self, name: str) -> SourceCodec | SourceBytesCodec: + """Resolve ``name`` in either mode, or raise fatal :class:`StoreUnavailable`. + + This is availability, not decoding: a registered raw-byte codec is + installed, and resolving it here never turns it into a Frame codec. + """ + + for loaders in (self._loaders, self._byte_loaders): + try: + return loaders[name] + except KeyError: + continue + raise StoreUnavailable(f"Source codec {name!r} is not installed.") + def _frame_loader(self, name: str) -> SourceCodec: + if name in self._byte_loaders: + raise TypeError( + f"Source codec {name!r} is a raw-bytes codec; read it with " + "load_bytes. Bytes are never presented as a Frame." + ) try: return self._loaders[name] except KeyError as error: @@ -72,6 +158,19 @@ def get(self, name: str) -> SourceCodec: f"Source codec {name!r} is not installed." ) from error + def _bytes_loader(self, name: str) -> SourceBytesCodec: + if name in self._loaders: + raise TypeError( + f"Source codec {name!r} is a Frame codec; read it with load. " + "A Frame is never presented as raw bytes." + ) + try: + return self._byte_loaders[name] + except KeyError as error: + raise StoreUnavailable( + f"Source codec {name!r} is not installed." + ) from error + def load( self, name: str, @@ -79,9 +178,9 @@ def load( *, store: ContentStore | None = None, ) -> Frame: - """Decode ``path`` with ``name`` and require a Frame result.""" + """Decode ``path`` with the Frame codec ``name`` and require a Frame.""" - loader = self.get(name) + loader = self._frame_loader(name) try: frame = loader(Path(path), store=store) except StoreUnavailable: @@ -96,16 +195,66 @@ def load( ) return frame + def load_bytes( + self, + name: str, + path: Path, + *, + store: ContentStore | None = None, + ) -> bytes: + """Read ``path`` with the raw-byte codec ``name`` and require bytes. + + This decodes an external source path, unlike + :meth:`ContentStore.load_bytes`, which reads a stored object back by + its content key. + + Unavailability keeps the classification :meth:`load` gives it: an + uninstalled codec or a missing dependency is fatal + :class:`StoreUnavailable`, never a decoded value. + """ + + loader = self._bytes_loader(name) + try: + payload = loader(Path(path), store=store) + except StoreUnavailable: + raise + except ImportError as error: + raise StoreUnavailable( + f"Source codec {name!r} needs an unavailable dependency." + ) from error + if not isinstance(payload, bytes): + raise TypeError( + f"Source codec {name!r} returned {type(payload).__name__}, not bytes." + ) + return payload + def names(self) -> tuple[str, ...]: - """Registered names in canonical order.""" + """The registered Frame codec names, in canonical order. + + Each mode is enumerated by its own pair — ``names``/``as_mapping`` + here, :meth:`bytes_names`/:meth:`as_bytes_mapping` there — so a + snapshot taken through either pair stays resolvable through it. The + name space is still shared: a name in neither tuple is not therefore + free, because registering it in one mode reserves it in both. + """ return tuple(sorted(self._loaders)) + def bytes_names(self) -> tuple[str, ...]: + """The registered raw-byte codec names, in canonical order.""" + + return tuple(sorted(self._byte_loaders)) + def as_mapping(self) -> Mapping[str, SourceCodec]: - """A read-only snapshot of registered loaders.""" + """A read-only snapshot of the registered Frame loaders.""" return MappingProxyType(dict(self._loaders)) + def as_bytes_mapping(self) -> Mapping[str, SourceBytesCodec]: + """A read-only snapshot of the registered raw-byte loaders.""" + + return MappingProxyType(dict(self._byte_loaders)) + def load_frame_store(path: Path, *, store: ContentStore | None = None) -> Frame: """Load a verified frame object from its content-store directory.""" @@ -372,9 +521,68 @@ def load_csv_tables(path: Path, *, store: ContentStore | None = None) -> Frame: return Frame(tables, schema, weights, strata) +def _nonblocking_opener(path: str, flags: int) -> int: + """Open without blocking, so a FIFO in a source's place cannot hang a run.""" + + return os.open(path, flags | getattr(os, "O_NONBLOCK", 0)) + + +def load_raw_bytes(path: Path, *, store: ContentStore | None = None) -> bytes: + """Read one regular file's bytes, bounded at :data:`RAW_BYTES_MAX_BYTES`. + + The codec claims nothing about the payload: a lookup NPZ, a crosswalk + CSV, and a corrupt file are all just bytes here, and the kernel that + imports them validates their format. What this function does own is the + refusal to read something that is not one bounded regular file. + + Symlinks are followed, as the executor's ``resolve(strict=True)`` and + ``source_content_key`` already do. A directory, a FIFO, a device, or a + socket is refused: the mode is read from the descriptor this function + itself opened, and the open is non-blocking, so the codec cannot be made + to wait on a pipe or stream a device. The read is bounded rather than + trusted to ``st_size``, because a file may grow after it is measured. + Detecting that a source moved is the executor's post-run content check; + this bound only keeps the codec from reading an unbounded amount first. + """ + + del store # raw bytes are self-describing; no content store is consulted + source = Path(path) + try: + handle = open(source, "rb", opener=_nonblocking_opener) + except IsADirectoryError as error: + raise ValueError( + f"Raw source at {source} is a directory; raw-bytes-v1 reads one " + "regular file." + ) from error + except OSError as error: + raise ValueError(f"Raw source at {source} is not readable: {error}") from error + with handle: + # A directory never reaches here: opening one raises IsADirectoryError + # above. Everything else that is not a regular file is refused from the + # descriptor's own mode, not from a second look at the path. + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + raise ValueError( + f"Raw source at {source} is not a regular file; raw-bytes-v1 " + "reads one regular file." + ) + try: + payload = handle.read(RAW_BYTES_MAX_BYTES + 1) + except OSError as error: + raise ValueError( + f"Raw source at {source} is not readable: {error}" + ) from error + if len(payload) > RAW_BYTES_MAX_BYTES: + raise ValueError( + f"Raw source at {source} is larger than the " + f"{RAW_BYTES_MAX_BYTES}-byte raw-bytes-v1 limit." + ) + return payload + + SOURCE_CODECS = SourceCodecRegistry() SOURCE_CODECS.register("frame-store", load_frame_store) SOURCE_CODECS.register("csv-tables", load_csv_tables) +SOURCE_CODECS.register_bytes("raw-bytes-v1", load_raw_bytes) def load_source( @@ -384,6 +592,18 @@ def load_source( store: ContentStore | None = None, registry: SourceCodecRegistry = SOURCE_CODECS, ) -> Frame: - """Decode one source through the selected registry.""" + """Decode one source into a Frame through the selected registry.""" return registry.load(codec, path, store=store) + + +def load_source_bytes( + codec: str, + path: Path, + *, + store: ContentStore | None = None, + registry: SourceCodecRegistry = SOURCE_CODECS, +) -> bytes: + """Read one source's bytes through the selected registry.""" + + return registry.load_bytes(codec, path, store=store) diff --git a/packages/microcosm-graph/tests/test_graph_codecs.py b/packages/microcosm-graph/tests/test_graph_codecs.py index 64d7c522e..1053a2eb4 100644 --- a/packages/microcosm-graph/tests/test_graph_codecs.py +++ b/packages/microcosm-graph/tests/test_graph_codecs.py @@ -3,15 +3,39 @@ from __future__ import annotations import json +import os from pathlib import Path import numpy as np import pandas as pd import pytest +import microcosm.graph.codecs as codecs_module from microcosm.frame import EntitySchema, Frame, WeightKind, Weights -from microcosm.graph.codecs import SOURCE_CODECS, SourceCodecRegistry, load_source -from microcosm.graph.store import ContentStore, StoreUnavailable +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import ( + SOURCE_CODECS, + SourceCodecRegistry, + load_raw_bytes, + load_source, + load_source_bytes, +) +from microcosm.graph.keys import source_content_key +from microcosm.graph.store import StoreUnavailable def _frame() -> Frame: @@ -156,3 +180,171 @@ def missing_dependency(_path: Path, *, store: ContentStore | None = None) -> Fra registry.register("engine", missing_dependency) with pytest.raises(StoreUnavailable, match="dependency"): registry.load("engine", tmp_path) + + +# --- raw-byte codecs --------------------------------------------------------- + + +def test_raw_bytes_codec_reads_one_regular_file_verbatim(tmp_path: Path) -> None: + """``raw-bytes-v1`` ships registered in raw-byte mode and interprets nothing.""" + + payload = b"\x00lookup\xff" * 3 + source = tmp_path / "table.npz" + source.write_bytes(payload) + assert SOURCE_CODECS.bytes_names() == ("raw-bytes-v1",) + assert "raw-bytes-v1" not in SOURCE_CODECS.names() + assert SOURCE_CODECS.get("raw-bytes-v1") is load_raw_bytes + assert load_source_bytes("raw-bytes-v1", source) == payload + assert SOURCE_CODECS.load_bytes("raw-bytes-v1", source) == payload + empty = tmp_path / "empty.bin" + empty.write_bytes(b"") + assert load_source_bytes("raw-bytes-v1", empty) == b"" + # A symlink is followed, as the executor's source resolution already does. + link = tmp_path / "link.bin" + link.symlink_to(source) + assert load_source_bytes("raw-bytes-v1", link) == payload + + +def test_raw_bytes_codec_refuses_anything_but_one_bounded_regular_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(ValueError, match="is a directory"): + load_raw_bytes(tmp_path) + with pytest.raises(ValueError, match="not readable"): + load_raw_bytes(tmp_path / "absent.bin") + fifo = tmp_path / "pipe" + os.mkfifo(fifo) + # Opened non-blocking, so a FIFO with no writer is refused from its mode + # instead of hanging the run. + with pytest.raises(ValueError, match="not a regular file"): + load_raw_bytes(fifo) + monkeypatch.setattr(codecs_module, "RAW_BYTES_MAX_BYTES", 8) + big = tmp_path / "big.bin" + big.write_bytes(b"x" * 9) + with pytest.raises(ValueError, match="larger than the 8-byte"): + load_raw_bytes(big) + bounded = tmp_path / "bounded.bin" + bounded.write_bytes(b"x" * 8) + assert load_raw_bytes(bounded) == b"x" * 8 + + +def test_registry_modes_are_exclusive_and_names_reserved_across_modes( + tmp_path: Path, +) -> None: + registry = SourceCodecRegistry() + registry.register("frame", lambda path, *, store=None: _frame()) + registry.register_bytes("bytes", lambda path, *, store=None: b"raw") + assert registry.names() == ("frame",) + assert registry.bytes_names() == ("bytes",) + assert set(registry.as_mapping()) == {"frame"} + assert set(registry.as_bytes_mapping()) == {"bytes"} + # Availability resolves in either mode; decoding holds each to its own. + registry.get("frame") + registry.get("bytes") + source = tmp_path / "source" + source.write_bytes(b"") + assert registry.load_bytes("bytes", source) == b"raw" + assert isinstance(registry.load("frame", source), Frame) + with pytest.raises(TypeError, match="is a raw-bytes codec"): + registry.load("bytes", source) + with pytest.raises(TypeError, match="is a Frame codec"): + registry.load_bytes("frame", source) + with pytest.raises(ValueError, match="already registered as a raw-bytes codec"): + registry.register("bytes", lambda path, *, store=None: _frame()) + with pytest.raises(ValueError, match="already registered as a Frame codec"): + registry.register_bytes("frame", lambda path, *, store=None: b"") + with pytest.raises(ValueError, match="already registered"): + registry.register_bytes("bytes", lambda path, *, store=None: b"other") + with pytest.raises(ValueError, match="non-empty strings"): + registry.register_bytes("", lambda path, *, store=None: b"") + with pytest.raises(TypeError, match="must be callable"): + registry.register_bytes("thing", object()) # type: ignore[arg-type] + with pytest.raises(StoreUnavailable, match="not installed"): + registry.load_bytes("absent", source) + with pytest.raises(StoreUnavailable, match="not installed"): + registry.get("absent") + + registry.register_bytes("text", lambda path, *, store=None: "text") # type: ignore[arg-type,return-value] + with pytest.raises(TypeError, match="returned str, not bytes"): + registry.load_bytes("text", source) + + def needs_dependency(path: Path, *, store: ContentStore | None = None) -> bytes: + raise ImportError("no dependency") + + registry.register_bytes("needs", needs_dependency) + with pytest.raises(StoreUnavailable, match="unavailable dependency"): + registry.load_bytes("needs", source) + + +class _BytesCreate: + """A CREATE kernel whose only source is a raw-byte lookup table.""" + + ref = "bytes.create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self) -> str: + return "0" * 64 + + def run(self, context: KernelContext) -> KernelResult: + payload = load_source_bytes("raw-bytes-v1", context.sources["table"]) + frame = _frame() + household = frame.table("household").assign( + size=pd.Series([len(payload)] * 2, dtype="int64") + ) + return KernelResult( + frame=Frame( + {"person": frame.table("person"), "household": household}, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + ), + receipt={"payload_bytes": len(payload)}, + ) + + +def test_a_graph_source_may_be_raw_bytes_and_keys_on_its_content( + tmp_path: Path, +) -> None: + """The executor admits a raw-byte SourceRef and hands the kernel its path. + + Identity, the pre-run content key and the post-run mutation check stay in + the executor, so the codec adds no second identity scheme. + """ + + table = tmp_path / "table.bin" + table.write_bytes(b"0123456789") + graph = Graph( + "toy", + (SourceRef("table", "raw-bytes-v1"),), + ( + Node( + "survey", + _BytesCreate.ref, + sources=("table",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "flag", "boolean"), + Owned("household", "size", "int64"), + ), + ), + ), + ) + registry = KernelRegistry() + registry.register(_BytesCreate()) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compile_graph(graph), sources={"table": table}, store=store, kernels=registry + ) + assert manifest.nodes["survey"].receipt["payload_bytes"] == 10 + assert set(manifest.populations["survey"].household["size"]) == {10} + cold_key = manifest.nodes["survey"].key + cold_source = source_content_key("table", table) + table.write_bytes(b"0123456789!") + assert source_content_key("table", table) != cold_source + changed = run_graph( + compile_graph(graph), sources={"table": table}, store=store, kernels=registry + ) + assert changed.nodes["survey"].key != cold_key + assert changed.nodes["survey"].receipt["payload_bytes"] == 11 From db1b7821a3cedf6659b158dcf94411493c44f91d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:15:17 -0400 Subject: [PATCH 093/255] Re-apply complete Frame metadata storage and non-finite JSON refusal in the store The not-yet-amended store piece from the integration branch, re-applied on main's store.py: the frame object format moves to microcosm-graph-frame-v2, Frame.metadata is encoded losslessly (mapping/tuple/frozenset/float64 bytes/ scalar tags) into the frame manifest and restored on load, its SHA-256 is recorded in the object header and checked on read, ContentStore._put takes a validate_existing hook so a same-key frame whose metadata differs is StoreCorrupt and a v1 frame is StoreUnavailable rather than silently losing its metadata, and the JSON decode boundary refuses NaN/Infinity constants and overflowing numerals so a corrupt manifest is StoreCorrupt, not a TypeError from the canonical re-encoder. The one hunk of the branch's store diff that is NOT re-applied is the removal of main's #907 datetime64/timedelta64 object-leaf refusal: that removal was the branch being older than main, not a piece of this change. Consumers on this branch outside the graph package: survey_population_replay (same_replayed_frame compares encoded metadata), population_input_coverage and graph_full_puf_enrichment (record encoded metadata), and every US stage that expects Frame.metadata to survive a store round trip. Its test file test_frame_metadata_store.py comes back with it. test_frame_metadata_store, test_graph_store, test_graph_population, test_graph_executor, test_graph_manifest: 274 passed, exit 0. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/graph/store.py | 129 +++++++++++- .../tests/test_frame_metadata_store.py | 198 ++++++++++++++++++ 2 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 packages/microcosm-graph/tests/test_frame_metadata_store.py diff --git a/packages/microcosm-graph/src/microcosm/graph/store.py b/packages/microcosm-graph/src/microcosm/graph/store.py index 2d943d4a4..d30050b00 100644 --- a/packages/microcosm-graph/src/microcosm/graph/store.py +++ b/packages/microcosm-graph/src/microcosm/graph/store.py @@ -62,7 +62,7 @@ type ResumePolicy = Literal["auto", "require", "forbid"] _STORE_FORMAT = "microcosm-graph-content-store-v1" -_FRAME_FORMAT = "microcosm-graph-frame-v1" +_FRAME_FORMAT = "microcosm-graph-frame-v2" _KEY = re.compile(r"[0-9a-f]{64}\Z") _ENCODING_NUMPY = "numpy-v1" @@ -194,10 +194,32 @@ def _require_key(key: str) -> str: return key +def _reject_non_finite_constant(token: str) -> float: + """Refuse the ``NaN``/``Infinity`` literals ``json`` accepts by default.""" + raise ValueError(f"Stored JSON carries the non-finite constant {token}.") + + +def _finite_json_number(token: str) -> float: + """Refuse numeric literals that overflow to an infinity (e.g. ``1e999``).""" + value = float(token) + if not math.isfinite(value): + raise ValueError(f"Stored JSON carries the non-finite number {token}.") + return value + + def _load_json_file(path: Path, *, label: str) -> Any: + # ``_canonical_json`` writes every store JSON with ``allow_nan=False``, so + # no valid payload can carry a non-finite value and the decode boundary is + # the right place to refuse one. Without these hooks a corrupt frame + # manifest survives the load and only fails later inside the canonical + # re-encode, which escapes as TypeError instead of StoreCorrupt. try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + return json.loads( + path.read_text(encoding="utf-8"), + parse_constant=_reject_non_finite_constant, + parse_float=_finite_json_number, + ) + except (OSError, UnicodeDecodeError, ValueError) as error: raise StoreCorrupt(f"Stored {label} is not readable canonical JSON.") from error @@ -704,11 +726,14 @@ def _put( build: Callable[[Path], Mapping[str, object]], *, verify_existing: bool = True, + validate_existing: Callable[[Mapping[str, Any]], None] | None = None, ) -> Path: key = _require_key(key) destination = self.object_path(key) if verify_existing and destination.exists(): - _verified_meta(destination, expected_kind=kind) + existing = _verified_meta(destination, expected_kind=kind) + if validate_existing is not None: + validate_existing(existing) return destination staging = self.tmp / uuid.uuid4().hex staging.mkdir(parents=False, exist_ok=False) @@ -736,7 +761,9 @@ def _put( if verify_existing: if not destination.exists(): raise - _verified_meta(destination, expected_kind=kind) + existing = _verified_meta(destination, expected_kind=kind) + if validate_existing is not None: + validate_existing(existing) else: self._replace_write_only_collision(staging, destination) _fsync_directory(destination.parent) @@ -902,12 +929,35 @@ def put_frame( raise TypeError(f"frame must be a Frame, got {type(frame).__name__}.") frame.revalidate() bound_node_key = key if node_key is None else node_key + metadata_sha256 = hashlib.sha256( + _canonical_json(_encode_frame_metadata(frame.metadata)) + ).hexdigest() + + def validate_existing(metadata: Mapping[str, Any]) -> None: + if metadata.get("frame_format") != _FRAME_FORMAT: + raise StoreUnavailable( + "Stored frame predates complete metadata storage." + ) + if metadata.get("frame_metadata_sha256") != metadata_sha256: + raise StoreCorrupt( + "The same frame key cannot carry different metadata." + ) def build(root: Path) -> Mapping[str, object]: _write_frame(root, frame) - return {"frame_format": _FRAME_FORMAT, "node_key": bound_node_key} + return { + "frame_format": _FRAME_FORMAT, + "node_key": bound_node_key, + "frame_metadata_sha256": metadata_sha256, + } - return self._put(key, "frame", build, verify_existing=verify_existing) + return self._put( + key, + "frame", + build, + verify_existing=verify_existing, + validate_existing=validate_existing, + ) write_frame = put_frame @@ -1006,6 +1056,61 @@ def _schema_payload(schema: EntitySchema) -> dict[str, object]: } +def _encode_frame_metadata(value: object) -> object: + """Preserve Frame metadata kinds without pickle or lossy scalar coercion.""" + if isinstance(value, Mapping): + return [ + "mapping", + [[key, _encode_frame_metadata(item)] for key, item in value.items()], + ] + if isinstance(value, tuple): + return ["tuple", [_encode_frame_metadata(item) for item in value]] + if isinstance(value, frozenset): + return [ + "frozenset", + sorted( + (_encode_frame_metadata(item) for item in value), key=_canonical_json + ), + ] + if isinstance(value, float): + # Binary float bytes preserve signed zero and any allowed NaN payload. + return ["float64", struct.pack(">d", value).hex()] + if value is None or isinstance(value, (str, int, bool)): + return ["scalar", value] + raise TypeError(f"Unsupported Frame metadata value {type(value).__name__}.") + + +def _decode_frame_metadata(encoded: object) -> object: + try: + if not isinstance(encoded, list) or len(encoded) != 2: + raise ValueError + kind, value = encoded + if kind == "mapping" and isinstance(value, list): + result = {} + for pair in value: + if not isinstance(pair, list) or len(pair) != 2: + raise ValueError + key, item = pair + if not isinstance(key, str) or not key or key in result: + raise ValueError + result[key] = _decode_frame_metadata(item) + return result + if kind in ("tuple", "frozenset") and isinstance(value, list): + items = [_decode_frame_metadata(item) for item in value] + return tuple(items) if kind == "tuple" else frozenset(items) + if ( + kind == "float64" + and isinstance(value, str) + and re.fullmatch(r"[0-9a-f]{16}", value) + ): + return struct.unpack(">d", bytes.fromhex(value))[0] + if kind == "scalar" and (value is None or type(value) in (str, int, bool)): + return value + raise ValueError + except (ValueError, TypeError, KeyError, struct.error) as error: + raise StoreCorrupt("Stored Frame metadata is malformed.") from error + + def _write_frame(root: Path, frame: Frame) -> None: _write_json(root / "schema.json", _schema_payload(frame.schema)) table_specs: list[dict[str, object]] = [] @@ -1060,6 +1165,7 @@ def _write_frame(root: Path, frame: Frame) -> None: "weights": weight_specs, "strata": strata_spec, "mass_log": mass_log, + "metadata": _encode_frame_metadata(frame.metadata), }, ) @@ -1106,6 +1212,14 @@ def _read_frame(path: Path, metadata: Mapping[str, Any]) -> Frame: raw_weights = manifest.get("weights") strata_spec = manifest.get("strata") raw_mass_log = manifest.get("mass_log") + raw_metadata = manifest.get("metadata") + if hashlib.sha256(_canonical_json(raw_metadata)).hexdigest() != metadata.get( + "frame_metadata_sha256" + ): + raise StoreCorrupt("Stored Frame metadata identity is missing or incorrect.") + frame_metadata = _decode_frame_metadata(raw_metadata) + if not isinstance(frame_metadata, Mapping): + raise StoreCorrupt("Stored Frame metadata must be a mapping.") if not all( ( isinstance(raw_tables, list), @@ -1245,6 +1359,7 @@ def _read_frame(path: Path, metadata: Mapping[str, Any]) -> Frame: weights, strata, mass_log=tuple(mass_log), + metadata=frame_metadata, ) except ImportError as error: raise StoreUnavailable( diff --git a/packages/microcosm-graph/tests/test_frame_metadata_store.py b/packages/microcosm-graph/tests/test_frame_metadata_store.py new file mode 100644 index 000000000..917119d77 --- /dev/null +++ b/packages/microcosm-graph/tests/test_frame_metadata_store.py @@ -0,0 +1,198 @@ +"""Invented complete Frame metadata persistence and stale codec refusal.""" + +from __future__ import annotations + +import hashlib +import json +import math +import struct + +import numpy as np +import pandas as pd +import pytest + +import microcosm.graph.store as store_module +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.store import ContentStore, StoreCorrupt, StoreUnavailable + + +def _frame(metadata): + return Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [1, 1]} + ), + "household": pd.DataFrame({"household_id": [1]}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([100.0]), WeightKind.DESIGN)}, + metadata=metadata, + ) + + +def test_roundtrip_preserves_complete_nested_source_metadata_and_types(tmp_path): + original = _frame( + { + "us_spine_assembly_manifest": { + "source_arm": "invented", + "sources": ({"sha256": "a" * 64, "rows": 2, "design_total": 100.0},), + "flags": frozenset({"native", "unknown"}), + "nullable": None, + "flag": True, + }, + "signed_zero": -0.0, + "nan": struct.unpack(">d", bytes.fromhex("7ff8000000000011"))[0], + "infinity": float("inf"), + } + ) + store = ContentStore(tmp_path / "invented-store") + store.put_frame("a" * 64, original) + restored = store.load_frame("a" * 64) + assert store_module._encode_frame_metadata( + restored.metadata + ) == store_module._encode_frame_metadata(original.metadata) + assert isinstance(restored.metadata["us_spine_assembly_manifest"]["sources"], tuple) + assert isinstance( + restored.metadata["us_spine_assembly_manifest"]["flags"], frozenset + ) + assert math.copysign(1.0, restored.metadata["signed_zero"]) == -1.0 + assert struct.pack(">d", restored.metadata["nan"]).hex() == "7ff8000000000011" + assert ( + restored.weights_for("household").values.tobytes() + == original.weights_for("household").values.tobytes() + ) + with pytest.raises(TypeError): + restored.metadata["us_spine_assembly_manifest"]["source_arm"] = "changed" + + +def test_same_key_with_changed_metadata_refuses_without_replacing_original(tmp_path): + store = ContentStore(tmp_path / "invented-store") + original = _frame({"source_sha256": "a" * 64}) + store.put_frame("b" * 64, original) + store.put_frame("b" * 64, original) + with pytest.raises(StoreCorrupt, match="different metadata"): + store.put_frame("b" * 64, _frame({"source_sha256": "b" * 64})) + assert store.load_frame("b" * 64).metadata == original.metadata + + +def test_v1_frame_is_unavailable_instead_of_silently_losing_metadata( + tmp_path, monkeypatch +): + store = ContentStore(tmp_path / "invented-store") + with monkeypatch.context() as old: + old.setattr(store_module, "_FRAME_FORMAT", "microcosm-graph-frame-v1") + store.put_frame("c" * 64, _frame({"source": "invented"})) + with pytest.raises(StoreUnavailable, match="codec"): + store.load_frame("c" * 64) + with pytest.raises(StoreUnavailable, match="predates complete metadata"): + store.put_frame("c" * 64, _frame({"source": "invented"})) + + +@pytest.mark.parametrize( + "value", + [ + ["mapping", [["duplicate", ["scalar", 1]], ["duplicate", ["scalar", 2]]]], + ["float64", "bad"], + ["scalar", 1.2], + ["mapping", [["", ["scalar", None]]]], + ["unknown", []], + ], +) +def test_malformed_metadata_refuses(value): + with pytest.raises(StoreCorrupt, match="metadata"): + store_module._decode_frame_metadata(value) + + +def _restated_manifest(store, key, *, literal=None): + """Rewrite one stored frame manifest, keeping the store's own gate honest. + + The manifest is re-registered with its true size and SHA-256 so + :func:`_verified_meta` still runs and still passes; the refusal under test + therefore comes from the decode boundary, not from a skipped checksum. + Passing ``literal=None`` restates identical content, which is the control + proving valid bytes are unaffected. + """ + object_path = store.object_path(key) + manifest_path = object_path / "frame.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if literal is not None: + manifest["metadata"] = [ + "mapping", + [["us_spine_assembly_manifest", ["float64", "@NON_FINITE@"]]], + ] + text = json.dumps(manifest, separators=(",", ":"), sort_keys=True) + manifest_path.write_text( + text.replace('"@NON_FINITE@"', literal or ""), encoding="utf-8" + ) + meta_path = object_path / "meta.json" + meta = json.loads(meta_path.read_text(encoding="utf-8")) + record = meta["payloads"]["frame.json"] + record["sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + record["size"] = manifest_path.stat().st_size + meta_path.write_text( + json.dumps(meta, separators=(",", ":"), sort_keys=True), encoding="utf-8" + ) + return manifest_path + + +@pytest.mark.parametrize( + "literal", ["NaN", "Infinity", "-Infinity", "1e999", "-1e999", "1E9999"] +) +def test_non_finite_stored_manifest_is_store_corrupt_not_type_error(tmp_path, literal): + """A checksum-consistent but non-finite manifest keeps the store taxonomy.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("d" * 64, _frame({"source": "invented"})) + manifest_path = _restated_manifest(store, "d" * 64, literal=literal) + assert literal in manifest_path.read_text(encoding="utf-8") + with pytest.raises(StoreCorrupt, match="frame manifest") as caught: + store.load_frame("d" * 64) + # The escape this pins is a bare TypeError from the canonical re-encode. + assert type(caught.value) is StoreCorrupt + + +def test_restating_the_same_manifest_still_loads_the_identical_frame(tmp_path): + """Control: the stricter decoder does not reject any valid stored bytes.""" + store = ContentStore(tmp_path / "invented-store") + original = _frame({"source": "invented", "share": 0.25, "count": 3}) + store.put_frame("e" * 64, original) + _restated_manifest(store, "e" * 64) + restored = store.load_frame("e" * 64) + assert store_module._encode_frame_metadata( + restored.metadata + ) == store_module._encode_frame_metadata(original.metadata) + + +@pytest.mark.parametrize("literal", ["NaN", "-Infinity", "1e999"]) +def test_non_finite_object_metadata_is_store_corrupt(tmp_path, literal): + """The same boundary covers meta.json, which no payload checksum guards.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("f" * 64, _frame({"source": "invented"})) + meta_path = store.object_path("f" * 64) / "meta.json" + meta = json.loads(meta_path.read_text(encoding="utf-8")) + meta["invented_non_finite"] = "@NON_FINITE@" + meta_path.write_text( + json.dumps(meta, separators=(",", ":"), sort_keys=True).replace( + '"@NON_FINITE@"', literal + ), + encoding="utf-8", + ) + with pytest.raises(StoreCorrupt, match="canonical JSON"): + store.load_frame("f" * 64) + + +def test_non_finite_json_never_reaches_the_canonical_encoder(tmp_path, monkeypatch): + """The refusal happens at decode, before any canonical re-encode runs.""" + store = ContentStore(tmp_path / "invented-store") + store.put_frame("0" * 64, _frame({"source": "invented"})) + _restated_manifest(store, "0" * 64, literal="NaN") + seen = [] + canonical = store_module._canonical_json + + def recorded(value): + seen.append(value) + return canonical(value) + + monkeypatch.setattr(store_module, "_canonical_json", recorded) + with pytest.raises(StoreCorrupt): + store.load_frame("0" * 64) + assert seen == [], "malformed JSON reached the canonical encoder" From d1019762bc89572a4e143dd0335ff0436e4ec79c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:23:10 -0400 Subject: [PATCH 094/255] Re-apply executor execution states: gate exceptions leave their consumers unreached The not-yet-amended availability piece from the integration branch, re-applied on main's executor.py and manifest.py (both otherwise byte-for-byte main's, amendment 19's payload-free edge authentication and its StoreCorrupt-on-hit miss decision included). availability.py defines the executor-owned execution state ("microcosm.graph.execution.v1"): a gate that declares typed artifact outputs and raises still becomes a `fail` verdict and the run continues (amendment 7), but its receipt now records `gate_exception` with the outputs it could not produce; a node whose predecessor is unreached, or whose declared byte input its producer recorded as unavailable, is recorded `unreached` with its blockers named by node key, runs no kernel and invents no columns, frame, weights or bytes; a release behind such an edge stays evidence-tier. Cache records carrying a state use schema 3, a cached unreached record is a hit only while the same inputs are unavailable for the same reason and is StoreCorrupt otherwise, `require` preflight derives blockers from the cached parents, a kernel that authors the executor's state is rejected, and a run manifest carrying any state serializes at schema 4 and authenticates every blocker on load. This replaces amendment 19's interim refusal of a gate that declares a typed output (the Amendment 19 lane's one ruling made on Max's behalf); main's test pinning that refusal is replaced by five tests of the regime that supersedes it. Consumers on this branch: graph_atomic_survey_clone declares the geography gate's validation artifact (emit_validation_artifact=True) and survey_age_calibration, graph_current_survey_predictors and survey_origin_budget consume it; population_input_coverage checks execution_state on every producer receipt. The frozen files are untouched. packages/microcosm-graph/tests: 518 passed, exit 0; ruff check and format clean. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/graph/availability.py | 92 ++++++ .../src/microcosm/graph/executor.py | 299 +++++++++++++++-- .../src/microcosm/graph/manifest.py | 90 ++++- .../tests/test_graph_executor.py | 312 +++++++++++++++++- 4 files changed, 749 insertions(+), 44 deletions(-) create mode 100644 packages/microcosm-graph/src/microcosm/graph/availability.py diff --git a/packages/microcosm-graph/src/microcosm/graph/availability.py b/packages/microcosm-graph/src/microcosm/graph/availability.py new file mode 100644 index 000000000..e89b1d90f --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/availability.py @@ -0,0 +1,92 @@ +"""Executor-owned exceptional outcomes, with no fabricated data products.""" + +from collections.abc import Mapping + +from .decl import StructuralDelta +from .kernel import Capabilities, KernelRole + +EXECUTION_SCHEMA = "microcosm.graph.execution.v1" + + +def has_execution(receipt: Mapping[str, object]) -> bool: + """Leave pre-existing free-form execution diagnostics uninterpreted.""" + execution = receipt.get("execution") + return ( + isinstance(execution, Mapping) and execution.get("schema") == EXECUTION_SCHEMA + ) + + +def execution_state(receipt: Mapping[str, object]) -> str | None: + execution = receipt.get("execution") + state = execution.get("state") if has_execution(receipt) else None + return state if isinstance(state, str) else None + + +def unavailable_artifacts( + receipt: Mapping[str, object], outputs: Mapping[str, object] +) -> frozenset[str]: + return ( + frozenset(outputs) + if execution_state(receipt) in {"gate_exception", "unreached"} + else frozenset() + ) + + +def validate_execution( + receipt: Mapping[str, object], + capabilities: Capabilities, + outputs: Mapping[str, object], + opaque: Mapping[str, object], + *, + has_products: bool, +) -> None: + """Validate exceptional receipt shape at cache and manifest boundaries.""" + if not has_execution(receipt): + return + execution = receipt["execution"] + if not isinstance(execution, Mapping): + raise ValueError("Executor execution metadata must be a mapping.") + state = execution.get("state") + if state == "gate_exception": + evidence = receipt.get("evidence") + missing = execution.get("unavailable_artifacts") + if ( + set(execution) != {"schema", "state", "unavailable_artifacts"} + or capabilities.role is not KernelRole.GATE + or capabilities.structural is not StructuralDelta.NONE + or receipt.get("outcome") != "fail" + or not outputs + or not isinstance(missing, list | tuple) + or tuple(missing) != tuple(sorted(outputs)) + or opaque + or not isinstance(evidence, Mapping) + or not isinstance(evidence.get("exception_type"), str) + or not isinstance(evidence.get("message"), str) + ): + raise ValueError("Invalid gate exception or unavailable artifact evidence.") + elif state == "unreached": + blockers = execution.get("blocked_by") + if ( + set(execution) != {"schema", "state", "blocked_by"} + or receipt.get("outcome") != "unreached" + or not isinstance(blockers, Mapping) + or not blockers + or any( + not isinstance(name, str) + or not name + or not isinstance(key, str) + or len(key) != 64 + or any(char not in "0123456789abcdef" for char in key) + for name, key in blockers.items() + ) + or has_products + or opaque + ): + raise ValueError("Invalid unreached blocker evidence or invented products.") + if ( + capabilities.role is KernelRole.RELEASE + and receipt.get("tier") != "evidence" + ): + raise ValueError("An unreached release must remain evidence-tier.") + else: + raise ValueError(f"Unknown executor execution state {state!r}.") diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 10d3488cf..3d15217c6 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -18,6 +18,13 @@ from . import keys as graph_keys from .artifact_edges import scope_payload, typed_contracts, value_from_descriptor +from .availability import ( + EXECUTION_SCHEMA, + execution_state, + has_execution, + unavailable_artifacts, + validate_execution, +) from .canonical import canonical_json, sha256_domain from .codecs import SOURCE_CODECS, SourceCodecRegistry from .decl import ( @@ -136,6 +143,23 @@ def _failed_gate_result( columns=MappingProxyType(columns), receipt={ "outcome": "fail", + # A gate that declares typed outputs and raises produced none of + # them. The executor, not the kernel, records that state so the + # outputs' consumers are left unreached rather than the run + # aborted (amendment 7 stays true for every legal node shape). + **( + { + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "gate_exception", + "unavailable_artifacts": sorted( + output.name for output in node.artifact_outputs + ), + } + } + if node.artifact_outputs + else {} + ), "evidence": { "exception_type": type(error).__name__, "message": str(error), @@ -1146,8 +1170,9 @@ def _validate_result( if not isinstance(payload, bytes): raise NodeRejected(f"Node {node.id!r} artifact {name!r} is not bytes.") artifacts[name] = payload + unavailable = execution_state(result.receipt) == "gate_exception" for output in node.artifact_outputs: - if output.name not in artifacts: + if output.name not in artifacts and not unavailable: # A cached record that lacks a declared artifact is a miss, but # that decision belongs to `_require_record_shape`, which runs # inside the miss-to-recompute fallback. By the time a restored @@ -1158,6 +1183,20 @@ def _validate_result( f"Node {node.id!r} is missing declared artifact {output.name!r}." ) receipt = _normal_json_mapping(result.receipt, f"Node {node.id!r} receipt") + try: + validate_execution( + receipt, + kernel_capabilities, + {output.name: output for output in node.artifact_outputs}, + artifacts, + has_products=bool( + result.columns or result.frame is not None or result.weights is not None + ), + ) + except ValueError as error: + raise NodeRejected( + f"Node {node.id!r} execution evidence rejected: {error}" + ) from error if node.structural is StructuralDelta.EXPAND: if cache_hit: if not isinstance(receipt.get("expand"), dict): @@ -1496,8 +1535,13 @@ def _write_node( record: dict[str, object] = { # Schema 2 only when the node declares typed artifacts, so a graph # that predates amendment 19 keeps its schema-1 records and its - # store hits. - "schema_version": 2 if typed_artifacts else 1, + # store hits; schema 3 only for a record carrying an executor + # execution state (gate exception or unreached). + "schema_version": 3 + if execution_state(receipt) + else 2 + if typed_artifacts + else 1, **({"typed_artifacts": dict(typed_artifacts)} if typed_artifacts else {}), "node_id": node.id, "node_key": key, @@ -1550,11 +1594,34 @@ def _require_record_shape( f"Cached receipt for node {node.id!r} has fields {sorted(raw)}, " f"not {sorted(required)}." ) - if raw["schema_version"] != (2 if typed_artifacts else 1): + raw_receipt = raw["receipt"] + if not isinstance(raw_receipt, Mapping): + raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + exceptional = execution_state(raw_receipt) + if raw["schema_version"] != (3 if exceptional else 2 if typed_artifacts else 1): raise StoreUnavailable( f"Cached receipt for node {node.id!r} uses unsupported schema " f"{raw['schema_version']!r}." ) + try: + validate_execution( + raw_receipt, + capabilities, + (typed_artifacts or {}).get("outputs", {}), + { + entry.get("name"): entry.get("key") + for entry in _record_entries(raw, "opaque") + }, + has_products=bool( + raw["columns"] + or raw["frame_key"] is not None + or raw["weight"] is not None + ), + ) + except ValueError as error: + raise StoreCorrupt( + f"Cached node {node.id!r} execution evidence rejected: {error}" + ) from error if typed_artifacts: if raw.get("typed_artifacts") != dict(typed_artifacts): raise StoreCorrupt( @@ -1567,6 +1634,10 @@ def _require_record_shape( raise StoreCorrupt(f"Cached node {node.id!r} repeats an opaque artifact.") actual_outputs = {entry.get("name"): entry.get("key") for entry in opaque} for output in node.artifact_outputs: + if exceptional: + # The record proves the output was never produced; its + # absence is that evidence, not a miss. + continue if output.name not in actual_outputs: raise StoreMiss( f"Cached node {node.id!r} is missing declared artifact " @@ -1603,10 +1674,7 @@ def _require_record_shape( f"Cached receipt capabilities for node {node.id!r} disagree with " "the registered kernel contract." ) - if node.structural is StructuralDelta.EXPAND: - raw_receipt = raw["receipt"] - if not isinstance(raw_receipt, Mapping): - raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + if node.structural is StructuralDelta.EXPAND and exceptional != "unreached": if "expand_writes" not in raw_receipt: raise StoreMiss( f"Cached EXPAND node {node.id!r} predates expand_writes provenance." @@ -1912,6 +1980,134 @@ def _all_node_keys( return keys, implementations +def _blocked_by( + compiled: CompiledGraph, + node: Node, + keys: Mapping[str, str], + receipts: Mapping[str, Mapping[str, object]], +) -> dict[str, str]: + """The predecessors whose outputs this node cannot have, by node key. + + An unreached causal parent propagates (its version, base, cells or bytes + were never produced); a typed byte input whose producer recorded it as + unavailable blocks its consumer. Every other predecessor is available, + including a failed gate that produced its verdict column. + """ + + blocked = { + parent: keys[parent] + for parent in compiled.predecessors[node.id] + if execution_state(receipts.get(parent, {})) == "unreached" + } + for binding in node.artifact_inputs: + producer = compiled.graph.node(binding.producer) + if binding.artifact in unavailable_artifacts( + receipts.get(binding.producer, {}), + {output.name: output for output in producer.artifact_outputs}, + ): + blocked[binding.producer] = keys[binding.producer] + return dict(sorted(blocked.items())) + + +def _unreached_node( + compiled: CompiledGraph, + node: Node, + *, + blockers: Mapping[str, str], + receipts: Mapping[str, NodeReceipt], + store: ContentStore, + key: str, + implementation: str, + capabilities: Capabilities, + typed: Mapping[str, object], + resume: ResumePolicy, +) -> NodeReceipt: + """Record a proven lack of inputs without running or inventing products. + + The receipt names each blocker by node key, so a cached unreached record + is a hit only while the same inputs are unavailable for the same reason; + the record holds no columns, frame, weights or bytes. A release that is + unreached has a failed or unreached gate in its ancestry by construction + and stays evidence-tier. + """ + + receipt: dict[str, object] = { + "outcome": "unreached", + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": dict(blockers), + }, + "evidence": {"reason": "Required graph inputs are unavailable."}, + "capabilities": _capabilities_projection(capabilities), + } + if capabilities.role is KernelRole.RELEASE: + tier, gate_ids = _release_tier(compiled, node.id, receipts) + if tier != "evidence": + raise NodeRejected( + f"Release node {node.id!r} is unreached but no ancestral gate " + "failed or was unreached." + ) + receipt.update( + tier=tier, + gate_ancestry=list(gate_ids), + requires_decisions=list(_required_decision_names(node)), + ) + hit = False + replace_stale_record = False + if resume != "forbid": + try: + record = _load_record( + store, + node, + key=key, + kernel_impl_hash=implementation, + capabilities=capabilities, + typed_artifacts=typed, + ) + if record["receipt"] != receipt: + raise StoreCorrupt( + f"Cached node {node.id!r} blocked provenance disagrees with " + "its inputs." + ) + hit = True + except StoreMiss: + replace_stale_record = store.has(_cache_record_key(key)) + if resume == "require": # defended by preflight; handles races + raise + if not hit: + record = { + "schema_version": 3, + **({"typed_artifacts": dict(typed)} if typed else {}), + "node_id": node.id, + "node_key": key, + "kernel_ref": node.kernel, + "kernel_impl_hash": implementation, + "capabilities": _capabilities_projection(capabilities), + "receipt": receipt, + "columns": [], + "frame_key": None, + "weight": None, + "opaque": [], + } + store.put_json( + _cache_record_key(key), + record, + node_key=key, + verify_existing=resume != "forbid" and not replace_stale_record, + ) + return NodeReceipt( + key=key, + hit=hit, + seed=seed(key), + kernel_ref=node.kernel, + kernel_impl_hash=implementation, + capabilities=capabilities, + receipt=receipt, + typed_artifacts=typed, + ) + + def _preflight_require( compiled: CompiledGraph, store: ContentStore, @@ -1920,6 +2116,7 @@ def _preflight_require( kernels: KernelRegistry, ) -> None: missing: list[str] = [] + receipts: dict[str, Mapping[str, object]] = {} for node_id in compiled.order: node = compiled.graph.node(node_id) try: @@ -1931,13 +2128,35 @@ def _preflight_require( capabilities=kernels.get(node.kernel).capabilities, typed_artifacts=typed_contracts(compiled, node, keys, kernels), ) - _require_tolerance_writer_receipt( - node, - record, - _input_writers(compiled, node_id), - exact=False, - ) + if any(parent not in receipts for parent in compiled.predecessors[node_id]): + # A missing parent already made this run a miss; without its + # receipt the blockers below could not be derived honestly. + missing.append(node_id) + continue + blockers = _blocked_by(compiled, node, keys, receipts) + if blockers: + if record["receipt"].get("execution") != { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": blockers, + }: + raise StoreCorrupt( + f"Cached node {node_id!r} blocked provenance disagrees " + "with its inputs." + ) + elif execution_state(record["receipt"]) == "unreached": + raise StoreCorrupt( + f"Cached node {node_id!r} has no unavailable input blocker." + ) + else: + _require_tolerance_writer_receipt( + node, + record, + _input_writers(compiled, node_id), + exact=False, + ) _preflight_record(store, record) + receipts[node_id] = record["receipt"] except StoreMiss: missing.append(node_id) if missing: @@ -1993,26 +2212,12 @@ def run_graph( node_id: typed_contracts(compiled, compiled.graph.node(node_id), keys, kernels) for node_id in compiled.order } - for node_id in compiled.order: - node = compiled.graph.node(node_id) - # A gate whose kernel raises becomes a `fail` verdict and the run - # continues (amendment 7), so its synthesized result carries no - # artifacts. Amendment 19 has no regime for an output a node was - # unable to produce, so a gate that declares one is refused rather - # than allowed to turn a verdict into an aborted run. - if node.artifact_outputs and ( - kernels.get(node.kernel).capabilities.role is KernelRole.GATE - ): - raise NodeRejected( - f"Node {node_id!r}: a gate kernel may not declare a typed artifact " - "output, because a gate exception is a verdict and would leave the " - "output unproduced." - ) if resume == "require": _preflight_require(compiled, store, keys, implementations, kernels) populations: dict[str, Population] = {} receipts: dict[str, NodeReceipt] = {} + receipt_payloads: dict[str, Mapping[str, object]] = {} for node_id in compiled.order: node_started = time.perf_counter() node = compiled.graph.node(node_id) @@ -2025,6 +2230,23 @@ def run_graph( "capabilities." ) + blockers = _blocked_by(compiled, node, keys, receipt_payloads) + if blockers: + receipts[node_id] = _unreached_node( + compiled, + node, + blockers=blockers, + receipts=receipts, + store=store, + key=key, + implementation=implementation, + capabilities=kernel.capabilities, + typed=contracts[node_id], + resume=resume, + ) + receipt_payloads[node_id] = receipts[node_id].receipt + continue + if node.structural is StructuralDelta.CREATE: incumbent: Population | None = None elif node.structural is StructuralDelta.NONE: @@ -2084,6 +2306,10 @@ def run_graph( capabilities=kernel.capabilities, typed_artifacts=typed, ) + if execution_state(record["receipt"]) == "unreached": + raise StoreCorrupt( + f"Cached node {node_id!r} has no unavailable input blocker." + ) try: _require_tolerance_writer_receipt( node, record, input_writers, exact=True @@ -2131,6 +2357,20 @@ def run_graph( raise NodeRejected( f"Node {node.id!r} kernel {node.kernel!r} failed: {error}" ) from error + else: + # The execution state is executor evidence about what a + # kernel could not produce; a kernel that returns one is a + # contract rejection, not an exception raised while a gate + # computed its verdict. + if ( + isinstance(result, KernelResult) + and isinstance(result.receipt, Mapping) + and has_execution(result.receipt) + ): + raise NodeRejected( + f"Node {node.id!r} kernel receipt may not author executor " + "execution metadata." + ) after = _context_digest(context) if before != after: raise NodeRejected(f"Node {node.id!r} mutated its input context.") @@ -2300,6 +2540,7 @@ def run_graph( weight_key=receipt_weight_key, opaque_artifacts=MappingProxyType(receipt_opaque), ) + receipt_payloads[node_id] = receipts[node_id].receipt return RunManifest( country=compiled.graph.country, diff --git a/packages/microcosm-graph/src/microcosm/graph/manifest.py b/packages/microcosm-graph/src/microcosm/graph/manifest.py index e61680bf2..1adb8ded8 100644 --- a/packages/microcosm-graph/src/microcosm/graph/manifest.py +++ b/packages/microcosm-graph/src/microcosm/graph/manifest.py @@ -14,6 +14,12 @@ from microcosm.frame import Frame from .artifact_edges import require_compatible_scope, value_from_descriptor +from .availability import ( + execution_state, + has_execution, + unavailable_artifacts, + validate_execution, +) from .canonical import canonical_json, sha256_domain from .decl import GATE_OUTCOMES, StructuralDelta from .errors import NodeRejectedError, StoreCorruptError @@ -36,6 +42,7 @@ _SCHEMA_VERSION = 2 _TYPED_SCHEMA_VERSION = 3 +_EXCEPTION_SCHEMA_VERSION = 4 _LEGACY_SCHEMA_VERSION = 1 _CERTIFYING_GATE_OUTCOMES = frozenset({"pass", "not_applicable"}) @@ -301,6 +308,21 @@ def __post_init__(self) -> None: opaque_artifacts[name] = key object.__setattr__(self, "opaque_artifacts", MappingProxyType(opaque_artifacts)) + if has_execution(self.receipt): + if self.legacy_capabilities: + raise ValueError("Legacy receipts cannot carry executor outcomes.") + validate_execution( + self.receipt, + self.capabilities, + self.typed_artifacts.get("outputs", {}), + self.opaque_artifacts, + has_products=bool( + self.artifacts + or self.frame_key is not None + or self.weight_key is not None + ), + ) + wall_time = float(self.wall_time) if not math.isfinite(wall_time) or wall_time < 0: raise ValueError("NodeReceipt.wall_time must be finite and non-negative") @@ -621,7 +643,9 @@ def to_json(self) -> str: """Serialize the complete portable provenance as canonical JSON.""" payload = { - "schema_version": _TYPED_SCHEMA_VERSION + "schema_version": _EXCEPTION_SCHEMA_VERSION + if any(execution_state(node.receipt) for node in self.nodes.values()) + else _TYPED_SCHEMA_VERSION if any(node.typed_artifacts for node in self.nodes.values()) else _SCHEMA_VERSION, "key": self.key, @@ -665,6 +689,7 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: _LEGACY_SCHEMA_VERSION, _SCHEMA_VERSION, _TYPED_SCHEMA_VERSION, + _EXCEPTION_SCHEMA_VERSION, }: raise ValueError(f"unsupported manifest schema version {schema_version!r}") @@ -693,6 +718,10 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: node.typed_artifacts for node in nodes.values() ): raise ValueError("Schema-v3 manifest must carry typed artifact provenance.") + if schema_version == _EXCEPTION_SCHEMA_VERSION and not any( + execution_state(node.receipt) for node in nodes.values() + ): + raise ValueError("Schema-v4 manifest must carry executor outcomes.") body = raw.get("content_addressed") if not isinstance(body, Mapping): raise ValueError("manifest content-addressed body must be an object") @@ -1127,8 +1156,17 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec weight_key = value.get("weight_key") opaque_artifacts = value.get("opaque_artifacts", {}) typed_artifacts = value.get("typed_artifacts", {}) - if "typed_artifacts" in value and schema_version != _TYPED_SCHEMA_VERSION: - raise ValueError("Typed artifact provenance requires manifest schema 3.") + if ( + isinstance(receipt, Mapping) + and has_execution(receipt) + and schema_version != _EXCEPTION_SCHEMA_VERSION + ): + raise ValueError("Executor exceptional outcomes require manifest schema 4.") + if "typed_artifacts" in value and schema_version not in { + _TYPED_SCHEMA_VERSION, + _EXCEPTION_SCHEMA_VERSION, + }: + raise ValueError("Typed artifact provenance requires manifest schema 3 or 4.") capabilities_payload = value.get("capabilities") if schema_version == _LEGACY_SCHEMA_VERSION: # Every schema-v1 receipt is legacy: v1 never recorded a tolerance, so @@ -1228,7 +1266,16 @@ def _validate_typed_ancestry(nodes: Mapping[str, NodeReceipt]) -> None: entry["producer"] != node_id or entry["artifact"] != name or value.producer_key != node.key - or node.opaque_artifacts.get(name) != value.key + or ( + # A gate exception recorded the output as unavailable, so + # no stored identity exists for it; every other output + # must resolve to its stored bytes. + name + not in unavailable_artifacts( + node.receipt, node.typed_artifacts["outputs"] + ) + and node.opaque_artifacts.get(name) != value.key + ) ): raise ValueError( f"Node {node_id!r} typed artifact output provenance mismatch." @@ -1268,8 +1315,43 @@ def _validate_typed_ancestry(nodes: Mapping[str, NodeReceipt]) -> None: f"Node {node_id!r} typed artifact carries a numeric scope its " f"consumer may not read: {error}" ) from error + if entry["artifact"] in unavailable_artifacts( + producer.receipt, producer.typed_artifacts.get("outputs", {}) + ) and ( + execution_state(node.receipt) != "unreached" + or producer_id not in node.receipt["execution"]["blocked_by"] + ): + raise ValueError( + f"Node {node_id!r} consumes an unavailable artifact but is not " + "unreached by its producer." + ) edges[node_id].add(producer_id) + for node_id, node in nodes.items(): + if execution_state(node.receipt) != "unreached": + continue + for parent_id, parent_key in node.receipt["execution"]["blocked_by"].items(): + parent = nodes.get(parent_id) + if parent is None or parent.key != parent_key: + raise ValueError( + f"Node {node_id!r} unreached blocker {parent_id!r} is missing or " + "has a different key." + ) + if execution_state(parent.receipt) not in {"gate_exception", "unreached"}: + raise ValueError( + f"Node {node_id!r} unreached blocker {parent_id!r} has no " + "unavailable outputs." + ) + if execution_state(parent.receipt) == "gate_exception" and not any( + entry["producer"] == parent_id + for entry in node.typed_artifacts.get("inputs", {}).values() + ): + raise ValueError( + f"Node {node_id!r} names a gate exception blocker that is not " + "one of its declared typed inputs." + ) + edges[node_id].add(parent_id) + memo: dict[str, frozenset[str]] = {} def visit(node_id: str, trail: frozenset[str]) -> frozenset[str]: diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index e6c12a8f1..c47cc985d 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -16,6 +16,7 @@ import microcosm.graph.executor as graph_executor from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.availability import EXECUTION_SCHEMA from microcosm.graph.decl import ( ArtifactInput, ArtifactOutput, @@ -3620,13 +3621,15 @@ def test_an_artifact_payload_enters_the_input_context_digest(tmp_path: Path) -> ) -def test_a_gate_kernel_may_not_declare_a_typed_artifact_output(tmp_path: Path) -> None: - """Amendment 19 holds amendment 7: a gate exception stays a verdict. +EVIDENCE = ArtifactType("gate.evidence", 1) - A gate whose kernel raises produces a synthesized ``fail`` result with no - artifacts, so a declared typed output would turn that verdict into an - aborted run. Amendment 19 carries no regime for an output a node was - unable to produce, so the declaration is refused outright. + +def _gate_artifact_graph(*, release_behind: bool, answer: str) -> Graph: + """A gate declaring typed evidence, its byte consumer, and a cell consumer. + + ``release`` reads the cell consumer's column when ``release_behind`` is + true (so it sits behind the byte edge) and the gate's own verdict column + otherwise (so it sits beside it). """ gate = Node( "gate", @@ -3634,22 +3637,309 @@ def test_a_gate_kernel_may_not_declare_a_typed_artifact_output(tmp_path: Path) - inputs=(Slice("person", ("age",)),), outputs=(Owned("household", "gate_verdict", "string"),), population="survey", - artifact_outputs=( - ArtifactOutput("evidence", ArtifactType("gate.evidence", 1)), + artifact_outputs=(ArtifactOutput("evidence", EVIDENCE),), + ) + use = Node( + "use", + "use@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "used", "float64"),), + population="survey", + artifact_inputs=(ArtifactInput("evidence", "gate", "evidence", EVIDENCE),), + ) + after = Node( + "after", + "after@1", + inputs=(Slice("person", ("used",)),), + outputs=(Owned("person", "after", "float64"),), + population="survey", + ) + release = Node( + "release", + "release@1", + inputs=( + (Slice("person", ("after",)),) + if release_behind + else (Slice("household", ("gate_verdict",)),) ), + outputs=(Owned("household", "tier", "string"),), + params={"answer": answer, "requires_decisions": ()}, + population="survey", ) - graph = Graph("toy", (SOURCE,), (CREATE, gate)) + return Graph("toy", (SOURCE,), (CREATE, gate, use, after, release)) + + +def _gate_artifact_registry(*, raising: bool) -> KernelRegistry: + def failing_gate(context: KernelContext) -> KernelResult: + raise RuntimeError("evidence unavailable") + + def passing_gate(context: KernelContext) -> KernelResult: + ids = context.tables["household"]["household_id"] + return KernelResult( + columns={ + ("household", "gate_verdict"): pd.Series( + "pass", index=ids, dtype="string" + ) + }, + artifacts={"evidence": b"evidence-bytes"}, + receipt={"outcome": "pass", "evidence": {"fixture": True}}, + ) + + def use(context: KernelContext) -> KernelResult: + table = context.tables["person"] + payload = context.artifacts["evidence"].payload + return KernelResult( + columns={ + ("person", "used"): pd.Series( + np.full(len(table), float(len(payload))), + index=pd.Index(table["person_id"], name="person_id"), + dtype="float64", + ) + } + ) + + def after(context: KernelContext) -> KernelResult: + table = context.tables["person"] + return KernelResult( + columns={ + ("person", "after"): pd.Series( + table["used"].to_numpy(dtype=np.float64) * 2.0, + index=pd.Index(table["person_id"], name="person_id"), + dtype="float64", + ) + } + ) + + def release(context: KernelContext) -> KernelResult: + ids = context.tables["household"]["household_id"] + answer = str(context.params["answer"]) + return KernelResult( + columns={ + ("household", "tier"): pd.Series(answer, index=ids, dtype="string") + }, + receipt={"outcome": "pass"}, + ) + + deterministic = Capabilities(Determinism.DETERMINISTIC) registry = _registry() registry.register( _Kernel( "gate@1", Capabilities(Determinism.DETERMINISTIC, role=KernelRole.GATE), - lambda context: KernelResult(receipt={"outcome": "pass"}), + failing_gate if raising else passing_gate, + ) + ) + registry.register(_Kernel("use@1", deterministic, use)) + registry.register(_Kernel("after@1", deterministic, after)) + registry.register( + _Kernel( + "release@1", + Capabilities(Determinism.DETERMINISTIC, role=KernelRole.RELEASE), + release, ) ) + return registry + + +def test_a_gate_that_declares_evidence_and_raises_leaves_its_consumers_unreached( + tmp_path: Path, +) -> None: + """A gate exception is still a verdict (amendment 7); its outputs are absent. + + The gate records ``fail`` with a ``gate_exception`` execution state naming + the outputs it could not produce; the byte consumer and everything causally + behind it are ``unreached`` with their blockers named by node key, no + kernel behind the edge runs, nothing is invented for them, the release + behind the edge stays evidence-tier, and the manifest serializes at schema + 4, round-trips, and replays as hits under every resume policy. + """ store = ContentStore(tmp_path / "store") - with pytest.raises(NodeRejected, match="gate kernel may not declare"): + source = _source_path(tmp_path / "src") + graph = _gate_artifact_graph(release_behind=True, answer="evidence") + registry = _gate_artifact_registry(raising=True) + manifest = _run(graph, source, store, registry) + + gate = manifest.nodes["gate"] + assert gate.receipt["outcome"] == "fail" + assert gate.receipt["execution"] == { + "schema": EXECUTION_SCHEMA, + "state": "gate_exception", + "unavailable_artifacts": ("evidence",), + } + assert gate.receipt["evidence"]["exception_type"] == "RuntimeError" + assert not gate.opaque_artifacts + assert set(gate.typed_artifacts["outputs"]) == {"evidence"} + verdict = manifest.populations["survey"].household["gate_verdict"] + assert set(verdict.to_numpy()) == {"fail"} + assert "used" not in manifest.populations["survey"].person.columns + + use = manifest.nodes["use"] + assert use.receipt["outcome"] == "unreached" + assert use.receipt["execution"] == { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": {"gate": gate.key}, + } + assert not use.artifacts and use.frame_key is None and not use.opaque_artifacts + after = manifest.nodes["after"] + assert after.receipt["execution"]["blocked_by"] == {"use": use.key} + release = manifest.nodes["release"] + assert release.receipt["execution"]["blocked_by"] == {"after": after.key} + assert release.receipt["tier"] == "evidence" + assert release.receipt["gate_ancestry"] == ("gate",) + assert manifest.tier == "evidence" + calls = _calls(registry) + assert calls["gate@1"] == 1 + assert calls["use@1"] == calls["after@1"] == calls["release@1"] == 0 + + text = manifest.to_json() + assert '"schema_version":4' in text + restored = RunManifest.from_json(text) + assert restored.key == manifest.key + assert restored.nodes["use"].receipt == use.receipt + assert restored.tier == "evidence" + + for resume in ("auto", "require"): + again = _gate_artifact_registry(raising=True) + replay = _run(graph, source, store, again, resume=resume) + assert all(receipt.hit for receipt in replay.nodes.values()) + assert replay.key == manifest.key + assert sum(_calls(again).values()) == 0 + + +def test_a_gate_that_declares_evidence_and_passes_produces_it( + tmp_path: Path, +) -> None: + """The same declaration on a gate that succeeds is an ordinary byte edge.""" + store = ContentStore(tmp_path / "store") + source = _source_path(tmp_path / "src") + graph = _gate_artifact_graph(release_behind=True, answer="certified") + manifest = _run(graph, source, store, _gate_artifact_registry(raising=False)) + gate = manifest.nodes["gate"] + assert gate.receipt["outcome"] == "pass" + assert "execution" not in gate.receipt + assert gate.opaque_artifacts["evidence"] == opaque_artifact_key( + gate.key, "evidence" + ) + assert store.load_bytes(gate.opaque_artifacts["evidence"]) == b"evidence-bytes" + used = manifest.populations["survey"].person["used"] + assert set(used.to_numpy()) == {float(len(b"evidence-bytes"))} + assert manifest.nodes["release"].receipt["gate_ancestry"] == ("gate",) + assert manifest.tier == "certified" + assert '"schema_version":3' in manifest.to_json() + + +def test_a_kernel_may_not_author_the_executor_execution_state( + tmp_path: Path, +) -> None: + """The execution state is executor evidence; a kernel returning one is rejected.""" + + def authoring(context: KernelContext) -> KernelResult: + table = context.tables["person"] + return KernelResult( + columns={ + ("person", "a"): pd.Series( + np.zeros(len(table)), + index=pd.Index(table["person_id"], name="person_id"), + dtype="float64", + ) + }, + receipt={ + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": {}, + } + }, + ) + + registry = KernelRegistry() + registry.register( + _Kernel( + "source@1", + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE), + _source, + ) + ) + registry.register( + _Kernel("a@1", Capabilities(Determinism.DETERMINISTIC), authoring) + ) + graph = Graph("toy", (SOURCE,), (CREATE, _ordinary("a", "a@1", "age", "a"))) + store = ContentStore(tmp_path / "store") + with pytest.raises(NodeRejected, match="may not author executor execution"): _run(graph, _source_path(tmp_path / "src"), store, registry) + # A free-form "execution" diagnostic that does not claim the executor's + # schema is still just a receipt field. + assert ( + graph_executor.has_execution({"execution": {"literal_full_scans": 1}}) is False + ) + + +def test_a_cached_unreached_record_is_refused_once_its_inputs_exist( + tmp_path: Path, +) -> None: + """An unreached record is a hit only while the same inputs are unavailable.""" + store = ContentStore(tmp_path / "store") + source = _source_path(tmp_path / "src") + graph = _gate_artifact_graph(release_behind=True, answer="certified") + manifest = _run(graph, source, store, _gate_artifact_registry(raising=False)) + gate_key = manifest.nodes["gate"].key + use_key = manifest.nodes["use"].key + record_key = graph_executor._cache_record_key(use_key) + raw = store.load_json(record_key) + raw.update( + schema_version=3, + receipt={ + "outcome": "unreached", + "execution": { + "schema": EXECUTION_SCHEMA, + "state": "unreached", + "blocked_by": {"gate": gate_key}, + }, + "evidence": {"reason": "Required graph inputs are unavailable."}, + "capabilities": raw["capabilities"], + }, + columns=[], + frame_key=None, + weight=None, + opaque=[], + ) + store.put_json(record_key, raw, node_key=use_key, verify_existing=False) + for resume in ("auto", "require"): + with pytest.raises(StoreCorrupt, match="has no unavailable input blocker"): + _run( + graph, + source, + store, + _gate_artifact_registry(raising=False), + resume=resume, + ) + + +def test_a_manifest_authenticates_its_unreached_blockers(tmp_path: Path) -> None: + """Portable provenance names each blocker by key and the manifest checks it.""" + store = ContentStore(tmp_path / "store") + source = _source_path(tmp_path / "src") + graph = _gate_artifact_graph(release_behind=True, answer="evidence") + manifest = _run(graph, source, store, _gate_artifact_registry(raising=True)) + payload = json.loads(manifest.to_json()) + + forged = json.loads(json.dumps(payload)) + forged["nodes"]["use"]["receipt"]["execution"]["blocked_by"] = {"gate": "0" * 64} + with pytest.raises(ValueError, match="missing or has a different key"): + RunManifest.from_json(json.dumps(forged)) + + forged = json.loads(json.dumps(payload)) + forged["nodes"]["after"]["receipt"]["execution"]["blocked_by"] = { + "gate": payload["nodes"]["gate"]["key"] + } + with pytest.raises(ValueError, match="not one of its declared typed inputs"): + RunManifest.from_json(json.dumps(forged)) + + forged = json.loads(json.dumps(payload)) + forged["schema_version"] = 3 + with pytest.raises(ValueError, match="require manifest schema 4"): + RunManifest.from_json(json.dumps(forged)) def test_a_gate_reached_only_through_bytes_still_derives_the_tier( From a16eeacf8379e9c436178b625b9e917f37b2e762 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:23:38 -0400 Subject: [PATCH 095/255] Re-apply the private population observer on run_graph The last of the four not-yet-amended graph pieces the US runtime on this branch consumes: `run_graph(_population_observer=...)` hands each node's admitted population (design anchors included) to an integrating verifier, cold and on cache hits alike, before the node is persisted. It is not a kernel capability, enters no key or receipt, and an exception it raises refuses the run; an unreached node has no population to observe. Consumers: survey_age_calibration, graph_atomic_survey_financial, graph_survey_population and graph_atomic_survey_population, plus eight US test files. The branch carried it inside the lazy-retention change and had no graph-level test; one is added here. test_graph_executor, test_acceptance_b_ownership, test_acceptance_f_gates green, exit 0. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/graph/executor.py | 16 +++++- .../tests/test_graph_executor.py | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 3d15217c6..53c119329 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -6,7 +6,7 @@ import json import socket import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from datetime import UTC, datetime from pathlib import Path from types import MappingProxyType @@ -2189,8 +2189,17 @@ def run_graph( kernels: KernelRegistry, resume: ResumePolicy = "auto", decisions: tuple[Decision, ...] = (), + _population_observer: Callable[[str, Population], None] | None = None, ) -> RunManifest: - """Execute a compiled graph with content-addressed reuse and receipts.""" + """Execute a compiled graph with content-addressed reuse and receipts. + + The private population observer exposes each node's admitted population, + design anchors included, to an integrating verifier. It runs for cold + execution and for restored cache hits alike, before the node is persisted; + it must not mutate the population, and an exception it raises refuses the + run. It is never a kernel capability, enters no key or receipt, and an + unreached node has no population to observe. + """ if resume not in ("auto", "require", "forbid"): raise ValueError("resume must be 'auto', 'require', or 'forbid'.") @@ -2490,6 +2499,9 @@ def run_graph( else: populations[node.id] = updated + if _population_observer is not None: + _population_observer(node_id, updated) + if not hit: manifest_artifacts, record = _write_node( store, diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index c47cc985d..46501139f 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -48,6 +48,7 @@ ) from microcosm.graph.keys import opaque_artifact_key, platform_fingerprint from microcosm.graph.manifest import Decision, RunManifest +from microcosm.graph.population import Population from microcosm.graph.store import ( ContentStore, StoreCorrupt, @@ -3942,6 +3943,58 @@ def test_a_manifest_authenticates_its_unreached_blockers(tmp_path: Path) -> None RunManifest.from_json(json.dumps(forged)) +def test_the_private_population_observer_sees_every_admitted_population( + tmp_path: Path, +) -> None: + """``_population_observer`` runs per node, cold and on hits, before persistence. + + It is an integration seam for verifiers, not a kernel capability: nothing + it does enters a key or a receipt, and an exception it raises refuses the + run. + """ + store = ContentStore(tmp_path / "store") + source = _source_path(tmp_path / "src") + seen: list[tuple[str, int]] = [] + + def observe(node_id: str, population: Population) -> None: + seen.append((node_id, population.frame.n("person"))) + + cold = run_graph( + compile_graph(_graph()), + sources={"survey": source}, + store=store, + kernels=_registry(), + _population_observer=observe, + ) + assert [node_id for node_id, _ in seen] == list(cold.nodes) + assert {n for _, n in seen} == {3} + plain = _run(_graph(), source, store, _registry()) + assert plain.key == cold.key + + seen.clear() + warm = run_graph( + compile_graph(_graph()), + sources={"survey": source}, + store=store, + kernels=_registry(), + _population_observer=observe, + ) + assert all(receipt.hit for receipt in warm.nodes.values()) + assert [node_id for node_id, _ in seen] == list(warm.nodes) + + def refuse(node_id: str, population: Population) -> None: + raise RuntimeError(f"verifier refused {node_id}") + + with pytest.raises(RuntimeError, match="verifier refused survey"): + run_graph( + compile_graph(_graph()), + sources={"survey": source}, + store=ContentStore(tmp_path / "other"), + kernels=_registry(), + _population_observer=refuse, + ) + + def test_a_gate_reached_only_through_bytes_still_derives_the_tier( tmp_path: Path, ) -> None: From 2e1857a4b38c95c2822d06cf7e3479d5bfe22bc6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:33:14 -0400 Subject: [PATCH 096/255] Bring the #893 reconciliation journal up to the piece commits Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 55 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index cb40fc386..bdd76df6e 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -8,26 +8,53 @@ state" in `CLAUDE.md`. ## State -In progress. Baseline before any change: `packages/microcosm-graph/tests` -5 failed / 362 passed (exit 1) — exactly the five the brief names. +Merges done, four graph pieces re-applied, graph package green (518 passed), +fit green (191), spec check / groups verify / ruff clean. Build-side consumer +suites running at the time of this entry; final results in `out.md` §5 and +`experiments/893-reconciliation-amendments-19-20-20260912.md`. Nothing +pushed, no PR, no new branch, `uv.lock` untouched. ## Done -- Read `CLAUDE.md`, the charter's "Interface freeze" at the amendment-20 head - (`23ba24770`), PR #893's body, and the Amendment 19 lane's "Scope" list on - main. Diffed the graph package against `23ba24770` file by file and located - every consumer of each not-yet-amended piece outside the graph package. -- `uv sync --all-packages --locked --extra us --extra uk` exit 0. +- Read `CLAUDE.md`, the charter's "Interface freeze" at `23ba24770`, PR + #893's body, the Amendment 19 lane's "Scope" list; diffed the graph package + against `23ba24770` per file; AST-scanned every non-graph consumer. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. Baseline + graph suite: 5 failed / 362 passed (the five the brief names). +- `3010b7788` merge `origin/main`: graph package, fit sources, lock and + charter resolved to main's bytes; branch-only `attachments.py`, + `availability.py`, `schema.py` and the two branch-only graph tests removed + in the merge, to return only where consumed. +- `051357909` merge `23ba24770` (amendment 20, merged from the branch head + because #912 was still on CI; the dispatcher re-runs `git merge origin/main` + after it lands — expected no-op for graph/fit/lock/charter). +- `dc621c14c` seed digests re-pinned (the branch's `acs_transfer` and + `housing_inputs` plus amendment 20's `fit.qrf` move them); coverage report + regenerated; `--check` exit 0. +- `cff8fbf32` calibrate/simulate H1 pins re-recorded (the branch's solver and + `Frame.__reduce__` changes move them); parity files 27 passed. +- Pieces re-applied on main's files, each with graph-level tests the branch + never had: `752ab840f` raw-byte codec (13 US consumers; 8 codec tests), + `db1b7821a` Frame-metadata store (3 named consumers; 274 passed across the + store/population/executor/manifest files), `d1019762b` execution states + (the post-clone geography gate's typed artifact; replaces the amendment-19 + gate refusal and its test; whole graph package 518 passed), `a16eeacf8` + population observer (4 consumers; 94 passed on the executor + B/F files). +- Dropped for lack of a consumer: lazy retention (`attachments.py`, layer + 08), `schema.py`, `keys._stream_file`, the `_write_node` refactor. +- Layer map computed from `git log --name-only` attribution plus an AST + import scan with hard / name-hard / soft link classes; written to `out.md`. ## Next -- `git merge origin/main`, then `git merge origin/amend-keyed-seed-and-uniform-draws` - (`23ba24770`), resolving every graph/fit/charter file to main's bytes. -- Re-apply, as separate named commits, only the not-yet-amended pieces the US - runtime on this branch consumes; drop the rest and say so in `out.md`. -- Run the graph and fit packages, the spine-blindness file, the spec-engine - check, `ci_test_groups --verify`, ruff, then the touched US suites. -- Write the layer map and the draft amendment entries to `out.md`. +- Record the build-side consumer results in `out.md` §5, commit the + `experiments/` copy of the report, leave `out.md` uncommitted (it is + another lane's tracked report; see the memory note). +- For Max: piece C supersedes amendment 19's gate refusal (`out.md` §8.1); + the dropped lazy retention / `_stream_file` / `_write_node` pieces (§8.2–3). + +--- + # Amendment 19 — typed opaque artifacts on the graph interface Lane: `amend-typed-artifacts`, off `origin/main` at `3094bfe84`. Started From c704a1be634271a8e6feb43244f3151e631f8f2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:34:32 -0400 Subject: [PATCH 097/255] Pin that unreached propagates through a structural node and its version A FILTER whose input column comes from an unreached node is unreached with the filter's blocker named, the version it would have opened carries no population, a node placed on that version is blocked by the filter (its version is a compiled predecessor), a node beside the edge still runs, the manifest round-trips, and resume='require' replays every record as a hit. Piece C's earlier tests covered only non-structural consumers. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_graph_executor.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index 46501139f..09c376cd7 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -3808,6 +3808,97 @@ def test_a_gate_that_declares_evidence_and_raises_leaves_its_consumers_unreached assert sum(_calls(again).values()) == 0 +def test_unreached_propagates_through_a_structural_node_and_its_version( + tmp_path: Path, +) -> None: + """A FILTER whose input is unreached is unreached, and so is its version. + + The version the filter would have opened has no population, so a node + placed on it is blocked by the filter itself (its version is one of its + compiled predecessors), while a node beside the edge still runs. + """ + gate = Node( + "gate", + "gate@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("household", "gate_verdict", "string"),), + population="survey", + artifact_outputs=(ArtifactOutput("evidence", EVIDENCE),), + ) + use = Node( + "use", + "use@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "used", "float64"),), + population="survey", + artifact_inputs=(ArtifactInput("evidence", "gate", "evidence", EVIDENCE),), + ) + boundary = Node( + "boundary", + "keep@1", + inputs=(Slice("person", ("used",)),), + structural=StructuralDelta.FILTER, + base="survey", + ) + on_boundary = Node( + "on_boundary", + "after@1", + inputs=(Slice("person", ("used",)),), + outputs=(Owned("person", "after", "float64"),), + population="boundary", + ) + beside = Node( + "beside", + "a@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "a", "float64"),), + params={"source": "age", "target": "a", "scale": 1.0}, + population="survey", + ) + graph = Graph("toy", (SOURCE,), (CREATE, gate, use, boundary, on_boundary, beside)) + + def keep(context: KernelContext) -> KernelResult: + person = context.tables["person"] + return KernelResult( + keep=pd.Series(True, index=person["person_id"], dtype="bool") + ) + + def registry_with_filter() -> KernelRegistry: + registry = _gate_artifact_registry(raising=True) + registry.register( + _Kernel( + "keep@1", + Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ), + keep, + ) + ) + return registry + + registry = registry_with_filter() + store = ContentStore(tmp_path / "store") + source = _source_path(tmp_path / "src") + manifest = _run(graph, source, store, registry) + nodes = manifest.nodes + assert nodes["boundary"].receipt["execution"]["blocked_by"] == { + "use": nodes["use"].key + } + assert nodes["on_boundary"].receipt["execution"]["blocked_by"] == { + "boundary": nodes["boundary"].key + } + assert nodes["boundary"].frame_key is None + assert "boundary" not in manifest.populations + assert "execution" not in nodes["beside"].receipt + assert set(manifest.populations["survey"].person["a"]) == {10.0, 20.0, 30.0} + calls = _calls(registry) + assert calls["keep@1"] == calls["after@1"] == 0 and calls["a@1"] == 1 + restored = RunManifest.from_json(manifest.to_json()) + assert restored.key == manifest.key + replay = _run(graph, source, store, registry_with_filter(), resume="require") + assert all(receipt.hit for receipt in replay.nodes.values()) + + def test_a_gate_that_declares_evidence_and_passes_produces_it( tmp_path: Path, ) -> None: From 07566eb1733a97347355b4f9d2866ece254262d3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:40:54 -0400 Subject: [PATCH 098/255] Re-pin the US implementation inventory for the reconciled graph package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit graph_implementation_inventory.json is the reviewed dependency fence every US source stage checks before it runs. Three of its entries no longer held: - the microcosm.graph package roster listed attachments.py and schema.py, both dropped by the reconciliation (no consumer), so every stage that binds microcosm.graph refused with "Python inventory differs" and the refusal surfaced as PREPARATION_ISSUANCE_REFUSED in 23 survey tests; - the contracts for executor.py and manifest.py named the attachments import, keys.py's resource-access digest reflected the dropped _stream_file, and population.py's contract predated main's #907 import of the store's object-scalar codec; attachments.py itself was a listed stage module in ten stages; - congressional_district_vintage.py's contract predated main's 05ff48f05 (merged into this branch on 2026-09-11 at 9fba91070), which added the microcosm.calibrate.geography_constants import — an inherited drift the branch already carried, classified here as the same scoped-use tripwire route as the other microcosm.calibrate imports. Recomputed with the module's own _dependency_contract / _covered_imports; import_classifications is the exact union again. implementation_manifest now builds for seven of the ten stages; asec_prepared_v3, composed_asec_binding_v1 and composed_population_v1 still fail because they name a resource, asec_current_money_engine_defaults_v1.json, that exists neither on this branch nor on main (pre-existing; reported, not invented). Co-Authored-By: Claude Fable 5.1 --- .../graph_implementation_inventory.json | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index 2bc25474c..b769a9365 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -474,11 +474,12 @@ }, "microcosm.build/us_runtime/congressional_district_vintage.py": { "imports": [ + "microcosm.calibrate.geography_constants", "numpy", "pandas" ], "resource_accesses_sha256": "cba8c434f995a8a867f6b67951909b2038954efc13cf25c04ea958ffac6a8f61", - "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + "unbound_uses_sha256": "d49a89e818222a896afaaed38a0f448e33b38a5656f2eb9591e1bd9195eed687" }, "microcosm.build/us_runtime/cps_carried.py": { "imports": [ @@ -1115,18 +1116,6 @@ "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, - "microcosm.graph/attachments.py": { - "imports": [ - "microcosm.frame", - "microcosm.graph.canonical", - "microcosm.graph.decl", - "microcosm.graph.manifest", - "microcosm.graph.population", - "microcosm.graph.store" - ], - "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", - "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" - }, "microcosm.graph/availability.py": { "imports": [ "microcosm.graph.decl", @@ -1167,7 +1156,6 @@ "microcosm.frame", "microcosm.graph", "microcosm.graph.artifact_edges", - "microcosm.graph.attachments", "microcosm.graph.availability", "microcosm.graph.canonical", "microcosm.graph.codecs", @@ -1210,14 +1198,13 @@ "microcosm.graph.decl", "microcosm.graph.kernel" ], - "resource_accesses_sha256": "5e91517c603797fcf9131c3cffd0119c3226dd706f2c219658ffd571c5aa893c", + "resource_accesses_sha256": "5950b5602a721f5334ac91d4027a61707cf58600a827a6f1a4e3b73e690f9748", "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, "microcosm.graph/manifest.py": { "imports": [ "microcosm.frame", "microcosm.graph.artifact_edges", - "microcosm.graph.attachments", "microcosm.graph.availability", "microcosm.graph.canonical", "microcosm.graph.decl", @@ -1236,6 +1223,7 @@ "microcosm.graph.canonical", "microcosm.graph.decl", "microcosm.graph.kernel", + "microcosm.graph.store", "numpy", "pandas" ], @@ -1502,6 +1490,7 @@ "microcosm.build.us_runtime.weeks_unemployed": "live registry projection; other shared-module routes inactive", "microcosm.build.us_runtime.wic_claim": "live registry projection; other shared-module routes inactive", "microcosm.build.us_runtime.workers_compensation": "live registry projection; other shared-module routes inactive", + "microcosm.calibrate.geography_constants": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", "microcosm.calibrate.registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", "microcosm.calibrate.solve": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", "microcosm.fit": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", @@ -1517,7 +1506,6 @@ "microcosm.frame.weights": "whole-module in explicit stages", "microcosm.graph": "whole-module in explicit stages", "microcosm.graph.artifact_edges": "whole-module in explicit stages", - "microcosm.graph.attachments": "whole-module in explicit stages", "microcosm.graph.availability": "whole-module in explicit stages", "microcosm.graph.canonical": "whole-module in explicit stages", "microcosm.graph.codecs": "whole-module in explicit stages", @@ -1581,7 +1569,6 @@ "microcosm.graph": [ "__init__.py", "artifact_edges.py", - "attachments.py", "availability.py", "canonical.py", "codecs.py", @@ -1594,7 +1581,6 @@ "manifest.py", "population.py", "randomness.py", - "schema.py", "serialize.py", "store.py", "view.py" @@ -1665,7 +1651,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -1739,7 +1724,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -1811,7 +1795,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -1896,7 +1879,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -1972,7 +1954,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -2039,7 +2020,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -2149,7 +2129,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -2270,7 +2249,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -2390,7 +2368,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", @@ -2464,7 +2441,6 @@ "microcosm.frame/weights.py", "microcosm.graph/__init__.py", "microcosm.graph/artifact_edges.py", - "microcosm.graph/attachments.py", "microcosm.graph/availability.py", "microcosm.graph/canonical.py", "microcosm.graph/codecs.py", From 80fef2217beddea06d1f5d85a96003594309d2d5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:42:31 -0400 Subject: [PATCH 099/255] Record the #893 reconciliation report and layer map (draft; build battery pending) Co-Authored-By: Claude Fable 5.1 --- ...econciliation-amendments-19-20-20260912.md | 663 ++++++++++++++++++ 1 file changed, 663 insertions(+) create mode 100644 experiments/893-reconciliation-amendments-19-20-20260912.md diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md new file mode 100644 index 000000000..b910ebbba --- /dev/null +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -0,0 +1,663 @@ +# #893 reconciliation to main's amended graph interface — lane report + +Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, +branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is +`microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no +new branch, no stash, `uv.lock` untouched. Every commit below is on the +checked-out branch. + +## 1. Outcome + +- `origin/main` (`0c3f4f651`, amendment 19) and + `origin/amend-keyed-seed-and-uniform-draws` (`23ba24770`, amendment 20 / PR + #912) are merged. **#912 was merged from its branch head**, so once it lands + the dispatcher should re-run `git merge origin/main`; for the graph and fit + packages, `docs/graph-interface.lock` and `docs/graph-acceptance.md` that is + expected to be a no-op (this tree already carries `23ba24770`'s bytes for all + of them, verified with `git diff --quiet 23ba24770 HEAD -- `). +- Every frozen or amended file equals `23ba24770` byte for byte: `decl.py`, + `kernel.py`, `keys.py`, `serialize.py`, `view.py`, `artifact_edges.py`, + `randomness.py`, `population.py`, `fit/qrf.py`, the lock, the charter, + `test_graph_kernel_contract.py`, and every `test_acceptance_*` file. The + interface-lock test passes (it is part of the 518-test graph run below). +- Four not-yet-amended graph pieces the US runtime consumes were re-applied + as separate named commits on top of main's files; four pieces with no + consumer were dropped (section 3). +- The five graph tests the brief named are green; the whole graph package is + green (518 passed); the fit package is green (191 passed); the spec-engine + check, `ci_test_groups --verify` and ruff are clean. Build-side results are + in section 5. +- The layer map (section 6) partitions the branch's remaining 407-file diff + against `origin/main` into the eleven staged layers, the later closure and + post-staging work, and the graph pieces, with per-layer file lists, test + files, import dependencies, and a proposed landing order. Section 7 has a + draft charter amendment entry per kept graph piece. Section 8 lists what + needs Max. + +Lane commits (newest first): + +``` +07566eb17 Re-pin the US implementation inventory for the reconciled graph package +c704a1be6 Pin that unreached propagates through a structural node and its version +2e1857a4b Bring the #893 reconciliation journal up to the piece commits +a16eeacf8 Re-apply the private population observer on run_graph +d1019762b Re-apply executor execution states: gate exceptions leave their consumers unreached +db1b7821a Re-apply complete Frame metadata storage and non-finite JSON refusal in the store +752ab840f Re-apply the raw-byte source codec on main's graph package +cff8fbf32 Re-pin the calibrate and simulate H1 parity cases the branch's kernels move +dc621c14c Re-pin the seed protocol and compiled seed map digests on the merged tree +051357909 Merge origin/amend-keyed-seed-and-uniform-draws (23ba24770, amendment 20) +3010b7788 Merge origin/main (0c3f4f651, amendment 19) into the #893 integration branch +3ad1b1aae Open the #893 reconciliation lane journal +``` + +## 2. The two merges and the per-file resolution + +Baseline before any change (`uv run --no-sync pytest packages/microcosm-graph/tests`): +5 failed, 362 passed, exit 1 — exactly the five the brief names +(`test_acceptance_b_ownership::test_b2_executor_enforces_ownership`, +`test_acceptance_h_parity::test_h1_kernel_parity`, +`test_graph_executor::test_fit_qrf_tolerance_source_hash_pin_is_current`, +`test_graph_kernel_contract::test_context_numerics_default_empty_and_carry_scopes`, +`test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes`). + +### Merge 1: `git merge origin/main` (commit `3010b7788`) + +Git reported 8 content conflicts plus one in `PROGRESS.md`; `store.py`, +`__init__.py` and `PROGRESS.md` auto-merged. Resolution: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/{artifact_edges,decl,executor,kernel,keys,manifest,serialize,view}.py` (conflicts) | main's bytes (`git checkout origin/main -- packages/microcosm-graph`) | +| `packages/microcosm-graph/src/microcosm/graph/{store,__init__}.py` (auto-merged) | main's bytes, same checkout; the branch's hunks return as pieces A and B | +| `packages/microcosm-graph/src/microcosm/graph/{attachments,availability,schema}.py` (branch-only) | removed in the merge (`git rm`); `availability.py` returns as piece C, the other two are dropped | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (branch-only until amendment 20) | kept as the branch had it; replaced by `23ba24770`'s in merge 2 | +| `packages/microcosm-graph/tests/*` | main's bytes; the branch's copies were strict subsets of main's (numstat showed only removals), so this also dropped nothing of the branch's; `test_frame_metadata_store.py` and `test_lazy_snapshot_metadata_integration.py` (branch-only) removed here, the first returns with piece B, the second is dropped | +| `packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json` | main's (consistent with fit/qrf.py being main's in this merge) | +| `packages/microcosm-fit/src/microcosm/fit/qrf.py` (auto-merged to the branch's) | main's bytes; the branch's `predict_from_uniforms` returns with amendment 20 in merge 2 (the branch's qrf.py was byte-identical to `23ba24770`'s) | +| `packages/microcosm-fit/src/microcosm/fit/{_graph_legacy_apply,_graph_legacy_qrf,graph_legacy_apply,graph_legacy_apply_matrix,graph_legacy_qrf,graph_legacy_train,model_input,qrf_target}.py` (branch-only) | kept: not amended content, consumed by the US runtime (layer 01) | +| `docs/graph-interface.lock`, `docs/graph-acceptance.md` | main's bytes | +| `PROGRESS.md` | both sides kept: this lane's section, then main's Amendment 19 and #907 sections, then the branch's history | +| everything else | ordinary auto-merge | + +### Merge 2: `git merge 23ba24770` (commit `051357909`) + +Six conflicts: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (add/add) | `23ba24770`'s (adds the signed-zero normalisation the branch lacked) | +| `packages/microcosm-fit/tests/test_qrf_stateless.py` (add/add) | `23ba24770`'s (182 lines; the branch's 159-line copy was a strict subset) | +| `docs/evidence/spec-engine/us-f0-coverage.json` | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` (seed digests) | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/tests/test_spec_engine_country_bundles.py` (AM/BE/UK digests) | `23ba24770`'s; see section 5 for the merged-tree check | +| `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` (pool tool `spec_sha256`) | `23ba24770`'s; see section 5 | + +After merge 2 the graph package, fit sources, lock and charter were +byte-identical to `23ba24770` except the eight branch-only fit modules above; +`docs/graph-interface.lock` matched `shasum -a 256` of the two frozen files. + +## 3. The not-yet-amended graph pieces + +Consumers were found by an AST scan of every `.py` under +`packages/microcosm-{build,calibrate,fit,frame,data}` and `tools/` for names +imported from `microcosm.graph.*` that `23ba24770`'s graph package does not +define, plus greps for the keyword-level uses (`_population_observer=`, +`population_retention=`, `emit_validation_artifact=`). + +### Kept (each re-applied on main's file as its own commit, with tests) + +| Piece | Commit | Consumers outside the graph package | Graph-level tests | +| --- | --- | --- | --- | +| A. `codecs.py` raw-byte mode: `SourceBytesCodec`, `register_bytes`/`load_bytes`, `raw-bytes-v1`, `RAW_BYTES_MAX_BYTES`, `load_source_bytes` (+ two `__init__` exports) | `752ab840f` | 13 modules: `graph_atomic_geography`, `graph_geography`, `graph_sources`, `graph_atomic_survey_population`, `graph_puf55_canonical_donor`, `puf_raw_source`, `puf_monetary_source`, `puf_monetary_agi_projection`, `puf_full_source_graph`, `atomic_block_sources`, `atomic_block_api_sources`, `survey_atomic_geography`, `puf55_survey_recipients`; 8 US test files | the branch had none; 4 added to `test_graph_codecs.py` (8 passed) | +| B. `store.py` Frame-metadata storage (`microcosm-graph-frame-v2`, `_encode/_decode_frame_metadata`, `frame_metadata_sha256`, `_put(validate_existing=…)`) and the non-finite JSON decode hooks | `db1b7821a` | `survey_population_replay.same_replayed_frame`, `population_input_coverage`, `graph_full_puf_enrichment` (all call `_encode_frame_metadata`), and every US stage relying on `Frame.metadata` surviving a store round trip | `test_frame_metadata_store.py` (the branch's file, 19 collected tests) | +| C. `availability.py` + executor/manifest execution states: `gate_exception`, `unreached`, `blocked_by`, cache-record schema 3, manifest schema 4, refusal of kernel-authored execution metadata; replaces amendment 19's refusal of a gate declaring a typed output | `d1019762b` | `graph_atomic_survey_clone` (`atomic_geography_nodes(..., emit_validation_artifact=True)` makes the geography gate declare a typed output, which main's executor refuses outright), its consumers `survey_age_calibration`, `graph_current_survey_predictors`, `survey_origin_budget`; `population_input_coverage` (`execution_state` on every producer receipt); build test `test_atomic_geography.py` parametrises the declaration | the branch had none; main's `test_a_gate_kernel_may_not_declare_a_typed_artifact_output` replaced by 6 tests in `test_graph_executor.py` (five in `d1019762b`, the structural-propagation one in `c704a1be6`) | +| D. `run_graph(_population_observer=…)` | `a16eeacf8` | `survey_age_calibration`, `graph_atomic_survey_financial`, `graph_survey_population`, `graph_atomic_survey_population`; 8 US test files | the branch had none; 1 added | + +What was deliberately not re-applied inside those files: the branch's +docstring deletions on the amended surface; its `_opaque_artifact_key` +inline hash (main's `graph_keys.opaque_artifact_key` is the same +derivation); its `KernelContext` argument order (frozen file); its eager +artifact payload reads on cache hits (main authenticates edges without +reading, `9018c4420`); its `StoreMiss`-on-hit for a missing declared artifact +(main's `5c4a8efde` decision, `StoreCorrupt`); the removal of main's #907 +datetime64/timedelta64 object-leaf refusal in `store.py`; and main's +`_validate_typed_ancestry` gate-ancestry shape check and the +`NodeRejectedError`→`ValueError` conversion (`402d9a631`, `7d45c32e5`), which +the branch's older copy lacked. + +### Dropped (no consumer left) + +| Piece | Why dropped | +| --- | --- | +| `attachments.py`, `_PopulationRetention`, `_LazyPopulations`, `run_graph(population_retention="lazy")`, `PopulationView.__slots__ = ("__weakref__",)` (staged as layer 08 GRAPH-ATTACHMENT-METADATA) | no call site outside the graph package passes `population_retention`; its only consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, dropped with it. It is a memory optimisation for long graphs (the native pilot's 11.5 GB peak is in the PR body); if it is wanted it is a self-contained later PR on top of piece B, which it imports. | +| `schema.py` (`graph_schema`, 316 lines) | imported by nothing in the repository (the `graph_schema_version` hits in the spec bundle are unrelated). | +| `keys.py` `_stream_file` chunked source hashing | no consumer and no test; it changes no identity (same digest sequence, it only bounds memory and refuses a file that changes while hashed). Main's `keys.py` is kept byte-identical. Candidate for a small standalone PR if large-source memory matters. | +| the `_write_node` per-coordinate memory refactor and the `del series` | no consumer; behaviour-preserving; main's amended `_write_node` kept. | + +## 4. Identities re-pinned on the merged tree, and why + +- **Seed protocol and compiled seed map digests** (`inventory_coverage.py` + `EXPECTED_HASHES`, `docs/evidence/spec-engine/us-f0-coverage.json`; + commit `dc621c14c`). Both hash the exact source bytes of the attested kernel + modules in `spec_engine/seeds.py` (no `microcosm.graph` module is on that + list, so the graph pieces do not move them). The merged tree carries the + branch's `acs_transfer` and `housing_inputs` changes and, through amendment + 20, the same `fit.qrf` bytes the branch already carried, so the actual + digests are the values the branch pinned in `1734b9e90` + (`a775cccd…` / `c15bff65…`), not `23ba24770`'s (computed on a tree without + the two US kernel changes). Computed directly (`spec.seed_protocol.implementation_sha256`, + `sha256_json(compiled.seed_stream_map.to_wire())`), then the report was + regenerated with `tools/spec_engine_coverage.py` and `--check` exits 0. +- **H1 parity pins for `calibrate` and `simulate`** (commit `cff8fbf32`). + Six graph tests were red on the merged tree, all on these two cases. + `CalibrateKernel.implementation_hash` binds `microcosm.calibrate`'s solve and + diagnostics modules, which the branch's grouped-bound solver path changes + (layer 05); `SimulateRulesKernel.implementation_hash` binds + `microcosm.frame.bundle`, to which the branch adds `Frame.__reduce__` (layer + 01). Re-recorded with `tools/graph_parity_repin.py calibrate` and + `… simulate`, which keep every pinned platform (both cases pin only + `arm64/darwin/py3.14`) and leave `direct.csv` untouched. The `fit.qrf` pin + came from `23ba24770` and needed no move. +- **The US implementation inventory** (`us_runtime/graph_implementation_inventory.json`; + commit `07566eb17`). This reviewed dependency fence is checked by every US + source stage before it runs (`graph_implementation.implementation_manifest`), + and its refusal surfaces as `PREPARATION_ISSUANCE_REFUSED` (the preparation + wraps every exception; the true cause was recovered from the exception's + `__context__`). Three entries no longer held: the `microcosm.graph` roster + listed the dropped `attachments.py` and `schema.py` (39 survey tests red on + that alone); the contracts for `executor.py`, `manifest.py`, `keys.py` and + `population.py` reflected the branch's versions rather than main's plus the + kept pieces, and `attachments.py` was a listed stage module in ten stages; + and `congressional_district_vintage.py`'s contract predated main's + `05ff48f05` (merged into the branch on 2026-09-11), which added the + `microcosm.calibrate.geography_constants` import — an inherited drift the + branch already carried at `069d5ed9a`, verified by recomputing the + pre-merge tree's contracts against its own inventory. Recomputed with the + module's own `_dependency_contract` / `_covered_imports`; the import + classification set is the exact union again. Seven of the ten stages now + build their manifest; three cannot (section 8.5). +- **Country-bundle digests and the pool tool's `spec_sha256`**: resolved to + `23ba24770`'s values in merge 2; their merged-tree check is in section 5. + +## 5. Commands, exit codes, counts + +All from the worktree root with the locked engine environment +(`uv sync --all-packages --locked --extra us --extra uk` → exit 0 before any +change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. + +| Stage | Command | Result | +| --- | --- | --- | +| baseline (`069d5ed9a`) | `pytest packages/microcosm-graph/tests` | 5 failed, 362 passed, exit 1 | +| baseline | `python tools/ci_test_groups.py --verify` | exit 0 | +| after both merges (`051357909`) | `pytest packages/microcosm-graph/tests` | 6 failed, 485 passed, exit 1 (calibrate/simulate parity pins; section 4) | +| after both merges | `pytest packages/microcosm-fit/tests` | 191 passed, exit 0 | +| after both merges | `python tools/spec_engine_coverage.py --check` | exit 1 (`seed protocol content digest differs`, `compiled seed map digest differs`) | +| after both merges | `python tools/ci_test_groups.py --verify` | `verification=ok`, exit 0 | +| after both merges | `ruff check .` | All checks passed, exit 0 | +| seed re-pin (`dc621c14c`) | `python tools/spec_engine_coverage.py` then `--check` | exit 0 / exit 0 | +| parity re-pin (`cff8fbf32`) | `python tools/graph_parity_repin.py calibrate`; `… simulate` | exit 0 / exit 0 | +| parity re-pin | `pytest test_acceptance_h_parity.py test_graph_parity_pins.py test_graph_parity_repin.py test_graph_serialize.py` | 27 passed, exit 0 | +| piece A (`752ab840f`) | `pytest packages/microcosm-graph/tests/test_graph_codecs.py` | 8 passed, exit 0 | +| piece B (`db1b7821a`) | `pytest test_frame_metadata_store.py test_graph_store.py test_graph_population.py test_graph_executor.py test_graph_manifest.py` | 274 passed, exit 0 | +| piece C (`d1019762b`) | `pytest packages/microcosm-graph/tests` (whole package, includes the interface-lock test) | **518 passed, exit 0** | +| piece C | `ruff check .`; `ruff format --check packages/microcosm-graph` | exit 0 / exit 0 | +| piece D (`a16eeacf8`) | `pytest test_graph_executor.py test_acceptance_b_ownership.py test_acceptance_f_gates.py` | 94 passed, exit 0 | +| structural test (`c704a1be6`) | `pytest packages/microcosm-graph/tests/test_graph_executor.py` | 82 passed, exit 0 | +| inventory re-pin (`07566eb17`) | `pytest test_us_survey_origin_budget.py::test_actual_initial_budget_and_zero test_us_survey_age_calibration_run.py::test_actual_seven_node_calibration_preserves_full_population` (two of the 39 formerly refused) | 2 passed, exit 0 | +| final tree | `python tools/ci_test_groups.py --verify` | exit 0 | +| final tree | `python tools/spec_engine_coverage.py --check` | exit 0 | + +Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: + +_(battery running at the time of this draft; the final table replaces this line)_ + +## 6. Layer map: landing #893 as sequential PRs + +Basis: `git diff --name-status origin/main HEAD` = 407 files (340 added, 67 +modified, 0 deleted), each attributed to the first branch commit that touched +it (`git log --name-only --no-merges 3094bfe84..HEAD`), with the eleven +`Stage integration layer NN` commits mapped to their layer names, the later +staging commits to layers 12–14, the sixty post-staging commits of 10–12 +September grouped by subject (P1–P6), and the graph pieces of section 3 as +their own layer. Import dependencies come from an AST scan of every changed +`.py` file resolved against that attribution (`microcosm.*` imports only). +The scripts and their JSON output are in this lane's scratchpad +(`layers.py`, `layers2.py`, `layers.json`, `per-file-deps.json`). + +Two corrections to the staged layer names as they sit in git history: + +- Layer 08 GRAPH-ATTACHMENT-METADATA is empty in the map: its only file, + `attachments.py`, is dropped (section 3). +- Layer 09 F-JOINT-GEOGRAPHY-GATE has no file of its own: its 27-line change + lives in `graph_geography.py`, which layer 01 created, so it lands with + layer 01. +- Nineteen files differ from `origin/main` only because `origin/main` does not + yet carry amendment 20 (`kernel.py`, `randomness.py`, `fit/qrf.py`, + `test_graph_kernel_contract.py`, `test_graph_parity_*`, + `test_graph_randomness.py`, `tools/graph_parity_repin.py`, the three + `pins.json`, the lock, the charter, the amendment-20 journal/receipts/ + changelog fragment) or were re-pinned by this lane. They vanish from the diff + when #912 lands and are not #893 content; they are listed under + "already on main's way" below and belong to no PR. + +### Dependency facts the order rests on + +Three kinds of link were measured, strongest first: + +1. **Hard**: a file imports a module that does not exist on `origin/main` + (a file the branch adds). These force order. +2. **Name-hard**: a file imports, from a module main already has, a *name* + that main's version of that module does not define (checked by parsing + `git show origin/main:`). Only three survive after discarding + submodule imports that merely look like names: `GroupedUpperBounds` + (layer 03's two-line re-export in `calibrate/__init__.py` of layer 01's + `group_bounds.py`; imported by layer 12's and 13's tests), + `decode_us_puma_ladder` (layer 06's `puma_ladder.py`; imported by layer + 13's `test_us_geography_integration.py`), and the graph pieces' names + (`load_source_bytes`, `load_raw_bytes`, `RAW_BYTES_MAX_BYTES`, + `_encode_frame_metadata`, `execution_state`). +3. **Soft**: a file imports a module main already has that the branch also + modifies (`frame_checkpoint`, `asec_checkpoint`, `operator_boundary`, + `us_runtime/__init__`, `calibrate/__init__`, `solve.py`, `bundle.py`, + `variable_labels.py`, and amendment 20's `qrf.py`/`kernel.py`). These do + not force order by themselves; each PR's own test run is what proves the + modified behaviour is or is not needed. + +Hard links between layers (source and tests): + +| Layer | hard on | of which tests only | +| --- | --- | --- | +| 01 SAFE-ADDITIVE | nothing | — | +| 03 ACCEPTED-SHARED-RESTORE | 01 (`group_bounds`, `reported_coverage_source`, `_person_signal_summary`, `operator_column_contracts`, `table_identity`), P3 (`asec_raw_stage_v4`) | P3 link is `test_us_asec_checkpoint.py` only | +| 04, 06 | nothing hard (soft on 01/03) | — | +| 05 SOLVE-MERGE-PROPOSAL | 01 (`group_bounds`) | — | +| 07 F-CATALOGUE-OPTIMIZATION | 01 (`survey_population_preparation`) | all three files are tests | +| 10 SOURCE-CLOSURE | G1 (soft) | — | +| 11 PLACEMENT-ADDITIONS | 01 (the eight fit modules, `full_puf_enrichment`, `graph_sources`, `survey_population_replay`), G1, G2 | — | +| 12 ORDINARY-CLOSURE | 01 (fit modules), 03 (`GroupedUpperBounds`) | all files are tests/fixtures | +| 13 INTEGRATION-REGRESSIONS | 01 (`graph_geography`), 03 (`GroupedUpperBounds`), 06 (`decode_us_puma_ladder`) | all three are tests | +| P1 POST-CLONE-ATOMIC-GEOGRAPHY | 01 (`randomness` = amendment 20, fit modules, `graph_combined_clone`, `graph_sources`, `graph_survey_population`, `survey_*`), G1, G3 (the gate's declared artifact), P2 | P2 link is two tests (`test_us_current_survey_predictor_demographics.py`, `test_us_survey_origin_budget_predictor_compatibility.py`) | +| P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR | 01 (`model_input`, `_graph_legacy_qrf`, `graph_legacy_*`, and eleven test-side modules), 11 (`graph_full_puf_enrichment`, two tests), P1 (`graph_atomic_survey_financial`, one test), G1 | 11 and P1 links are tests only | +| P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET | 01 (`reported_coverage_source` from `asec_raw_stage_v4`; six test-side modules), G2, G3 (`population_input_coverage`), P2 (`asec_demographic_source`) | P2 link is `test_us_asec_demographic_source.py` only | +| P4 | nothing hard (soft on 03) | — | +| P5 | nothing hard (soft on 01/03/04/06) | — | +| P6 UK-ADAPTER-DOCS-AND-RECORDS | P1 (`atomic_geography`) | — | + +Two cycles exist and both are test-only: P1↔P2 (three tests) and 03↔P3 +(one test). Every source-level dependency is acyclic. + +### Landing order + +1. **PR-G1 raw-byte codec** — piece A (`codecs.py`, `__init__.py`, + `test_graph_codecs.py`). No dependency. Charter entry 7.1. +2. **PR-G2 Frame metadata storage** — piece B (`store.py`, + `test_frame_metadata_store.py`, + `changelog.d/20260906-frame-metadata-storage.fixed.md`). No dependency. + Charter entry 7.2. +3. **PR-G3 execution states** — piece C (`availability.py`, `executor.py`, + `manifest.py`, `test_graph_executor.py`). No code dependency, but it + supersedes an amendment-19 ruling: **needs Max's decision (section 8) + before it is opened.** Charter entry 7.3. +4. **PR-G4 population observer** — piece D (`executor.py`, + `test_graph_executor.py`). Stacks on G3 only because both edit + `executor.py`; G3+G4 can be one PR. Charter entry 7.4. +5. **PR-1 SAFE-ADDITIVE** — all of layer 01: 101 non-test files (100 modules + plus `graph_implementation_inventory.json`; `randomness.py` is excluded, it + is amendment 20) and its 26 test files, plus layer 03's two-line + `calibrate/__init__.py` re-export of `GroupedUpperBounds` (it belongs with + `group_bounds.py`) and layer 09's 27 lines (already inside + `graph_geography.py`), and `graph_implementation_inventory.json`, which + pins the `microcosm.graph` roster and module contracts and therefore has + to be re-pinned against whatever main's graph package is when PR-1 opens + (`07566eb17` shows the procedure). Hard on G1, G2 and G4 only (`_population_observer` + in `survey_age_calibration` and `graph_survey_population`). Its 21 soft + links are listed in the scratchpad `pr1-refined.json`; the PR's gate is + its own 26 test files on main + G1–G4 + PR-1. +6. **PR-2 US runtime restore** — layers 03 (without the `__init__` re-export + moved to PR-1, and with `asec_raw_stage_v4.py` moved in from P3 so + `test_us_asec_checkpoint.py` runs), 04 (`puf_support.py`), 05 + (`calibrate/solve.py`), 06 (`us_runtime/__init__.py`, `puma_ladder*.py`, + `tools/build_us_puma_ladder_artifact.py`, six tests), 12's calibrate + tests and fixture, 13's three regressions and `docs/us-launch-review.md`, + 14 (`dfa7f872c`), and the two re-pins that these files move: the + calibrate H1 pin (`solve.py`) and the seed digests (`acs_transfer.py`, + `housing_inputs.py`). 03/04/05/06 touch disjoint files and could be four + PRs (05 first, since only it is hard on 01); they are one here because + their tests import across all four. +7. **PR-3 catalogue, source closure, placement** — layers 07 (three digest + tests + fragment), 10 (`graph_acs_housing_universe.py`, eight JSON source + definitions), 11 (`graph_full_puf_enrichment.py` + test), and 12's fit + tests (`test_graph_legacy_*`, `test_qrf_target.py`, + `legacy-slice-apply-v1.json`). Hard on PR-1 and G1/G2. +8. **PR-4 calibration diagnostics** — P4 (`variable_labels.py`, its test, + fragments, experiments). Soft on PR-2. +9. **PR-5 survey population, catalogue and budget** — P3 minus + `asec_raw_stage_v4.py`: `common_frame_export_contract`, + `input_coverage_profile`, `population_input_coverage`, + `frame/bundle.py`'s `__reduce__` (carries the simulate H1 re-pin), ten + tests including `test_us_spine_blindness.py`; + `test_us_asec_demographic_source.py` rides with PR-7 instead. Hard on + PR-1, G2, G3. +10. **PR-6 post-clone atomic geography** — P1 (`atomic_geography.py`, + `graph_atomic_geography.py`, `atomic_block_*`, `graph_atomic_survey_*`, + `survey_atomic_geography.py`, `current_survey_geography.py`, + `tools/ci_test_groups.py`, fourteen tests); its two P2-importing tests + ride with PR-7. Hard on PR-1, G1, G3. +11. **PR-7 PUF55 two-route and canonical donor** — P2 (nineteen modules, + twenty-one tests plus the three deferred from PR-5/PR-6, PUF docs and + growth evidence, `pyproject.toml`'s `pythonpath` line). Hard on PR-1, + PR-3 (11), PR-5, PR-6. +12. **PR-8 CI, seed-identity diagnostics and spine blindness** — P5 + (`.github/workflows/test.yml`, `tools/spec_seed_identity_diagnostics.py`, + the seven `test_spec_seed_identity_*` files, `CLAUDE.md`, + `test_spec_engine_loader.py`, `test_us_puf_support.py`, + `test_us_stacked_spine.py`, the AM/BE/UK bundle digests). Soft on PR-2 + and PR-5; last because the diagnostic's roster names modules from every + earlier PR. +13. **PR-9 UK adapter, docs and records** — P6 (`uk_runtime/atomic_area_support.py`, + `atomic_household_identity.py`, two tests, `.gitattributes`, records). + Hard on PR-6. + +Every `experiments/` record and `changelog.d/` fragment rides with the PR +whose code it describes (listed per layer below). The "already on main's +way" files (amendment 20, listed at the top of this section) belong to no PR. + +### Per-layer file lists + +#### 00 JOURNAL (1 non-test files, 0 test files) + +- `./`: `PROGRESS.md` + +#### 01 SAFE-ADDITIVE (101 non-test files, 26 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py` +- `packages/microcosm-build/src/microcosm/build/cd_benchmark/`: `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_housing_universe_source.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_2024_native_population.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_current_money_units.py`, `asec_engine_evaluation.py`, `asec_household_coverage_fields.py`, `asec_household_observations.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_person_income_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_native_origin_implementation.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_price_baseline.py`, `puf_raw_source.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_calibration.py`, `survey_age_sources.py`, `survey_calibration_diagnostics.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_origin_budget.py`, `survey_population_domains.py`, `survey_population_preparation.py`, `survey_population_replay.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `group_bounds.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py` +- `packages/microcosm-frame/src/microcosm/frame/adapters/`: `_policyengine_us_source_index.py`, `policyengine_us.py` +- `packages/microcosm-graph/src/microcosm/graph/`: `randomness.py` +- tests: `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` +- source imports from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece), P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) + +#### 02 GRAPH-RESTORE (1 non-test files, 0 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `kernel.py` + +#### 03 ACCEPTED-SHARED-RESTORE (20 non-test files, 2 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `frame_checkpoint.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_inputs.py`, `acs_pums.py`, `acs_transfer.py`, `asec_checkpoint.py`, `congressional_district_vintage.py`, `cps_carried.py`, `eligibility_inputs.py`, `hours_worked.py`, `housing_inputs.py`, `multispine_pool.py`, `operator_boundary.py`, `prior_year_income.py`, `puf_capital_gains_tail.py`, `qbi_inputs.py`, `relationship_inputs.py`, `spine_assembly.py`, `stacked_spine.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `__init__.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `qrf.py` +- tests: `test_us_acs_pums.py`, `test_us_asec_checkpoint.py` +- source imports from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### 04 PUF-SUPPORT-MERGE (1 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `puf_support.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE + +#### 05 SOLVE-MERGE-PROPOSAL (1 non-test files, 0 test files) + +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `solve.py` +- source imports from: 01 SAFE-ADDITIVE + +#### 06 J-GRAPH-COMPATIBILITY (4 non-test files, 6 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `__init__.py`, `puma_ladder.py`, `puma_ladder_sources.py` +- `tools/`: `build_us_puma_ladder_artifact.py` +- tests: `test_us_acs_multispine.py`, `test_us_acs_multispine_legacy_builder.py`, `test_us_base_pool.py`, `test_us_multispine_pool_tool.py`, `test_us_puma_ladder.py`, `test_us_puma_ladder_sources.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE + +#### 07 F-CATALOGUE-OPTIMIZATION (1 non-test files, 3 test files) + +- `changelog.d/`: `us-survey-catalogue-digest.changed.md` +- tests: `test_us_survey_catalogue_digest.py`, `test_us_survey_catalogue_digest_authority.py`, `test_us_survey_catalogue_digest_fast_path.py` +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY + +#### 10 SOURCE-CLOSURE (9 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_2024_housing_universe.json`, `asec_current_money_consumers_v1.json`, `asec_current_money_domains_v1.json`, `asec_current_money_price_basis_v1.json`, `graph_acs_housing_universe.py`, `native_origin_graph_inventory.json`, `puf_2015_monetary_agi_source_projection.json`, `puf_2015_monetary_source_projection.json`, `puf_2015_raw_source_definition.json` +- source imports from: G RAW-BYTES-CODEC (graph piece) + +#### 11 PLACEMENT-ADDITIONS (1 non-test files, 1 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `graph_full_puf_enrichment.py` +- tests: `test_us_graph_full_puf_enrichment.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### 12 ORDINARY-CLOSURE (0 non-test files, 8 test files) + +- tests: `legacy_prepatch.json`, `test_group_bounds.py`, `test_grouped_fixed_support.py`, `legacy-slice-apply-v1.json`, `test_graph_legacy_matrix.py`, `test_graph_legacy_qrf.py`, `test_qrf_stateless.py`, `test_qrf_target.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, G RAW-BYTES-CODEC (graph piece) + +#### 13 INTEGRATION-REGRESSIONS (2 non-test files, 3 test files) + +- `changelog.d/`: `us-launch-integration.fixed.md` +- `docs/`: `us-launch-review.md` +- tests: `test_us_geography_integration.py`, `test_us_runtime_facade_union.py`, `test_solve_us_launch_integration.py` +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### Already on main's way (amendment 20 / this lane's re-pins; not #893 content) (10 non-test files, 9 test files) + +- `./`: `PROGRESS-amendment-20-keyed-draws.md` +- `changelog.d/`: `amend-keyed-seed-and-uniform-draws.added.md` +- `docs/`: `graph-acceptance.md`, `graph-interface.lock` +- `experiments/`: `amendment-20-keyed-draws-receipts.md`, `us-native-atomic-financial-required-replay-20260910.json`, `us-puf-donor-boundaries-20260910.json`, `us-puf55-public-recipient-controls-20260910.json`, `us-puf55-two-route-numerical-20260910.json` +- `tools/`: `graph_parity_repin.py` +- tests: `test_us_puf55_model_boundaries.py`, `test_kernels.py`, `pins.json`, `pins.json`, `pins.json`, `test_graph_kernel_contract.py`, `test_graph_parity_pins.py`, `test_graph_parity_repin.py`, `test_graph_randomness.py` +- source imports from: G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### G RAW-BYTES-CODEC (graph piece) (2 non-test files, 1 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `__init__.py`, `codecs.py` +- tests: `test_graph_codecs.py` +- tests import from: G STORE-FRAME-METADATA (graph piece) + +#### G STORE-FRAME-METADATA (graph piece) (2 non-test files, 1 test files) + +- `changelog.d/`: `20260906-frame-metadata-storage.fixed.md` +- `packages/microcosm-graph/src/microcosm/graph/`: `store.py` +- tests: `test_frame_metadata_store.py` + +#### P1 POST-CLONE-ATOMIC-GEOGRAPHY (38 non-test files, 16 test files) + +- `changelog.d/`: `893-population-only-block-source.added.md`, `893-source-demographic-conditioning.added.md`, `893-survey-atomic-prefix.added.md`, `shared-atomic-geography.added.md`, `us-atomic-financial-composition.added.md`, `us-national-atomic-support.added.md` +- `experiments/`: `us-atomic-age-v2-1-controls-20260909.json`, `us-atomic-block-adapter-20260909.json`, `us-atomic-block-api-sources-35-controls-20260909.json`, `us-atomic-block-sources-36-controls-20260909.json`, `us-atomic-budget-semantic-8-controls-20260909.json`, `us-atomic-clone-graph-20260909.json`, `us-atomic-financial-4-controls-20260909.json`, `us-atomic-financial-composition-acceptance-20260909.json`, `us-atomic-financial-corrected-2-controls-20260909.json`, `us-atomic-national-acquisition-20260909.json`, `us-atomic-native-de-1-control-20260909.json`, `us-atomic-native-de-corrected-1-control-20260909.json`, `us-atomic-native-national-1-control-20260909.json`, `us-atomic-survey-population-controls-20260909.json`, `us-budget-predictor-compatibility-1-controls-20260909.json`, `us-financial-demographics-3-controls-20260909.json`, `us-postclone-geography-70-controls-20260910.json`, `us-puf55-native-donor-20260909.json`, `us-survey-geography-graph-controls-20260909.json`, `us-survey-geography-source-controls-20260909.json` +- `packages/microcosm-build/src/microcosm/build/`: `atomic_geography.py`, `graph_atomic_geography.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `atomic_block_api_sources.py`, `atomic_block_sources.py`, `atomic_block_support.py`, `current_survey_geography.py`, `graph_atomic_survey_clone.py`, `graph_atomic_survey_financial.py`, `graph_atomic_survey_population.py`, `graph_current_survey_geography.py`, `survey_atomic_geography.py` +- `tools/`: `ci_test_groups.py` +- tests: `test_atomic_geography.py`, `test_us_atomic_block_api_sources.py`, `test_us_atomic_block_api_sources_native_de.py`, `test_us_atomic_block_api_sources_native_national.py`, `test_us_atomic_block_sources.py`, `test_us_atomic_block_support.py`, `test_us_atomic_survey_clone_graph.py`, `test_us_current_survey_geography.py`, `test_us_current_survey_predictor_demographics.py`, `test_us_graph_atomic_survey_financial.py`, `test_us_graph_atomic_survey_population.py`, `test_us_graph_current_survey_geography.py`, `test_us_postclone_geography_identity.py`, `test_us_survey_age_calibration_atomic_geography.py`, `test_us_survey_origin_budget_atomic_geography.py`, `test_us_survey_origin_budget_predictor_compatibility.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR (51 non-test files, 21 test files) + +- `./`: `pyproject.toml` +- `changelog.d/`: `893-age-count-fixture.internal.md`, `893-diagnostic-refusal-context.internal.md`, `893-historical-zero-provenance.internal.md`, `puf55-canonical-donor.added.md`, `puf55-survey-social-security.added.md`, `survey-social-security-source.added.md`, `us-puf55-canonical-and-output-seals.added.md`, `us-puf55-survey-ss-measurement.added.md` +- `docs/`: `current-survey-puf59-progress.md`, `geography-assignment.md`, `puf2015-canonical59-and-growth.md`, `puf55-survey-ss-measurement-decision.md`, `survey-social-security-source.md`, `us-uk-release-path.md` +- `docs/evidence/puf2015-target2024/`: `GROWTH-RECIPE.json`, `INDEX-VALUES.json`, `NATIONAL-GROWTH-EXTRACT.json`, `PUBLIC-WORKBOOK-SOURCES.json` +- `experiments/`: `us-acs-compilation-cache-adoption-20260910.json`, `us-budget-detached-view-correction-20260910.json`, `us-financial-fixture-profile-20260910.json`, `us-financial-successor-positive-1-limit-stop-20260909.json`, `us-financial-successor-positive-20260910.json`, `us-financial-successor-remaining-controls-20260910.json`, `us-puf55-canonical-create-47-controls-20260910.json`, `us-puf55-numerical-output-seal-46-controls-20260910.json`, `us-puf55-population-controls-20260909.json`, `us-puf55-public-recipient-positives-20260910.json`, `us-puf55-survey-ss-measurement-31-controls-20260909.json`, `us-puf55-two-route-values-controls-20260910.json`, `us-survey-age-development-20260909.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_demographic_source.py`, `current_asec_demographics.py`, `current_social_security_source.py`, `current_survey_predictors.py`, `graph_current_survey_predictors.py`, `graph_puf55_canonical_donor.py`, `graph_puf55_survey_recipients.py`, `puf55_canonical_donor.py`, `puf55_route_finalization.py`, `puf55_survey_recipients.py`, `puf55_survey_ss_measurement.py`, `puf59_canonical.py`, `puf59_canonical_artifact.py`, `puf_full_source.py`, `puf_full_source_graph.py`, `puf_qbi_model.py`, `puf_target2024_growth.py`, `survey_financial_successor.py`, `survey_social_security.py` +- tests: `test_frame_checkpoint.py`, `test_spec_seed_diagnostic_refusal_context.py`, `test_us_acs_source_compile_cache.py`, `test_us_current_asec_demographics.py`, `test_us_current_survey_predictors.py`, `test_us_full_puf_output_profiles.py`, `test_us_graph_puf55_canonical_donor.py`, `test_us_graph_puf55_survey_recipients.py`, `test_us_graph_puf55_survey_ss.py`, `test_us_plan.py`, `test_us_puf55_route_finalization.py`, `test_us_puf55_route_numerical_finalization.py`, `test_us_puf55_survey_recipients.py`, `test_us_puf55_survey_ss_measurement.py`, `test_us_puf55_survey_ss_profile.py`, `test_us_puf59_canonical.py`, `test_us_puf_target2024_growth.py`, `test_us_runtime_import_order.py`, `test_us_survey_financial_successor.py`, `test_us_survey_origin_budget_detached_views.py`, `test_us_survey_social_security.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, 11 PLACEMENT-ADDITIONS, G RAW-BYTES-CODEC (graph piece), P1 POST-CLONE-ATOMIC-GEOGRAPHY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET (16 non-test files, 10 test files) + +- `changelog.d/`: `common-frame-export-contract.added.md`, `us-survey-copy-and-identity.fixed.md` +- `docs/`: `us-input-coverage-diagnostic.md` +- `experiments/`: `us-acs-code-memo-comparison-20260910.json`, `us-acs-code-memo-not-adopted-20260910.patch`, `us-acs-loaded-code-controls-20260910.json`, `us-common-frame-export36-20260910.json`, `us-input-coverage25-20260910.json`, `us-postclone-budget-replay-fence4-20260910.json`, `us-survey-catalogue-memo20-20260910.json`, `us-survey-numeric-diagnostics27-20260910.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_raw_stage_v4.py`, `common_frame_export_contract.py`, `input_coverage_profile.py`, `population_input_coverage.py` +- `packages/microcosm-frame/src/microcosm/frame/`: `bundle.py` +- tests: `test_us_acs_transfer.py`, `test_us_asec_catalogue_records_memo.py`, `test_us_asec_demographic_source.py`, `test_us_asec_household_coverage_fields.py`, `test_us_common_frame_export_contract.py`, `test_us_population_input_coverage.py`, `test_us_spine_blindness.py`, `test_us_survey_catalogue_immutable_memo.py`, `test_us_survey_frame_identity_encoding.py`, `test_us_survey_origin_budget_replay_producer.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS (6 non-test files, 1 test files) + +- `changelog.d/`: `us-launch-store-grouped-validation.fixed.md`, `us-survey-diagnostic-options.fixed.md` +- `experiments/`: `us-calibration-consolidation-149-20260910.json`, `us-puf55-canonical-create-failed-20260910.json`, `us-puf55-canonical-cycle39-failed-20260910.json` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `variable_labels.py` +- tests: `test_variable_labels.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE + +#### P5 CI-SEED-IDENTITY-AND-SPINE-BLINDNESS (10 non-test files, 10 test files) + +- `./`: `CLAUDE.md` +- `.github/workflows/`: `test.yml` +- `changelog.d/`: `893-diagnostic-current-lock.fixed.md`, `893-spec-identity-diagnostics.internal.md`, `893-worker-resource-trace.fixed.md` +- `docs/evidence/spec-engine/`: `us-f0-coverage.json` +- `experiments/`: `spec-seed-current-lock-20260910.json`, `us-acs-producer-profile-20260910.json` +- `packages/microcosm-build/src/microcosm/build/spec_engine/`: `inventory_coverage.py` +- `tools/`: `spec_seed_identity_diagnostics.py` +- tests: `test_spec_engine_country_bundles.py`, `test_spec_engine_loader.py`, `test_spec_seed_identity_cpu_bootstrap.py`, `test_spec_seed_identity_engine_parameter_files.py`, `test_spec_seed_identity_outer_failure.py`, `test_spec_seed_identity_owned_temp.py`, `test_spec_seed_identity_per_code_context.py`, `test_spec_seed_identity_system_metadata.py`, `test_us_puf_support.py`, `test_us_stacked_spine.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY + +#### P6 UK-ADAPTER-DOCS-AND-RECORDS (10 non-test files, 2 test files) + +- `./`: `.gitattributes` +- `changelog.d/`: `893-uk-atomic-area-and-lazy-exports.added.md` +- `docs/`: `validation-cost-and-borrow-boundaries.md` +- `experiments/`: `uk-atomic-area-lazy52-20260910.json`, `us-financial-default-2-controls-20260909.json`, `us-native-atomic-financial-cold-20260910.json`, `us-native-postclone-financial-incomplete-20260910.json` +- `packages/microcosm-build/src/microcosm/build/uk_runtime/`: `__init__.py`, `atomic_area_support.py`, `atomic_household_identity.py` +- tests: `test_uk_atomic_area_support.py`, `test_uk_runtime_lazy_exports.py` +- source imports from: P1 POST-CLONE-ATOMIC-GEOGRAPHY +- tests import from: P1 POST-CLONE-ATOMIC-GEOGRAPHY + +**Layer-01 modules with no cross-layer import at all (53; `randomness.py` among them is amendment 20, not #893 content):** `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py`, `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py`, `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_units.py`, `asec_household_coverage_fields.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_native_origin_implementation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_price_baseline.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_sources.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_population_domains.py`, `group_bounds.py`, `_policyengine_us_source_index.py`, `randomness.py` + + +**Layer-01 modules with a cross-layer import (48; 27 of them only on the graph pieces G1/G2, the other 21 soft on modified modules of 03/05/06/P3/P4 — see `pr1-refined.json`):** `acs_housing_universe_source.py`, `asec_2024_native_population.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_engine_evaluation.py`, `asec_household_observations.py`, `asec_person_income_source.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_raw_source.py`, `survey_age_calibration.py`, `survey_calibration_diagnostics.py`, `survey_origin_budget.py`, `survey_population_preparation.py`, `survey_population_replay.py`, `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py`, `policyengine_us.py` + +**Layer-01 tests (all ride with PR-1; 26):** `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` + +## 7. Draft charter amendment entries (one per kept graph piece) + +Written in the style of `docs/graph-acceptance.md` "Interface freeze". None of +the four touches `decl.py` or `kernel.py`, so none re-records the lock; they +are offered as numbered entries because each changes what a receipt, a store +object or an executor outcome means, which the charter records. + +**7.1 Raw-byte sources.** A source codec is registered in exactly one of two +modes and a name belongs to at most one: *Frame mode* (`SourceCodec`, +`SourceCodecRegistry.register` / `load`) decodes a source into a population +`Frame`, and *raw-byte mode* (`SourceBytesCodec`, `register_bytes` / +`load_bytes`, `load_source_bytes`) decodes it into immutable `bytes` that the +consuming kernel alone interprets. Neither mode can be loaded through the +other: the mismatch is a `TypeError` naming the mode the codec has, so no +caller receives bytes where it declared a Frame. The shipped `raw-bytes-v1` +codec reads one regular file, bounded at `RAW_BYTES_MAX_BYTES` (64 MiB) and +opened non-blocking so a directory, FIFO, device or socket is refused from +the descriptor's own mode. Identity, the pre-run content key and the post-run +mutation check stay in the executor (E1, E3 unchanged; C3's "declared +predecessors only" is untouched because a source is not a node). Why: a +lookup NPZ or a crosswalk CSV that an import kernel turns into a typed +`ArtifactOutput` (amendment 19) is not a population, and the only way to read +it before this was a Frame codec registered as a pretence. Raised by the US +launch integration branch (thirteen consumers); no key moves, because +`SourceRef.codec` was already normative and `raw-bytes-v1` is a new name. + +**7.2 Frames keep their metadata across the store.** A frame object is +written as `microcosm-graph-frame-v2`: `Frame.metadata` is encoded +losslessly (mappings, tuples, frozensets, `float64` bit patterns, scalars) +into the frame manifest, its SHA-256 is recorded in the object header, and +`_read_frame` refuses a mismatch as `StoreCorruptError`. A v1 object is +`StoreUnavailableError` on load rather than a Frame with its metadata silently +dropped, and writing a frame under a key that already holds one with different +metadata is `StoreCorruptError` (E1: a key identifies exactly one content). +The JSON decode boundary refuses `NaN`/`Infinity` and overflowing numerals, +because every store JSON is written with `allow_nan=False`, so a corrupt +manifest is `StoreCorruptError` at the boundary rather than a `TypeError` +from the canonical re-encoder (E2). Why: the US survey stages carry source +provenance in `Frame.metadata` and replay verifiers compare it +(`same_replayed_frame`); until now a population that went through the store +came back without it. No node key moves: the frame key is the node's, and +metadata was never part of a key. Raised by the US launch integration +branch (`8d9f62c40`); acceptance items E1 and E2. + +**7.3 Execution states: a gate exception leaves its consumers unreached.** +Amendment 19 refused a gate kernel that declares a typed artifact output +because it had no regime for an output a node was unable to produce. This +entry supplies the regime and lifts the refusal. The executor owns an +execution state (`"microcosm.graph.execution.v1"`) that a kernel may not +author: a gate that declares typed outputs and raises still becomes a `fail` +verdict and the run continues (amendment 7, now literally true for every +node shape), and its receipt records `gate_exception` with the outputs it +could not produce; a node whose predecessor is `unreached`, or whose declared +byte input its producer recorded as unavailable, is itself recorded +`unreached` with its blockers named by node key — it runs no kernel and +invents no column, frame, weight or byte (F1: nothing is fabricated); a +release behind such an edge derives its tier from the same ancestry and stays +`evidence` (F2). Cache records that carry a state use schema 3; a cached +unreached record is a hit only while the same inputs are unavailable for the +same reason and is `StoreCorruptError` once they exist (E3); `require` +preflight derives blockers from the cached parents. A run manifest carrying +any state serializes at schema 4 and authenticates every blocker on load +(F5, E4). No node key moves: states live in receipts and records, never in a +key. Why: the US post-clone geography gate emits a typed validation artifact +that the financial stages consume, so a geography failure must leave those +stages unreached rather than abort the run or launder the gate into a compute +node. Raised by the US launch integration branch; supersedes the interim +ruling recorded in amendment 19. + +**7.4 A private population observer.** `run_graph` accepts +`_population_observer`, a callable handed each node's admitted population +(design anchors included) after the result is applied and before the node is +persisted, on cold execution and on cache hits alike; it must not mutate the +population, an exception it raises refuses the run, and an unreached node +has no population to observe. It is an integration seam for verifiers (the +US survey stages assert design-weight provenance through it), not a kernel +capability: it enters no key, receipt or record (A1, B4 unchanged). Raised by +the US launch integration branch. + +## 8. Decisions needed from Max + +1. **Piece C supersedes an amendment-19 ruling.** The Amendment 19 lane + recorded "the gate-artifact-output refusal is the one interim ruling this + lane made on his behalf". This branch's US runtime needs the opposite (the + post-clone geography gate declares its validation artifact and three + stages consume it), so piece C re-applies the branch's execution-state + regime and replaces main's test that pins the refusal with five tests of + the new regime. Both the frozen files and every `test_acceptance_*` file + are untouched, but the executor's behaviour on that one node shape now + differs from what the charter's amendment 19 text says. Landing PR-G3 + requires adopting entry 7.3 (or an edited form) in the charter; until then + this branch's executor and the charter disagree on that sentence. +2. **Lazy population retention** (`attachments.py`, layer 08) is dropped for + lack of a consumer. If the native pilot's memory ceiling is a reason to + keep it, it is a self-contained PR on top of PR-G2. +3. **`_stream_file` and the `_write_node` refactor** are dropped for lack of a + consumer. Both are memory-only; say if either should come back as its own + small PR. +4. **The report's home.** `out.md` at the repo root is a tracked file holding + the F1 Sol gate round-1 report (2026-09-04). This report is written there + for the dispatcher but is **not committed** there; the committed copy is + `experiments/893-reconciliation-amendments-19-20-20260912.md`. +5. **Three US stages name a resource that does not exist anywhere.** + `asec_prepared_v3`, `composed_asec_binding_v1` and `composed_population_v1` + list `us_runtime/asec_current_money_engine_defaults_v1.json` in their + resources; the file is on neither this branch (at `069d5ed9a` or now) nor + `origin/main` (it appears only in two commits of another branch, + `b0e7f54b2` / `b6cfc84b0`). `implementation_manifest` for those stages has + therefore been failing since before this lane. Not invented here; the + owner of layer 10 SOURCE-CLOSURE has to admit the file or drop the entry, + and the inventory's "scope review" rule applies to `07566eb17` as well. +6. **Two build tests fail locally for reasons outside this lane** if they are + run: `test_release_target_parity.py`'s two `_feed_or_skip` tests need a 131 + MB feed outside the repository (already recorded by the Amendment 19 lane). + They were not part of this lane's runs. + +## 9. Where things are + +- Journal: `PROGRESS.md` (top section, cumulative file). +- This report: `out.md` (uncommitted, the `-o` path) and + `experiments/893-reconciliation-amendments-19-20-20260912.md` (committed). +- Scratch evidence (session-local, not committed): the AST consumer scan, + the layer scripts and JSON, and every gate log named in section 5. From a4b6fe88ca95e582aedc027ad7f092bd4a38ae31 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 07:58:40 -0400 Subject: [PATCH 100/255] Restore pinned resources for prepared ASEC stage manifests --- .../893-asec-prepared-resources.fixed.md | 1 + ...asec_current_money_engine_defaults_v1.json | 34 +++++ ...asec_current_money_graph_consumers_v1.json | 127 ++++++++++++++++++ .../asec_income_observations_v1.json | 122 +++++++++++++++++ .../tests/test_us_asec_prepared_resources.py | 109 +++++++++++++++ 5 files changed, 393 insertions(+) create mode 100644 changelog.d/893-asec-prepared-resources.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations_v1.json create mode 100644 packages/microcosm-build/tests/test_us_asec_prepared_resources.py diff --git a/changelog.d/893-asec-prepared-resources.fixed.md b/changelog.d/893-asec-prepared-resources.fixed.md new file mode 100644 index 000000000..95dc74d99 --- /dev/null +++ b/changelog.d/893-asec-prepared-resources.fixed.md @@ -0,0 +1 @@ +Restore three pinned ASEC resource files omitted during integration so prepared and composed source stages can compute their implementation manifests. Preserve the existing empty engine-default allowlist, source declarations, and reported-income contract. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json new file mode 100644 index 000000000..583c3684d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "artifact_kind": "microcosm.us.asec_current_money_engine_defaults.v1", + "engine_package": "policyengine-us", + "engine_version": "1.819.0", + "allowlist": [], + "allowlist_policy": "no engine default is an approved input to an admitted output; a closure leaf this graph does not produce blocks its root", + "admitted_roots": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "blocked_roots": { + "interest_income": "needs tax_exempt_interest_income, which this slice does not map", + "pension_income": "needs the public pension leaves, which this slice does not map", + "self_employment_income": "needs sstb_self_employment_income_before_lsr, which this slice does not map" + }, + "release_eligible": false, + "scoring_claim": "none; these are engine outputs on an engineering sample, not a tax or benefit score", + "baseline_runtime": { + "policyengine-us": { + "version": "1.819.0", + "files": 12313, + "source_and_parameters_sha256": "12c474205741021bb45ae618f85ddd6d15e8cbb51dece8334f2d52c04500c674" + }, + "policyengine-core": { + "version": "3.31.0", + "files": 175, + "source_and_parameters_sha256": "fbcc6f3d8e86c023c29e0782d95103155828cfa4ef6a80ed6f785f619be6439e" + } + }, + "baseline_runtime_file_policy": "all package Python source and parameters YAML/JSON; sorted relative paths and SHA256; no population data" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json new file mode 100644 index 000000000..cd0270dee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json @@ -0,0 +1,127 @@ +{ + "all_current_money_consumers_wired": false, + "artifact_kind": "microcosm.us.asec_current_money_graph_consumers.v1", + "blocked_engine_outputs": { + "interest_income": [ + "tax_exempt_interest_income" + ], + "pension_income": [ + "tax_exempt_public_pension_income", + "taxable_public_pension_income" + ], + "self_employment_income": [ + "sstb_self_employment_income_before_lsr" + ] + }, + "engine_defaults_allowlist": [], + "implementation_status": "prepared_current_money_graph_and_microunit_wired_other_nominal_routes_pending", + "pending_nominal_consumer_routes": [ + "acs_release_predictors_reported_ssi_anchor", + "asec_household_observations", + "asec_other_income_split", + "child_support", + "childcare", + "cps_carried", + "disability_benefits", + "education_inputs", + "eligibility_inputs", + "energy_subsidy", + "immigration", + "retirement_contributions", + "retirement_distributions", + "voluntary_filing_current_leaf", + "workers_compensation" + ], + "release_eligible": false, + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "source_declarations": { + "binding": "unchanged historical source recipe; this additive record supersedes implementation status only for the explicitly wired routes below; source address roster and T1/money identities are unchanged", + "resource": "asec_current_money_consumers_v1.json", + "sha256": "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7" + }, + "wired_routes": { + "asec_prepared_current_money_graph": { + "engine_output_storage": "evaluation_artifact_only", + "engine_outputs": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "module": "microcosm.build.us_runtime.graph_asec_prepared", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "typed selected current-money body; never standalone ReadyCurrentMoney", + "person_leaves": [ + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation" + ], + "spm_unit_leaves": [ + "spm_unit_pre_subsidy_childcare_expenses" + ] + }, + "microunit": { + "module": "microcosm.build.us_runtime.asec_current_money_units", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "authenticated full-source ReadyCurrentMoney", + "student_controls": "separately authenticated S attached before tax construction; interview-week proxy only" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations_v1.json new file mode 100644 index 000000000..b9d91d577 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations_v1.json @@ -0,0 +1,122 @@ +{ + "schema_version": 1, + "artifact_kind": "microcosm.us.asec_income_observations.v1", + "fields": { + "PAW_VAL": { + "dtype": "int64", + "minimum": 0, + "maximum": 99999, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone", + "universe": "PAW_YN = 1" + }, + "PAW_YN": { + "dtype": "int64", + "codes": { + "0": "niu", + "1": "yes", + "2": "no" + }, + "universe": "All persons aged 15+" + } + }, + "cohorts": { + "2022": { + "survey_year": 2023, + "source_references": { + "PAW_VAL": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "pdf_page": 49, + "excerpt_sha256": "cff5717e53b91eea99f1d87ecc29bea72f2c4a93dcb0adb37204b4173ed52b2c" + }, + "PAW_YN": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "pdf_page": 49, + "excerpt_sha256": "d860a892c16da095d47dfe2f4a7f6f412556dbd91c2a41b5a6ca26f20d0e661b" + } + } + }, + "2023": { + "survey_year": 2024, + "source_references": { + "PAW_VAL": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "pdf_page": 50, + "excerpt_sha256": "cff5717e53b91eea99f1d87ecc29bea72f2c4a93dcb0adb37204b4173ed52b2c" + }, + "PAW_YN": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "pdf_page": 50, + "excerpt_sha256": "d860a892c16da095d47dfe2f4a7f6f412556dbd91c2a41b5a6ca26f20d0e661b" + } + } + }, + "2024": { + "survey_year": 2025, + "source_references": { + "PAW_VAL": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "pdf_page": 51, + "excerpt_sha256": "57d6f91454cc6a19eb991f6ecda75ed615e4e321acc00cd71e9a5c865ee12487" + }, + "PAW_YN": { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "pdf_page": 52, + "excerpt_sha256": "96230020ee4159b4015853223734586f6993221724ead55b3af1d53e2a66644a" + } + } + } + }, + "monetary_basis": { + "2022": { + "currency": "USD", + "unit": "base_currency", + "period": "2022", + "temporal_basis": "annual_flow", + "sector": "households", + "perimeter": "cps_asec_person_money_income", + "valuation": "2024 annual average C-CPI-U dollars (census_ccpiu_annual_2024)" + }, + "2023": { + "currency": "USD", + "unit": "base_currency", + "period": "2023", + "temporal_basis": "annual_flow", + "sector": "households", + "perimeter": "cps_asec_person_money_income", + "valuation": "2024 annual average C-CPI-U dollars (census_ccpiu_annual_2024)" + }, + "2024": { + "currency": "USD", + "unit": "base_currency", + "period": "2024", + "temporal_basis": "annual_flow", + "sector": "households", + "perimeter": "cps_asec_person_money_income", + "valuation": "2024 annual average C-CPI-U dollars (census_ccpiu_annual_2024)" + } + }, + "price": { + "algorithm": "parse_float64_target_divide_source_then_multiply_nonzero_v1", + "target_year": 2024, + "cells": { + "2022": "163.6", + "2023": "170.0", + "2024": "174.4" + }, + "resource_sha256": "88cec207a3c73e5de93d677acf16e8db9ddae9761c644c06da92cacf0feb3f63" + }, + "zero_origin_classes": [ + "observed_amount", + "niu_below_receipt_age", + "reported_receipt_zero_amount", + "reported_no_receipt", + "niu_in_age_universe" + ], + "consumer": { + "semantic": "reported_cash_public_assistance_not_simulated_tanf", + "raw_frame_aliases": [], + "PAW_TYP_authority": "excluded_from_this_artifact_existing_tanf_handling_unchanged", + "acs_eight_categories_complete": false, + "release_eligible": false + } +} diff --git a/packages/microcosm-build/tests/test_us_asec_prepared_resources.py b/packages/microcosm-build/tests/test_us_asec_prepared_resources.py new file mode 100644 index 000000000..cffcb192c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_prepared_resources.py @@ -0,0 +1,109 @@ +"""Packaged admission evidence; no survey data or rules-engine execution.""" + +from hashlib import sha256 +from importlib import resources + +import pytest + +from microcosm.build.us_runtime import asec_engine_evaluation as evaluation +from microcosm.build.us_runtime.graph_implementation import implementation_manifest + +# These are the exact bytes admitted in b0e7f54b28145ee920ad0f4c5d0e2090c8b0785a, +# not newly observed identities of whatever engine a test environment installs. +_RESTORED_RESOURCE_SHA256 = ( + "090dd200ecdb837127dba1d1f22903d152cef8a434085e4ead4d379350ee5b82" +) +_RESTORED_RESOURCES = { + "asec_current_money_engine_defaults_v1.json": _RESTORED_RESOURCE_SHA256, + # Exact historical 4cfcd0723 and 41f2a7f72 resources, respectively. + "asec_current_money_graph_consumers_v1.json": ( + "67432a10d05e11dd4b97a80cd4f04ed0eb6e56c4c8e21d5dffe87c5f67e27c66" + ), + "asec_income_observations_v1.json": ( + "6e37855ddd6642d02231efc664644029ed6131885eb743a00197261bc5a6a51f" + ), +} + + +def test_packaged_defaults_restore_the_admitted_evidence_without_new_claims(): + payload = ( + resources.files("microcosm.build.us_runtime") + .joinpath(evaluation.ENGINE_DEFAULTS_RESOURCE) + .read_bytes() + ) + assert sha256(payload).hexdigest() == _RESTORED_RESOURCE_SHA256 + assert evaluation.ENGINE_DEFAULTS_SHA256 == _RESTORED_RESOURCE_SHA256 + document = evaluation.load_engine_defaults() + assert document["allowlist"] == [] + assert document["release_eligible"] is False + assert tuple(document["admitted_roots"]) == evaluation.ADMITTED_ROOTS + assert tuple(sorted(document["blocked_roots"])) == evaluation.BLOCKED_ENGINE_OUTPUTS + assert document["baseline_runtime"]["policyengine-us"]["version"] == "1.819.0" + assert document["baseline_runtime"]["policyengine-core"]["version"] == "3.31.0" + assert all( + not any( + marker in leaf for marker in ("last_year", "previous_year", "prior_year") + ) + for contract in evaluation.ADMITTED_ENGINE_OUTPUT_CONTRACTS + for leaf in contract.input_leaves + ) + + +@pytest.mark.parametrize( + "stage", + ("asec_prepared_v3", "composed_asec_binding_v1", "composed_population_v1"), +) +def test_prepared_stage_manifest_binds_the_packaged_resources(stage): + pytest.importorskip("microunit") + pytest.importorskip("tables") + pytest.importorskip("h5py") + manifest = implementation_manifest(stage) + for name, fingerprint in _RESTORED_RESOURCES.items(): + assert ( + manifest["resources"][f"microcosm.build/us_runtime/{name}"] == fingerprint + ) + + +def test_restored_graph_consumer_status_keeps_its_source_parent_and_scope(): + from microcosm.build.us_runtime.asec_current_money_graph_resources import ( + load_graph_current_money_consumers, + ) + + document = load_graph_current_money_consumers() + assert document["scope"] == "current_money_only_no_prior_wages" + assert document["engine_defaults_allowlist"] == [] + assert document["release_eligible"] is False + assert document["all_current_money_consumers_wired"] is False + + +def test_restored_income_contract_preserves_reported_source_semantics(): + from microcosm.build.us_runtime.asec_income_observations import _contract + + document = _contract() + assert set(document["fields"]) == {"PAW_VAL", "PAW_YN"} + assert document["consumer"]["semantic"] == ( + "reported_cash_public_assistance_not_simulated_tanf" + ) + assert document["consumer"]["raw_frame_aliases"] == [] + assert document["consumer"]["release_eligible"] is False + + +@pytest.mark.parametrize("alteration", ("missing", "tampered")) +def test_defaults_never_fall_back_when_packaged_evidence_is_unavailable( + tmp_path, monkeypatch, alteration +): + payload = ( + resources.files("microcosm.build.us_runtime") + .joinpath(evaluation.ENGINE_DEFAULTS_RESOURCE) + .read_bytes() + ) + monkeypatch.setattr(evaluation.resources, "files", lambda package: tmp_path) + if alteration == "tampered": + (tmp_path / evaluation.ENGINE_DEFAULTS_RESOURCE).write_bytes(payload + b"\n") + with pytest.raises( + evaluation.EngineAdmissionError, match="ENGINE_DEFAULTS_FINGERPRINT" + ): + evaluation.load_engine_defaults() + else: + with pytest.raises(FileNotFoundError): + evaluation.load_engine_defaults() From d44e048ba5ac198eb582f4b74a490d843f56e258 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:00:51 -0400 Subject: [PATCH 101/255] Record interrupted reconciliation battery and independent review blockers --- ...econciliation-amendments-19-20-20260912.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md index b910ebbba..7483a2539 100644 --- a/experiments/893-reconciliation-amendments-19-20-20260912.md +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -1,5 +1,24 @@ # #893 reconciliation to main's amended graph interface — lane report +> **12 September Codex continuation:** this is the historical layer proposal +> at `80fef2217`, not an approved extraction plan. The final battery +> `bt0nx8fcc` was killed during its first pytest group: CI-group verification +> and coverage check exited zero, but no final consumer result was produced. +> Independent review found that G4 exposes mutable admitted populations to +> observers and that the layer order omits hard packaged-resource dependencies. +> Fix and review both before extracting or landing layers. PR912 is also held +> for fixes from Astra's executed review; the branch-head assumption below is +> superseded until its corrected head lands. +> +> Resource repair `a4b6fe88c` restores all three omitted ASEC JSON contracts +> from their exact historical blobs, matching unchanged loader pins. Eight +> targeted tests and wheel-content verification pass. These three resources, +> `acs_2024_housing_universe.json`, and current-money domains/consumers/price-basis +> resources must accompany their first PR-1 loaders or an earlier prerequisite; +> Python import analysis alone cannot establish extraction order. No engine +> defaults were admitted and no source/engine identity was refreshed. + + Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is `microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no @@ -210,7 +229,7 @@ change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: -_(battery running at the time of this draft; the final table replaces this line)_ +_(Historical draft marker resolved: task `bt0nx8fcc` was killed during availability consumers. No final battery counts or exit code exist; later groups did not start.)_ ## 6. Layer map: landing #893 as sequential PRs From 6f66fa54541bf2e93bcebd7992b7156b11779190 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:17:41 -0400 Subject: [PATCH 102/255] Isolate graph population observers from execution and cache state --- ...893-population-observer-isolation.fixed.md | 1 + .../src/microcosm/graph/executor.py | 73 ++++++- .../tests/test_graph_executor.py | 206 +++++++++++++++++- 3 files changed, 271 insertions(+), 9 deletions(-) create mode 100644 changelog.d/893-population-observer-isolation.fixed.md diff --git a/changelog.d/893-population-observer-isolation.fixed.md b/changelog.d/893-population-observer-isolation.fixed.md new file mode 100644 index 000000000..710af6485 --- /dev/null +++ b/changelog.d/893-population-observer-isolation.fixed.md @@ -0,0 +1 @@ +Give graph population observers detached snapshots so callback mutations and retained references cannot change subsequent calculations or cached populations. Preserve the complete frame, provenance, weights and mass accounting on both cold runs and cache hits. diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 53c119329..b61a678d4 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -4,9 +4,12 @@ import hashlib import json +import pickle import socket import time from collections.abc import Callable, Mapping +from copy import deepcopy +from dataclasses import fields, replace from datetime import UTC, datetime from pathlib import Path from types import MappingProxyType @@ -348,6 +351,61 @@ def _freeze_frame(table: pd.DataFrame) -> pd.DataFrame: return frozen +def _observer_snapshot(population: Population) -> Population: + """Detach every observation from the executable population and its cache. + + Pandas deep copies retain object-cell referents and some immutable-by-API + axis/category buffers. An in-memory, in-band round trip copies those too; + only pandas objects from this admitted population are serialized here. + No external pickle bytes are accepted, retained, or persisted. Frame and + graph records are reconstructed explicitly to avoid reflective copy/pickle + writes to their dataclass namespaces (which source identities may seal). + + When enabled, this costs one full detached population and a temporary + serialized table buffer per callback. Observers may retain that snapshot; + changing it immediately or later cannot change a kernel input or store write. + """ + frame = population.frame + tables = {name: frame.table(name) for name in frame.entities} + tables.update({name: frame.link(name) for name in frame.links}) + tables, strata = pickle.loads(pickle.dumps((tables, frame.strata), protocol=5)) + + def copied_record(record): + return replace( + record, + **{ + field.name: deepcopy(getattr(record, field.name)) + for field in fields(record) + }, + ) + + snapshot = Frame( + tables, + replace( + frame.schema, + group_entities=deepcopy(frame.schema.group_entities), + links=tuple(copied_record(link) for link in frame.schema.links), + ), + { + entity: Weights( + frame.weights_for(entity).values, frame.weights_for(entity).kind + ) + for entity in frame.weighted_entities + }, + strata, + mass_log=tuple(copied_record(record) for record in frame.mass_log), + metadata=frame.metadata, + ) + return Population( + snapshot, + population.version, + dict(population.owners), + dict(population.weight_kind), + mass_ledger=tuple(copied_record(record) for record in population.mass_ledger), + design_weights=population.design_weights, + ) + + def _update_scalar(digest: hashlib._Hash, value: object) -> None: if value is pd.NA: payload = b"pd.NA" @@ -2193,12 +2251,13 @@ def run_graph( ) -> RunManifest: """Execute a compiled graph with content-addressed reuse and receipts. - The private population observer exposes each node's admitted population, - design anchors included, to an integrating verifier. It runs for cold - execution and for restored cache hits alike, before the node is persisted; - it must not mutate the population, and an exception it raises refuses the - run. It is never a kernel capability, enters no key or receipt, and an - unreached node has no population to observe. + The private population observer exposes a detached snapshot of each node's + admitted population, design anchors included, to an integrating verifier. + It runs for cold execution and for restored cache hits alike, before the + node is persisted; changes to the snapshot cannot alter execution or + persistence, and an exception it raises refuses the run. It is never a + kernel capability, enters no key or receipt, and an unreached node has no + population to observe. """ if resume not in ("auto", "require", "forbid"): @@ -2500,7 +2559,7 @@ def run_graph( populations[node.id] = updated if _population_observer is not None: - _population_observer(node_id, updated) + _population_observer(node_id, _observer_snapshot(updated)) if not hit: manifest_artifacts, record = _write_node( diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index 09c376cd7..ffb0ff86b 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -15,7 +15,14 @@ import pytest import microcosm.graph.executor as graph_executor -from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.frame import ( + EntitySchema, + Frame, + LinkSpec, + MassChangeRecord, + WeightKind, + Weights, +) from microcosm.graph.availability import EXECUTION_SCHEMA from microcosm.graph.decl import ( ArtifactInput, @@ -48,7 +55,7 @@ ) from microcosm.graph.keys import opaque_artifact_key, platform_fingerprint from microcosm.graph.manifest import Decision, RunManifest -from microcosm.graph.population import Population +from microcosm.graph.population import MassRecord, Population from microcosm.graph.store import ( ContentStore, StoreCorrupt, @@ -4086,6 +4093,201 @@ def refuse(node_id: str, population: Population) -> None: ) +@pytest.mark.parametrize("warm", (False, True)) +def test_mutating_and_retained_observers_cannot_change_execution_or_cache( + tmp_path: Path, warm: bool +) -> None: + source = _source_path(tmp_path / "source") + compiled = compile_graph(_graph(leaf=False)) + plain = run_graph( + compiled, + sources={"survey": source}, + store=ContentStore(tmp_path / "plain"), + kernels=_registry(), + ) + store = ContentStore(tmp_path / "observed") + if warm: + run_graph( + compiled, sources={"survey": source}, store=store, kernels=_registry() + ) + retained = [] + + def mutate(population): + population.frame.person.loc[:, "age"] += 100 + weights = population.frame.weights_for("household").values + weights.setflags(write=True) + weights[:] = 999 + population.frame._metadata = {"observer": "changed"} + object.__setattr__(population, "owners", {}) + + def observe(node_id, population): + # Also mutate earlier snapshots during later callbacks, after a simple + # before/after check around their own callback would have completed. + retained.append(population) + for previous in retained: + mutate(previous) + + observed = run_graph( + compiled, + sources={"survey": source}, + store=store, + kernels=_registry(), + _population_observer=observe, + ) + for population in retained: + mutate(population) # retained references remain harmless after return + replay = run_graph( + compiled, sources={"survey": source}, store=store, kernels=_registry() + ) + assert all(receipt.hit for receipt in observed.nodes.values()) is warm + assert all(receipt.hit for receipt in replay.nodes.values()) + for actual in (observed, replay): + assert actual.key == plain.key + assert {name: item.key for name, item in actual.nodes.items()} == { + name: item.key for name, item in plain.nodes.items() + } + for entity in plain.populations["survey"].entities: + pd.testing.assert_frame_equal( + actual.populations["survey"].table(entity), + plain.populations["survey"].table(entity), + ) + np.testing.assert_array_equal( + actual.populations["survey"].weights_for("household").values, + plain.populations["survey"].weights_for("household").values, + ) + assert ( + actual.populations["survey"].metadata + == plain.populations["survey"].metadata + ) + assert observed.populations["survey"].person["b"].tolist() == [60.0, 120.0, 180.0] + + +def test_observer_snapshot_detaches_complete_population_storage(tmp_path: Path) -> None: + original = _source_frame(_source_path(tmp_path / "source")) + person = original.person.copy() + person.index = pd.MultiIndex.from_tuples( + [("a", 1), ("a", 2), ("b", 3)], names=["part", "row"] + ) + person["object_cell"] = pd.Series( + [{"nested": [1]}, {"nested": [2]}, {"nested": [3]}], + index=person.index, + dtype=object, + ) + person["category"] = pd.Categorical(["x", "y", "x"]) + person["selected"] = pd.array([True, pd.NA, False], dtype="boolean") + person["selected"].array._data[1] = True # preserve storage beneath the mask + person.attrs["nested"] = {"values": [1, 2]} + schema = EntitySchema( + group_entities=("household",), + links=(LinkSpec("relations", "person", "household"),), + ) + link = pd.DataFrame({"person_id": [1, 2, 3], "household_id": [10, 10, 20]}) + mass_log = (MassChangeRecord("household", 3.0, 3.0, 1.0, "unchanged"),) + frame = Frame( + {"person": person, "household": original.table("household"), "relations": link}, + schema, + {"household": original.weights_for("household")}, + pd.Series(["a", "a", "b"], index=person.index, name="stratum"), + metadata={"nested": [{"source": "fixture"}], "signed_zero": -0.0}, + mass_log=mass_log, + ) + ledger = ( + MassRecord( + "fixture", + "reweight", + "conserve", + 3.0, + 3.0, + (("a", 3.0),), + (("a", 3.0),), + entity="household", + ), + ) + population = Population.from_frame(frame, "fixture", mass_ledger=ledger) + snapshot = graph_executor._observer_snapshot(population) + assert snapshot.frame.schema == population.frame.schema + assert snapshot.frame.mass_log == population.frame.mass_log + assert snapshot.mass_ledger == population.mass_ledger + assert dict(snapshot.owners) == dict(population.owners) + assert dict(snapshot.weight_kind) == dict(population.weight_kind) + assert snapshot.frame.metadata == population.frame.metadata + assert np.signbit(snapshot.frame.metadata["signed_zero"]) + assert snapshot.frame.metadata is not frame.metadata + assert snapshot.frame.metadata["nested"][0] is not frame.metadata["nested"][0] + for name in frame.entities: + pd.testing.assert_frame_equal(snapshot.frame.table(name), frame.table(name)) + pd.testing.assert_frame_equal( + snapshot.frame.link("relations"), frame.link("relations") + ) + pd.testing.assert_series_equal(snapshot.frame.strata, frame.strata) + np.testing.assert_array_equal( + snapshot.design_weights["household"], population.design_weights["household"] + ) + assert not np.shares_memory( + snapshot.design_weights["household"], population.design_weights["household"] + ) + np.testing.assert_array_equal( + snapshot.frame.person["selected"].array._data, + frame.person["selected"].array._data, + ) + + snapshot.frame.person.at[("a", 1), "object_cell"]["nested"].append(99) + snapshot.frame.person.attrs["nested"]["values"].append(99) + snapshot.frame.person.index.set_names(["changed", "row"], inplace=True) + level = snapshot.frame.person.index.levels[0].to_numpy(copy=False) + level.setflags(write=True) + level[0] = "changed" + categories = snapshot.frame.person["category"].cat.categories.to_numpy(copy=False) + categories.setflags(write=True) + categories[0] = "changed" + snapshot.frame.link("relations").iloc[0, 0] = 999 + snapshot.frame.strata.iloc[0] = "changed" + snapshot.frame.person["selected"].array._data[1] = False + snapshot.frame.weights_for("household").values.setflags(write=True) + snapshot.frame.weights_for("household").values[:] = 999 + captured_metadata = snapshot.frame.metadata["nested"][0] + object.__setattr__(captured_metadata, "_items", (("source", "changed"),)) + assert frame.metadata["nested"][0]["source"] == "fixture" + snapshot.frame._metadata = {"changed": True} + object.__setattr__(snapshot.frame.schema.links[0], "name", "changed") + object.__setattr__(snapshot.frame.schema, "group_entities", ("changed",)) + object.__setattr__(snapshot.frame.mass_log[0], "reason", "changed") + object.__setattr__(snapshot.mass_ledger[0], "policy", "changed") + object.__setattr__(snapshot, "owners", {}) + assert frame.person.at[("a", 1), "object_cell"] == {"nested": [1]} + assert frame.person.attrs["nested"] == {"values": [1, 2]} + assert frame.person.index.names == ["part", "row"] + assert frame.person.index.levels[0].tolist() == ["a", "b"] + assert frame.person["category"].cat.categories.tolist() == ["x", "y"] + assert frame.link("relations").iloc[0, 0] == 1 + assert frame.strata.iloc[0] == "a" + assert bool(frame.person["selected"].array._data[1]) is True + assert frame.weights_for("household").values.tolist() == [1.0, 2.0] + assert frame.metadata["nested"][0]["source"] == "fixture" + assert frame.schema.links[0].name == "relations" + assert frame.schema.group_entities == ("household",) + assert frame.mass_log[0].reason == "unchanged" + assert population.mass_ledger[0].policy == "conserve" + assert population.owners + + # Record annotations do not freeze nested members. Even a caller-supplied + # container inside a record must not remain an alias across the seam. + nested_record = replace(ledger[0], before_by_stratum=((["mutable"], 3.0),)) + nested = replace(population, mass_ledger=(nested_record,)) + nested_snapshot = graph_executor._observer_snapshot(nested) + nested_snapshot.mass_ledger[0].before_by_stratum[0][0].append("changed") + assert nested.mass_ledger[0].before_by_stratum[0][0] == ["mutable"] + + +def test_absent_observer_allocates_no_snapshot(tmp_path: Path, monkeypatch) -> None: + def forbidden(population): + raise AssertionError("snapshot without observer") + + monkeypatch.setattr(graph_executor, "_observer_snapshot", forbidden) + source = _source_path(tmp_path / "source") + _run(_graph(), source, ContentStore(tmp_path / "store"), _registry()) + + def test_a_gate_reached_only_through_bytes_still_derives_the_tier( tmp_path: Path, ) -> None: From cb27bdb0b550db1ee7deead28d1424bd5d3b3500 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:23:35 -0400 Subject: [PATCH 103/255] Show unavailable execution separately from graph cache reuse --- .../893-explain-execution-state.fixed.md | 2 ++ .../src/microcosm/graph/explain.py | 23 +++++++++++++++---- .../tests/test_graph_executor.py | 16 +++++++++++++ .../tests/test_graph_explain.py | 22 ++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 changelog.d/893-explain-execution-state.fixed.md diff --git a/changelog.d/893-explain-execution-state.fixed.md b/changelog.d/893-explain-execution-state.fixed.md new file mode 100644 index 000000000..87341666c --- /dev/null +++ b/changelog.d/893-explain-execution-state.fixed.md @@ -0,0 +1,2 @@ +Show unreached computations and gate exceptions independently from cache hits +in the offline graph explanation, including cached unavailable artifacts. diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index 26abfdbd4..d17d8be94 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -12,6 +12,7 @@ from enum import Enum from typing import TYPE_CHECKING +from .availability import execution_state from .decl import GATE_OUTCOMES, CompiledGraph, StructuralDelta from .manifest import NodeReceipt, RunManifest from .population import mass_record_receipt @@ -194,6 +195,10 @@ .graph-node.gate-unreached .node-box { stroke: var(--color-red-600); stroke-width: 3; } .graph-node.gate-pass .node-box, .graph-node.gate-not_applicable .node-box { stroke: var(--color-green-600); stroke-width: 3; } +.graph-node.execution-unreached .node-box { fill: var(--surface-muted); + stroke: var(--color-red-600); stroke-dasharray: 5 3; } +.graph-node.execution-gate_exception .node-box { fill: var(--color-red-100); + stroke: var(--color-red-600); } .graph-node:focus .node-box, .graph-node.selected .node-box { stroke-width: 4.5; } .node-title { fill: var(--text); font-size: 13px; font-weight: 760; } .node-line { fill: var(--muted); font-size: 10.5px; } @@ -308,11 +313,19 @@ def _role(receipt: NodeReceipt) -> str: def _node_status(receipt: NodeReceipt) -> tuple[str, str]: store = "hit" if receipt.hit else "miss" role = _role(receipt) - if role != "gate": - return f"status-{store}", store - outcome = str(receipt.receipt.get("outcome", "unrecorded")) - outcome_class = outcome if outcome in GATE_OUTCOMES else "unknown" - return f"status-{store} gate-{outcome_class}", f"{store} · gate {outcome}" + classes, label = f"status-{store}", store + if role == "gate": + outcome = str(receipt.receipt.get("outcome", "unrecorded")) + outcome_class = outcome if outcome in GATE_OUTCOMES else "unknown" + classes += f" gate-{outcome_class}" + label += f" · gate {outcome}" + # Cache reuse and execution are independent: a cached refusal did not run + # the downstream computation. Interpret only executor-owned state. + state = execution_state(receipt.receipt) + if state in {"unreached", "gate_exception"}: + classes += f" execution-{state}" + label += " · unreached" if state == "unreached" else " · exception" + return classes, label def _depths(compiled: CompiledGraph) -> dict[str, int]: diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index ffb0ff86b..6dc5ca1b6 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -41,6 +41,7 @@ ) from microcosm.graph.errors import NodeRejectedError from microcosm.graph.executor import NodeRejected, run_graph +from microcosm.graph.explain import explain_html from microcosm.graph.kernel import ( ArtifactValue, Capabilities, @@ -3807,12 +3808,27 @@ def test_a_gate_that_declares_evidence_and_raises_leaves_its_consumers_unreached assert restored.nodes["use"].receipt == use.receipt assert restored.tier == "evidence" + def assert_explained(outcome: RunManifest, cache: str) -> None: + rendered = explain_html(compile_graph(graph), outcome) + for node_id in ("use", "after", "release"): + role = "release" if node_id == "release" else "compute" + assert ( + f'aria-label="{node_id}; {node_id}@1; {role}; none; ' + f'{cache} · unreached"' + ) in rendered + assert rendered.count('execution-unreached" data-node-detail=') == 3 + assert f'status-{cache} gate-fail execution-gate_exception"' in rendered + assert f"{cache} · gate fail · exception" in rendered + + assert_explained(manifest, "miss") + for resume in ("auto", "require"): again = _gate_artifact_registry(raising=True) replay = _run(graph, source, store, again, resume=resume) assert all(receipt.hit for receipt in replay.nodes.values()) assert replay.key == manifest.key assert sum(_calls(again).values()) == 0 + assert_explained(replay, "hit") def test_unreached_propagates_through_a_structural_node_and_its_version( diff --git a/packages/microcosm-graph/tests/test_graph_explain.py b/packages/microcosm-graph/tests/test_graph_explain.py index da74a5385..b8bac925f 100644 --- a/packages/microcosm-graph/tests/test_graph_explain.py +++ b/packages/microcosm-graph/tests/test_graph_explain.py @@ -425,6 +425,28 @@ def test_manifest_only_page_omits_optional_sections(tmp_path: Path) -> None: assert "Incident replays" not in rendered +@pytest.mark.parametrize( + "diagnostic", + ({"state": "unreached"}, {"schema": "custom/v1", "state": "unreached"}), +) +def test_free_form_diagnostics_do_not_claim_executor_state( + tmp_path: Path, diagnostic: dict[str, str] +) -> None: + run = toy.run_toy(toy.full_graph(), tmp_path / "run") + original = run.manifest.nodes["calibrated"] + changed = replace( + original, receipt={**dict(original.receipt), "execution": diagnostic} + ) + manifest = replace( + run.manifest, nodes={**dict(run.manifest.nodes), "calibrated": changed} + ) + + rendered = explain_html(run.compiled, manifest) + + assert 'execution-unreached" data-node-detail=' not in rendered + assert 'execution-gate_exception" data-node-detail=' not in rendered + + def test_saved_run_cli_validates_store_and_reattaches_frames(tmp_path: Path) -> None: run = toy.run_toy(toy.full_graph(), tmp_path / "run") manifest_path = tmp_path / "run" / "manifest.json" From 4d82528db1c6a59ade3e7137b470ad0350e522a6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:26:12 -0400 Subject: [PATCH 104/255] Record reviewed graph dependency and current native release progress --- docs/us-uk-release-path.md | 35 +++++++++++++------ ...econciliation-amendments-19-20-20260912.md | 22 ++++++++++++ .../graph_implementation_inventory.json | 1 + 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index ddfff2328..a75930fc9 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -1,6 +1,6 @@ # US and UK release path -Planning snapshot, 10 September 2026. This is a release work plan, not a +Planning snapshot, 12 September 2026. This is a release work plan, not a certification of either country. Component tests, historical candidates and current-candidate acceptance are distinct evidence. @@ -43,20 +43,35 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | | Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; complete PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | -| Geography | Corrected post-clone order passes 70 invented controls plus separate default compatibility; native post-clone execution and complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict replay is running; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | | Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | -The US new graph has passed a small real survey → block assignment → clone → -financial-enrichment build and required replay. The combined survey population -has 1,584 source households and 3,168 records after cloning. That native result -uses the earlier assignment-before-cloning order. The corrected nine-node prefix -and nineteen-node financial extension now pass their invented controls; the -next native pilot must use their new order and a fresh graph store. The remaining -critical sequence is complete PUF/input integration, a small real full build, -model and calibration evaluation, then progressive scale and release verification. See +The 11 September native pilot passed the corrected survey → full initial +support clone → atomic block assignment → financial-enrichment order. At a +1/1000 source sample, its 1,584 source households and 3,464 people become +3,168 households and 6,928 people after cloning. The nineteen-node graph fits +and attaches seven financial fields. Its closed receipt is tied to frozen +source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. +This supersedes the earlier assignment-before-cloning milestone for ordering; +it does not establish full PUF enrichment or national/CD calibration. + +On 12 September, a strict required replay of that frozen source was launched +with a fresh output directory and a read-only retained store. It must verify +all source bytes again, reuse every node, and reproduce complete artifact +content, excluding only operational timing and cache-hit fields. Its outcome +is pending. Meanwhile the maintained graph's observer-mutation defect has been +fixed and independently reviewed, and the shared graph changes are being +consolidated against the amended main interface. These software changes still +need their own native candidate verification; the frozen pilot does not +certify the newer source. + +The remaining critical sequence is complete PUF/input integration, a small +real full build, model and calibration evaluation, then progressive scale +and release verification. The existing age-development runner is a separate +prefix demonstration, not calibration of the nineteen-node financial result. See [the US review guide](us-launch-review.md) for scoped acceptance records. UK work already includes raw-source enrichment, national/local calibration, diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md index 7483a2539..210c603a2 100644 --- a/experiments/893-reconciliation-amendments-19-20-20260912.md +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -18,6 +18,28 @@ > Python import analysis alone cannot establish extraction order. No engine > defaults were admitted and no source/engine identity was refreshed. +> **Later 12 September correction:** `6f66fa545` closes the demonstrated G4 +> mutation defect by handing observers detached snapshots, including nested +> table values, metadata, schema, weights and ledgers. Independent review is +> clean; 524 graph tests, five focused cases and all four existing synthetic +> consumers passed, including financial and calibration teardown checks. An +> earlier source-edit-during-test failure remains preserved and is excluded. +> Entry 7.4 below is historical: the current observer need not be trusted to +> leave its snapshot unmodified. Retaining snapshots still costs memory. +> +> `cb27bdb0b` makes the offline explanation display executor-owned unreached +> and gate-exception states independently from cache hits. Sixteen graph +> explanation/actual-gate cases and two free-form diagnostic controls passed. +> Its new `explain → availability` import is recorded in the implementation +> inventory; availability was already bound as a whole module in all ten +> using scopes. No scope, dependency, resource, or unbound-use exemption was +> added. All eight prepared-resource/full-manifest tests passed afterward. +> +> Corrected PR912 head `5e33d6971` is undergoing exact-head CI and Fable review. +> Graph-only extraction is being prepared against that revision, incorporating +> G1–G4 and numbered charter entries. Neither that extraction nor the remaining +> country layers are declared accepted by this historical report. + Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index b769a9365..c20f66b33 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -1174,6 +1174,7 @@ }, "microcosm.graph/explain.py": { "imports": [ + "microcosm.graph.availability", "microcosm.graph.decl", "microcosm.graph.manifest", "microcosm.graph.population", From 0acc786d3963588b326abf92c65baaf072ff0d49 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:51:41 -0400 Subject: [PATCH 105/255] Update joint PUMA fixture and record integration checks --- ...econciliation-amendments-19-20-20260912.md | 18 ++++++++++++++++ .../tests/test_us_multispine_pool_tool.py | 21 +++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md index 210c603a2..f1627eb2b 100644 --- a/experiments/893-reconciliation-amendments-19-20-20260912.md +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -40,6 +40,24 @@ > G1–G4 and numbered charter entries. Neither that extraction nor the remaining > country layers are declared accepted by this historical report. +> **Subsequent 12 September integration checks:** corrected PR912 head +> `5e33d6971` received Fable approval; its merge remains conditional on CI. +> The local integration merged that exact revision in `9e61d0d04` and +> recomputed the combined source/seed identities. All 734 graph and fitting +> tests passed. The six-file spec/build sweep collected 248 tests: 247 passed +> and one failed because its invented PUMA fixture still used schema 1 and +> omitted the required joint tract/CD population support. The fixture now +> supplies consistent schema-2 support and explicit invented provenance; that +> single test passed when rerun. The original failed sweep and intermediate +> incomplete fixture repair are preserved. This is separate-run evidence, +> not a claim that the complete 248-test sweep was rerun successfully. +> +> Shared graph consolidation is now public as PR913, temporarily based on +> PR912, with independent Fable review in progress. Its runtime matches this +> integration's graph implementation. Native strict replay, PUF55 host +> acceptance and complete national/CD release acceptance remain separate +> work and are not established by these software checks. + Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index cc60ec28d..723f18d56 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -320,15 +320,28 @@ def _fixture_puma_ladder(pool_tool: ModuleType): tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint PUMA/tract/CD fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint PUMA/tract/CD fixture", + }, + "tract": { + "vintage": "2020_census", + "source": "invented joint PUMA/tract/CD fixture", + }, }, }, ) From 26e51d123896d43c7ea0179024985b020104c80c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:00:07 -0400 Subject: [PATCH 106/255] Restore PUF55 host over the combined survey population --- changelog.d/puf55-survey-host.added.md | 1 + .../us-puf55-host-adoption-20260912.md | 101 ++ .../graph_puf55_route_attachment.py | 950 ++++++++++++++++++ .../build/us_runtime/graph_survey_puf55.py | 449 +++++++++ .../test_us_graph_puf55_canonical_donor.py | 87 +- .../tests/test_us_graph_survey_puf55.py | 855 ++++++++++++++++ ...t_us_puf55_heterogeneous_manifest_seals.py | 571 +++++++++++ 7 files changed, 3013 insertions(+), 1 deletion(-) create mode 100644 changelog.d/puf55-survey-host.added.md create mode 100644 experiments/us-puf55-host-adoption-20260912.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_route_attachment.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py create mode 100644 packages/microcosm-build/tests/test_us_graph_survey_puf55.py create mode 100644 packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py diff --git a/changelog.d/puf55-survey-host.added.md b/changelog.d/puf55-survey-host.added.md new file mode 100644 index 000000000..f0e410b85 --- /dev/null +++ b/changelog.d/puf55-survey-host.added.md @@ -0,0 +1 @@ +Add the PUF55 host over the combined ACS/ASEC support-clone population, with source-qualified Social Security routes, complete population preservation and required graph replay. diff --git a/experiments/us-puf55-host-adoption-20260912.md b/experiments/us-puf55-host-adoption-20260912.md new file mode 100644 index 000000000..37a15fe78 --- /dev/null +++ b/experiments/us-puf55-host-adoption-20260912.md @@ -0,0 +1,101 @@ +# PUF55 host restoration on the reconciled graph + +This change restores the existing PUF55 composition over an issued nineteen-node +post-clone financial run. The shared ACS/ASEC survey population is cloned before +atomic geography is assigned. Recipient measurement follows financial completion; +a common canonical PUF donor supplies two disjoint conditioning routes, followed +by one placement into the full survey population. The original survey channel, +source identities, memberships, Social Security fields and knownness, geography, +design anchors and household weights are retained. + +`run_survey_puf55` accepts the actual financial run and original PUF source paths. +It requires the financial issuer, source keys, producer implementations and typed +model ancestry. Its explicit `resume="require"` call must hit every graph node. +There are 25 fixed-prefix/finalization nodes plus 110 per nonempty route: 245 +nodes for the two-route fixture. Each route fits and applies 55 targets. The +known survey report total adds a ninth predictor; unknown totals use eight +predictors without a zero fallback. The four Social Security components remain +owned by the survey completion stage. Prior wages are excluded. + +The restoration consists of two new production modules, the existing canonical +donor fixture's opt-in weighted-tail support, the full host controls and the tiny +heterogeneous-manifest controls. It incorporates the later corrected mutation +control, which changes the retained receiving DataFrame and proves its complete +population seal changed. All earlier fixture helpers and newer tests remain. +Ruff formatting/import order is the only difference from the preserved source +candidate's executable AST. No existing production module or implementation +inventory JSON was replaced. The new kernels' implementation hashes bind both +new host modules and their consumed canonical, recipient and numerical owners. + +## Current-source validation + +The source base is `9e61d0d04f5930aa72aef2faf83b2a9c7f2b4284`, including the +reconciled graph and keyed-draw corrections. Tests run against a frozen inventory +of 976 source/configuration files and 15 exact code resources, with no native +input, country engine, network or child process access. Numeric worker counts +are one. Runtime stores contain newly constructed invented inputs only. + +The 28 heterogeneous-manifest controls and three canonical fixture controls pass +together: 31 tests, zero failures/errors/skips, 5.97 wall seconds, 5.61 CPU seconds +and 437 MB peak RSS. The guard's source/owned/resource before-and-after checks +pass. The preceding run also passed its tests but failed its reporting gate +because optional pytest-owned current-symlink attempts were newly refused. That +failed record is preserved. A fresh run kept the symlinks refused and classified +only explicitly bounded attempts within its owned directory as expected. + +The two full-host positive controls pass: two tests, zero failures/errors/skips, +741.87 wall seconds, 734.55 CPU seconds and 739 MB peak RSS. They share one actual +financial fixture, use 64 invented PUF donors and two trees, and execute the +245-node cold extension plus required replay. The direct whole-cohort oracle +checks all outputs and non-owned storage. The finite limits were 1,800 CPU +seconds, 2,700 wall seconds and 8 GiB measured peak RSS. The guard exits zero, +records no unexpected refusals, and confirms all 984 source/owned hashes and +15 resource hashes stayed unchanged. This validates the restored composition on +the current frozen source, not the entire twenty-case host test file. + +An owned diagnostic hook successfully ran the existing NATIONAL_CD input-coverage +operation on the actual attached population within the fixture's checked +lifetime, bracketed by the normal purity and financial-owner checks. The fixture +contains 12 households and 18 people spanning ACS/ASEC and original/clone channels. +Of 161 required input names, 61 are present and 100 are absent; no ambiguous grain +or block-storage issue is reported, and all 12 household block values are known. +The absent roster includes survey demographics and disability, health coverage, +assets, take-up and several survey-owned income names. These include the four +canonical Social Security component names, which this PUF stage deliberately +does not write; the synthetic roster does not establish native input coverage. Prior wages remain excluded. +Known/unknown and writer/carried counts are recorded by source/clone. Applicability, +source ancestry, statistical signal and release eligibility remain unverified. + +The closed host receipt is SHA256 +`d2288ee0e321297875509ee962faef8ee886399a717cad47356921dada8be698`; +the descriptive synthetic coverage is +`d405c7d559a2f2141d83c076d8cef8bbe33f30ba403a7eaedf76cff1161cb7a6`. +Both live under the durable local adoption packet's `run-host2-v2` directory; +`ACCEPTANCE-SUMMARY.json` provides a bounded index without producer payloads. +An independent source review found no actionable defect in the five-file +restoration and confirmed the preserved source pins and executable AST parity. + +CI test-file partition verification passes with 487 tracked files. The new host +file enters fast `rest` and engine `us-am:build`; the heterogeneous-manifest file +enters fast `rest` and engine `us-p`. The canonical fixture file retains its +existing `us-am:build` placement. The classifier and country spec/seed pins are +unchanged. Ruff and staged whitespace checks pass. + +## Native and release work remains separate + +A new native harness must retain the actual financial run in process and provide +a writable store for new PUF artifacts. The retained historical financial replay +store remains read-only. Genuine PUF source validation, conditional-fit quality, +complete applicable input coverage, national/CD calibration, holdouts, pruning +and exact release export/readback are not established by these synthetic tests. +The host emits a development result with release eligibility false. + +`SurveyPuf55Run` is a public frozen dataclass, not a retained issuer. The runner +checks source/artifact ancestry and the complete output immediately before +return, but supplies no post-return `check_run` or issuance registry. The genuine +financial handle retained inside it authenticates the upstream financial run, +not subsequent mutations or copies of the final PUF population. Receipt booleans +cannot grant that authority. A downstream calibration bridge must consume the +actual checked output within an explicitly fenced host operation or implement +and test a retained output check; it cannot infer portable ownership from this +dataclass or receipt. This restoration adds no such admission mechanism. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_route_attachment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_route_attachment.py new file mode 100644 index 000000000..9dcb830c7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_route_attachment.py @@ -0,0 +1,950 @@ +"""One source-bound PUF55 attachment over the complete survey clone cohort. + +This internal seam is constructed only from an actual retained financial run. +It authenticates source and typed producer inputs before trusted model decoding. +Its mutation seals are in-process checks, not a new source/admission authority. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, replace +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + artifact_edges, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import graph_full_puf_enrichment as physical +from . import graph_puf55_canonical_donor as canonical +from . import graph_puf55_survey_recipients as recipient +from . import puf55_route_finalization as numerical + +financial = recipient.financial +full, codec = physical.full, physical.codec +PROFILES = numerical.PROFILES +FILTER_NODE = "survey_puf55.receiving" +MASK_NODE = "survey_puf55.mask" +ATTACH_NODE = "survey_puf55.attach" +PLACEMENT_TYPE = ArtifactType("microcosm.us.survey_puf55.placement", 1) +FINALIZATION_TYPE = ArtifactType("microcosm.us.survey_puf55.finalization", 2) +PROTOCOL = "microcosm.us.survey_puf55.attachment.v1" + + +def require(condition, reason): + if not condition: + raise ValueError("SURVEY_PUF55_" + reason) + + +def _live(): + result = [] + for name in (__name__, __package__ + ".graph_survey_puf55"): + module = sys.modules.get(name) + require(module is not None, "HOST_MODULE") + for key, value in vars(module).items(): + if type(value) is FunctionType: + result.append( + (name, key, financial.values.source._function_seal(value)) + ) + elif isinstance(value, type) and value.__module__ == name: + result.append((name, key, value)) + for member, function in vars(value).items(): + if isinstance(function, (classmethod, staticmethod)): + function = function.__func__ + if type(function) is FunctionType: + result.append( + ( + name, + key, + member, + financial.values.source._function_seal(function), + ) + ) + return tuple(result) + + +def _result_seal(result): + require( + type(result) is KernelResult and type(result.columns) is dict, "RESULT_TYPE" + ) + require( + result.frame + is result.keep + is result.expand + is result.weights + is result.strata + is None + and type(result.artifacts) is dict + and all( + type(k) is str and type(v) is bytes for k, v in result.artifacts.items() + ), + "RESULT_FIELDS", + ) + return ( + tuple( + (e, c, physical._table_stamp(s.to_frame())) + for (e, c), s in sorted(result.columns.items()) + ), + tuple(sorted(result.artifacts.items())), + codec.encode_json(dict(result.receipt)), + ) + + +def _artifact_seal(values): + """Detach both typed provenance and exact immutable payloads before I/O.""" + require(type(values) is dict, "ARTIFACT_VALUES_TYPE") + result = [] + for name, value in sorted(values.items()): + require( + type(name) is str + and type(value) is physical.ArtifactValue + and type(value.payload) is bytes + and type(value.key) is str + and type(value.producer_key) is str, + "ARTIFACT_VALUE_TYPE", + ) + result.append( + ( + name, + value.payload, + codec.encode_json( + { + "type": { + "name": value.type.name, + "schema_version": value.type.schema_version, + }, + "key": value.key, + "producer_key": value.producer_key, + "numerics": artifact_edges.scope_payload(value.numerics), + } + ), + ) + ) + return tuple(result) + + +def _donor_seal(frame, donors): + require( + type(frame) is full.Frame + and type(donors) is tuple + and all(type(donor) is pd.DataFrame for donor in donors), + "DONOR_RESULT_TYPE", + ) + return canonical._frame_seal(frame), tuple(physical._table_stamp(d) for d in donors) + + +@dataclass(frozen=True) +class Route: + profile: full.PufOutputProfile + fits: tuple[Node, ...] + applies: tuple[Node, ...] + + +def route_nodes(profile, *, seed, n_estimators, zero_atol): + """Both training Slices read the one canonical donor's unchanged row axis.""" + require(profile in PROFILES, "PROFILE") + prefix = "survey_puf55." + ("nine" if profile is PROFILES[0] else "eight") + fits = legacy_qrf_train_nodes( + prefix=prefix + ".fit", + population=canonical.CANONICAL_DONOR_NODE, + entity="tax_unit", + predictors=profile.predictors, + targets=profile.targets, + seed=seed, + phase=profile.phase, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + applies = legacy_qrf_apply_matrix_nodes( + prefix=prefix + ".apply", + population=FILTER_NODE, + fit_nodes=fits, + matrix_producer=recipient.MATRIX_NODE, + matrix_artifact=recipient._NAMES[profile.value], + seed=seed, + phase=profile.phase, + ) + physical._profile_chain(profile, fits, applies) + return Route(profile, fits, applies) + + +def _base_edges(profiles): + return ( + ArtifactInput( + "recipient_projection", + recipient.PROJECTION_NODE, + "projection", + recipient.PROJECTION_TYPE, + ), + *( + ArtifactInput( + recipient._NAMES[p.value], + recipient.MATRIX_NODE, + recipient._NAMES[p.value], + recipient.model_input.RECIPIENT_MATRIX_TYPE, + ) + for p in profiles + ), + ) + + +def _donor_edges(): + node = canonical.CANONICAL_DONOR_NODE + return ( + ArtifactInput( + "full_return_source", + node, + "full_return_source", + canonical.source_graph.FULL_RETURN_SOURCE_TYPE, + ), + ArtifactInput( + "canonical_donor", node, "canonical_donor", canonical.CANONICAL_DONOR_TYPE + ), + ArtifactInput( + "donor_projection", + node, + "donor_projection", + canonical.DONOR_PROJECTION_TYPE, + ), + ) + + +def _chain_edges(routes): + result = [] + for r, route in enumerate(routes): + for i, (fit, apply) in enumerate(zip(route.fits, route.applies, strict=True)): + # Every model/history is an explicit typed dependency. Last-model + # equality alone cannot authenticate an earlier executable pickle. + result.extend( + ( + ArtifactInput( + f"r{r}_model_{i:03d}", + fit.id, + "model", + physical.qrf_target.LEGACY_QRF_TARGET_TYPE, + ), + ArtifactInput( + f"r{r}_training_{i:03d}", + fit.id, + "training_state", + codec.TRAINING_STATE_TYPE, + ), + ArtifactInput( + f"r{r}_raw_{i:03d}", apply.id, "raw_draw", codec.RAW_TARGET_TYPE + ), + ArtifactInput( + f"r{r}_apply_{i:03d}", + apply.id, + "apply_state", + physical.graph_legacy_apply_matrix.MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + return tuple(result) + + +def _mask_result(frame): + return KernelResult( + columns={ + (e, physical.MASKS[e]): pd.Series( + mask, index=frame.table(e)[frame.schema.entity_id_column(e)], dtype=bool + ) + for e, mask in physical._masks(frame).items() + } + ) + + +def _keep_result(frame): + entity = frame.schema.person_entity + return KernelResult( + keep=pd.Series( + True, + index=frame.table(entity)[frame.schema.entity_id_column(entity)], + dtype=bool, + ) + ) + + +def _params(qualified, donor_node, routes, seed): + return { + "protocol": PROTOCOL, + "financial_run_sha256": codec.sha( + financial._run_entry(qualified.financial_run)[1] + ), + "recipient_projection_sha256": codec.sha(qualified.receipt), + "route_matrices": tuple((name, codec.sha(p)) for name, p in qualified.matrices), + "donor_recipe": codec.encode_json(dict(donor_node.params)).decode(), + "route_profiles": tuple(r.profile.value for r in routes), + "seed": seed, + } + + +def attachment_nodes(qualified, donor_node, routes, *, seed): + """Declare the checked keep-all rewrite version, mask and single attachment.""" + run = qualified.financial_run + frame = run.financial_population.frame + profiles = tuple(r.profile for r in routes) + params = _params(qualified, donor_node, routes, seed) + survey_sources = tuple( + name for name, _ in financial._run_entry(run)[2].source_items + ) + all_sources = tuple(sorted((*survey_sources, *donor_node.sources))) + keep = Node( + FILTER_NODE, + SurveyPuf55KeepAllKernel.ref, + base=run.financial_population.version, + structural=StructuralDelta.FILTER, + mass="conserve", + inputs=physical._inputs(frame, profile=PROFILES[0]), + params=params, + sources=survey_sources, + artifact_inputs=_base_edges(profiles), + ) + mask = Node( + MASK_NODE, + SurveyPuf55MaskKernel.ref, + population=FILTER_NODE, + inputs=physical._inputs(frame, masks=True, profile=PROFILES[0]), + outputs=tuple(Owned(e, c, "bool") for e, c in physical.MASKS.items()), + sources=all_sources, + params=params, + artifact_inputs=(*_base_edges(profiles), *_donor_edges()), + artifact_outputs=(ArtifactOutput("placement", PLACEMENT_TYPE),), + ) + attach = Node( + ATTACH_NODE, + SurveyPuf55AttachKernel.ref, + population=FILTER_NODE, + inputs=( + *physical._inputs(frame, profile=PROFILES[0]), + *(Slice(e, (c,)) for e, c in physical.MASKS.items()), + ), + outputs=physical._outputs(frame, profile=PROFILES[0]), + sources=all_sources, + params=params, + artifact_inputs=( + *_base_edges(profiles), + *_donor_edges(), + ArtifactInput("placement", MASK_NODE, "placement", PLACEMENT_TYPE), + *_chain_edges(routes), + ), + artifact_outputs=(ArtifactOutput("finalization", FINALIZATION_TYPE),), + ) + return keep, mask, attach + + +class Boundary: + """Private retained live-owner boundary, created before extension execution.""" + + def __init__( + self, + run, + qualified, + donor_kernel, + donor_paths, + *, + seed, + n_estimators, + zero_atol, + ): + require(qualified.financial_run is run, "RECIPIENT_RUN") + self.run, self.entry, self.qualified = run, financial._run_entry(run), qualified + self.qualified_seal = recipient.values._result_stamp(qualified) + self.donor_kernel, self.donor_state = donor_kernel, donor_kernel._state + self.donor_paths = tuple(sorted(donor_paths.items())) + self.seed = seed + names = tuple(name for name, _ in qualified.matrices) + require( + names and names == tuple(p.value for p in PROFILES if p.value in names), + "ROUTES", + ) + self.routes = tuple( + route_nodes(p, seed=seed, n_estimators=n_estimators, zero_atol=zero_atol) + for p in PROFILES + if p.value in names + ) + self.donor_node = canonical._node( + donor_kernel._definition, json.loads(donor_kernel._state[2]) + ) + self.nodes = attachment_nodes( + qualified, self.donor_node, self.routes, seed=seed + ) + # Pure reconstruction from the retained upstream. No issuer/filter run. + kept = run.financial_population.frame.select( + np.ones(run.financial_population.frame.n("person"), dtype=bool) + ) + self.expected = population_ops.patch( + run.financial_population, self.nodes[0], KernelResult(frame=kept) + ) + self.masked = population_ops.patch( + self.expected, self.nodes[1], _mask_result(self.expected.frame) + ) + self.expected_stamp, self.masked_stamp = ( + physical._population_stamp(p) for p in (self.expected, self.masked) + ) + self.live = _live() + self.compiled = self.store = self.kernels = None + self.paths = self.source_keys = self.keys = self.implementations = ( + self.declaration + ) = None + self.configuration = ( + self.run, + self.entry, + self.qualified, + self.qualified_seal, + self.donor_kernel, + self.donor_state, + self.donor_paths, + self.seed, + self.routes, + self.donor_node, + self.nodes, + self.expected, + self.masked, + self.expected_stamp, + self.masked_stamp, + self.live, + ) + self.computed = None + + def bind(self, compiled, store, kernels, paths, source_keys, keys, implementations): + require(self.compiled is None, "REBIND") + self.compiled, self.store, self.kernels = compiled, store, kernels + self.paths = tuple(sorted(paths.items())) + self.source_keys = tuple(sorted(source_keys.items())) + self.keys = tuple(sorted(keys.items())) + self.implementations = tuple(sorted(implementations.items())) + self.declaration = graph_to_json(compiled.graph) + self.kernel_items = tuple(sorted(kernels.as_mapping().items())) + self.codec_items = ( + tuple(sorted(store.codecs.as_mapping().items())), + tuple(sorted(store.codecs.as_bytes_mapping().items())), + ) + self.bound_objects = (compiled, store, kernels, store.codecs) + self.bound_values = ( + self.paths, + self.source_keys, + self.keys, + self.implementations, + self.declaration, + self.kernel_items, + self.codec_items, + store.root, + ) + self.pure() + + def pure(self): + require( + all( + a is b + for a, b in zip( + ( + self.run, + self.entry, + self.qualified, + self.qualified_seal, + self.donor_kernel, + self.donor_state, + self.donor_paths, + self.seed, + self.routes, + self.donor_node, + self.nodes, + self.expected, + self.masked, + self.expected_stamp, + self.masked_stamp, + self.live, + ), + self.configuration, + strict=True, + ) + ), + "RETAINED_CONFIGURATION_CHANGED", + ) + financial._pure_run(self.run, self.entry) + self.donor_kernel._check_state(self.donor_state) + require( + recipient.values._result_stamp(self.qualified) == self.qualified_seal, + "RECIPIENT_CHANGED", + ) + require( + physical._population_stamp(self.expected) == self.expected_stamp + and physical._population_stamp(self.masked) == self.masked_stamp + and _live() == self.live, + "RECEIVING_OR_CODE_CHANGED", + ) + require( + self.nodes + == attachment_nodes( + self.qualified, self.donor_node, self.routes, seed=self.seed + ), + "NODE_CHANGED", + ) + if self.compiled is not None: + require( + all( + a is b + for a, b in zip( + (self.compiled, self.store, self.kernels, self.store.codecs), + self.bound_objects, + strict=True, + ) + ) + and ( + self.paths, + self.source_keys, + self.keys, + self.implementations, + self.declaration, + self.kernel_items, + self.codec_items, + self.store.root, + ) + == self.bound_values + and graph_to_json(self.compiled.graph) == self.declaration + and tuple(sorted(self.kernels.as_mapping().items())) + == self.kernel_items + and ( + tuple(sorted(self.store.codecs.as_mapping().items())), + tuple(sorted(self.store.codecs.as_bytes_mapping().items())), + ) + == self.codec_items, + "HOST_BINDING_CHANGED", + ) + require( + all( + self.compiled.graph.node(node.id) == node + for route in self.routes + for node in (*route.fits, *route.applies) + ), + "ROUTE_DECLARATION_CHANGED", + ) + if self.computed is not None: + require( + _result_seal(self.computed[0]) == self.computed[1], + "DETACHED_RESULT_CHANGED", + ) + + def borrow(self): + """All source/store/owner reads, followed by pure retained-object seals.""" + self.pure() + paths, sources = _source_paths_and_keys( + self.compiled, dict(self.paths), self.store + ) + keys, implementations = _all_node_keys(self.compiled, self.kernels, sources) + fresh = recipient.values.qualify_puf55_survey_recipients(self.run) + self.donor_kernel._read_resources(self.donor_state) + require( + tuple(sorted(paths.items())) == self.paths + and tuple(sorted(sources.items())) == self.source_keys + and tuple(sorted(keys.items())) == self.keys + and tuple(sorted(implementations.items())) == self.implementations + and recipient.values._result_stamp(fresh) == self.qualified_seal, + "SOURCE_IMPLEMENTATION_OR_RECIPIENT_CHANGED", + ) + self.pure() + + def context(self, context, node, population): + self.pure() + require( + tuple(sorted(context.sources.items())) + == tuple((n, p) for n, p in self.paths if n in node.sources), + "CONTEXT_SOURCE_PATHS", + ) + # The maintained projection helper deliberately handles source-free + # contexts. Validate the declared source roster above, then reuse only + # its complete table/weight/strata projection comparison. + physical._context_projection( + replace(context, sources={}), node, population.frame + ) + require( + set(context.artifacts) == {e.name for e in node.artifact_inputs}, + "CONTEXT_ARTIFACT_ROSTER", + ) + keys = dict(self.keys) + values = { + e.name: physical._value( + e, context.artifacts[e.name], producer_key=keys[e.producer] + ) + for e in node.artifact_inputs + } + self.projections(values) + return values + + def stored_artifacts(self, values): + """Bind every declared byte input to its actual computed producer key. + + The kernel context's public ArtifactValue constructor does not issue + producer authority. These reads use the retained actual store and graph + keys, before any numerical helper can decode a trusted model. + """ + seal = _artifact_seal(values) + retained_payloads = {name: payload for name, payload, _ in seal} + keys = dict(self.keys) + for edge in self.nodes[2].artifact_inputs: + value = physical._value( + edge, values[edge.name], producer_key=keys[edge.producer] + ) + payload = self.store.load_bytes(value.key) + require( + type(payload) is bytes and payload == retained_payloads[edge.name], + "STORED_ARTIFACT_BYTES", + ) + require(_artifact_seal(values) == seal, "ARTIFACT_CHANGED_DURING_STORE_READ") + self.pure() + + def projections(self, values): + require( + values["recipient_projection"].payload == self.qualified.receipt, + "RECIPIENT_PROJECTION", + ) + for name, payload in self.qualified.matrices: + require( + values[recipient._NAMES[name]].payload == payload, "RECIPIENT_MATRIX" + ) + + def canonical_donor(self, values): + """Reconstruct from the actual pinned originals, before any pickle read.""" + state = self.donor_state + definition, _, _, _, seed, scheme, *_ = state + ordered = tuple( + (p.source_name, dict(self.donor_paths)[p.source_name]) + for p in (definition.main, definition.demographic) + ) + buffers = self.donor_kernel._read_sources(ordered, state) + self.donor_kernel._read_resources(state) + decoded = canonical.source.decode_full_puf_source(*buffers, definition) + raw_payload = canonical.source.encode_full_puf_source(decoded) + constructed = canonical.canonical.construct_canonical_puf59( + decoded, + interest_bands=canonical.interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=canonical.canonical.INTEREST_ASSET_SHA256, + seed=seed, + growth_scheme=scheme, + ) + payload = canonical.envelope.encode_canonical_puf59( + constructed, expected_growth_scheme=scheme + ) + require( + values["full_return_source"].payload == raw_payload + and values["canonical_donor"].payload == payload, + "CANONICAL_SOURCE_RECONSTRUCTION", + ) + donors = tuple( + canonical.projection.canonical_puf55_donor_from_artifact( + payload, + expected_artifact_sha256=codec.sha(payload), + expected_growth_scheme=scheme, + profile=route.profile, + )[0] + for route in self.routes + ) + shared, evidence = canonical.projection.canonical_puf55_donor_from_artifact( + payload, + expected_artifact_sha256=codec.sha(payload), + expected_growth_scheme=scheme, + profile=PROFILES[0], + ) + frame = numerical._model_donor_frame( + full._validated_model_donor(shared, profile=PROFILES[0]) + ) + ids = decoded.status["RECID"][decoded.ordinary] + require( + np.array_equal(frame.table("tax_unit").index.to_numpy(), ids), + "DONOR_SOURCE_AXIS", + ) + require( + frame.weights_for("tax_unit").values.tobytes() + == ( + decoded.status["S006"][decoded.ordinary].astype(np.float64) / 100 + ).tobytes(), + "DONOR_SOURCE_WEIGHTS", + ) + receipt = codec.decode_json(values["donor_projection"].payload) + require( + receipt["protocol"] == canonical.PHASE + and receipt["projection"] == evidence + and receipt["model_frame_sha256"] == canonical._frame_seal(frame) + and receipt["full_return_source_sha256"] == codec.sha(raw_payload) + and receipt["canonical_donor_sha256"] == codec.sha(payload) + and all(receipt[k] == v for k, v in json.loads(state[2]).items()), + "DONOR_PROJECTION", + ) + # Carry this immutable seal through the function-return boundary. The + # model Frame omits incidence capacity, so seal complete donor tables too. + seal = _donor_seal(frame, donors) + self.pure() + require(_donor_seal(frame, donors) == seal, "DONOR_RESULT_CHANGED") + return frame, donors, seal + + def placement(self): + frame = self.expected.frame + return codec.encode_json( + { + **dict(self.nodes[1].params), + "selected_ids": { + e: frame.table(e).loc[m, frame.schema.entity_id_column(e)].tolist() + for e, m in physical._masks(frame).items() + }, + "target_order": list(PROFILES[0].targets), + "release_eligible": False, + } + ) + + def histories(self, values): + """Check every typed model's byte hash against both recorded histories.""" + for r, route in enumerate(self.routes): + models, raw = [], [] + for i, target in enumerate(route.profile.targets): + model = values[f"r{r}_model_{i:03d}"].payload + packet, _ = codec.read_training( + values[f"r{r}_training_{i:03d}"].payload + ) + app = physical.graph_legacy_apply_matrix.decode_matrix_apply_state( + values[f"r{r}_apply_{i:03d}"].payload + ) + require( + len(packet["models"]) == i + 1 + and packet["models"][:i] == models + and packet["models"][-1]["target"] == target + and packet["models"][-1]["sha256"] == codec.sha(model), + "TRAINING_HISTORY", + ) + models = packet["models"] + require(app["application"]["models"] == models, "APPLY_MODEL_HISTORY") + raw.append((target, values[f"r{r}_raw_{i:03d}"].payload)) + # The maintained raw merger checks each target's chain, matrix, + # phase/seed and complete entity-ID coverage without deserialization. + yield numerical.Puf55RouteDraws( + route.profile, + values[recipient._NAMES[route.profile.value]].payload, + dict(self.keys)[recipient.MATRIX_NODE], + tuple(raw), + values[f"r{r}_apply_{len(route.applies) - 1:03d}"].payload, + values[f"r{r}_training_{len(route.fits) - 1:03d}"].payload, + ) + + def finalize(self, values): + require( + set(values) == {e.name for e in self.nodes[2].artifact_inputs}, + "FINAL_ARTIFACT_ROSTER", + ) + keys = dict(self.keys) + values = { + e.name: physical._value(e, values[e.name], producer_key=keys[e.producer]) + for e in self.nodes[2].artifact_inputs + } + artifact_seal = _artifact_seal(values) + self.borrow() + self.stored_artifacts(values) + self.projections(values) + require(values["placement"].payload == self.placement(), "PLACEMENT") + donor_frame, donors, donor_seal = self.canonical_donor(values) + require(_donor_seal(donor_frame, donors) == donor_seal, "DONOR_RESULT_CHANGED") + require( + _artifact_seal(values) == artifact_seal, "ARTIFACT_CHANGED_BEFORE_DECODE" + ) + self.pure() + draws = tuple(self.histories(values)) + # JSON/history decoders are also callback boundaries. No trusted pickle + # is decoded until the retained source-derived inputs still match. + require( + _artifact_seal(values) == artifact_seal, "ARTIFACT_CHANGED_BEFORE_DECODE" + ) + require(_donor_seal(donor_frame, donors) == donor_seal, "DONOR_RESULT_CHANGED") + self.pure() + # Full receiving Population stamps above and below close the values-only + # numerical helper's deliberately narrower receiving-axis boundary. + candidate, evidence = numerical._finalize_puf55_routes( + self.expected.frame, + recipient_matrices=self.qualified.matrices, + routes=tuple( + numerical.Puf55RouteFinalizationInput( + draw, + donor, + values[f"r{r}_model_{len(route.fits) - 1:03d}"].payload, + route.profile.phase, + ) + for r, (route, draw, donor) in enumerate( + zip(self.routes, draws, donors, strict=True) + ) + ), + seed=self.seed, + ) + require(type(evidence) is bytes, "NUMERICAL_RECEIPT_TYPE") + numerical_receipt = codec.decode_json(evidence) + candidate_stamp = numerical_receipt.get("candidate_frame_sha256") + protocol = numerical_receipt.get("protocol") + # The decoder's mutable mapping is descriptive. Capture exact scalar + # literals and compare its canonical bytes with the retained receipt. + require( + type(protocol) is str + and protocol == numerical.FINALIZATION_PROTOCOL + and type(candidate_stamp) is str + and codec.encode_json(numerical_receipt) == evidence, + "NUMERICAL_RECEIPT_BINDING", + ) + require( + candidate_stamp == numerical._candidate_frame_sha256(candidate), + "NUMERICAL_OUTPUT_CHANGED", + ) + # The later-I/O baseline is the immutable digest made by the callee, + # never a first stamp of possibly changed post-return candidate values. + columns = {} + for owned in self.nodes[2].outputs: + mask = physical._masks(self.expected.frame)[owned.entity] + table = candidate.table(owned.entity) + selected = table.loc[mask, owned.column] + require( + selected.notna().all() + and np.isfinite(selected.to_numpy(dtype=np.float64)).all(), + "FINALIZED_VALUES", + ) + columns[owned.entity, owned.column] = pd.Series( + selected.astype(owned.dtype).array, + index=table.loc[mask, candidate.schema.entity_id_column(owned.entity)], + dtype=owned.dtype, + ) + # KernelResult/receipt constructors are detached-output boundaries too. + # Retain selected-column stamps before giving the mutable dict to either. + column_seal = tuple( + (e, c, physical._table_stamp(series.to_frame())) + for (e, c), series in sorted(columns.items()) + ) + result = KernelResult( + columns=columns, + artifacts={"finalization": evidence}, + receipt=codec.decode_json(evidence), + ) + require( + result.artifacts == {"finalization": evidence} + and codec.encode_json(dict(result.receipt)) == evidence, + "NUMERICAL_RECEIPT_BINDING", + ) + seal = _result_seal(result) + require(seal[0] == column_seal, "NUMERICAL_COLUMN_BINDING") + # The first complete seal must bind both mutable metadata containers to + # the immutable callee receipt, even if the earlier encoder return was + # a mutation boundary. Never adopt their current values as authority. + require( + seal[1:] == ((("finalization", evidence),), evidence), + "NUMERICAL_RECEIPT_BINDING", + ) + # Last I/O, then only immutable baselines and complete physical checks. + self.borrow() + require( + numerical._candidate_frame_sha256(candidate) == candidate_stamp + and _donor_seal(donor_frame, donors) == donor_seal + and _artifact_seal(values) == artifact_seal + and _result_seal(result) == seal, + "FINALIZATION_OUTPUT_CHANGED", + ) + self.pure() + if self.computed is not None: + require(self.computed[1] == seal, "REPEATED_FINALIZATION_CHANGED") + self.computed = (result, seal) + return result + + +class _Kernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, boundary): + self.boundary = boundary + + def implementation_hash(self): + # Code and package resources are freshly bound on cache hits as well. + return codec.sha( + codec.encode_json( + { + "attachment": source_hash( + sys.modules[__name__], + sys.modules[__package__ + ".graph_survey_puf55"], + physical, + numerical, + recipient, + recipient.values, + *canonical._modules(), + dependencies=self.capabilities.dependencies, + ), + "financial": recipient._Kernel.implementation_hash(self), + } + ) + ) + + +class SurveyPuf55KeepAllKernel(_Kernel): + ref = "us.survey_puf55.keep_all@1" + capabilities = replace(_Kernel.capabilities, structural=StructuralDelta.FILTER) + + def run(self, context): + b = self.boundary + b.context(context, b.nodes[0], b.run.financial_population) + fresh = recipient.values.qualify_puf55_survey_recipients(b.run) + require( + recipient.values._result_stamp(fresh) == b.qualified_seal, + "FILTER_RECIPIENT", + ) + result = _keep_result(b.run.financial_population.frame) + seal = physical._table_stamp(result.keep.to_frame()) + financial.check_atomic_survey_financial_run(b.run) + b.context(context, b.nodes[0], b.run.financial_population) + require(physical._table_stamp(result.keep.to_frame()) == seal, "KEEP_CHANGED") + return result + + +class SurveyPuf55MaskKernel(_Kernel): + ref = "us.survey_puf55.mask@1" + + def run(self, context): + b = self.boundary + b.borrow() + b.context(context, b.nodes[1], b.expected) + result = _mask_result(b.expected.frame) + result.artifacts["placement"] = b.placement() + seal = _result_seal(result) + b.borrow() + b.context(context, b.nodes[1], b.expected) + require(_result_seal(result) == seal, "MASK_CHANGED") + return result + + +class SurveyPuf55AttachKernel(_Kernel): + ref = "us.survey_puf55.attach@1" + + def run(self, context): + b = self.boundary + values = b.context(context, b.nodes[2], b.masked) + result = b.finalize(values) + b.context(context, b.nodes[2], b.masked) + b.pure() + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py new file mode 100644 index 000000000..5bfd0debb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py @@ -0,0 +1,449 @@ +"""Extend an issued postclone nineteen-node financial run with two PUF55 routes. + +The existing financial run is an explicit prerequisite. This host compiles and +executes the extension once; it does not claim that the upstream run's entire +lifetime used one execution. No donor or FILTER issuer is run in preparation. +Required replay is a separate explicit call over the same source paths/store. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ContentStore, + KernelRegistry, + artifact_edges, + codecs, + compile_graph, + run_graph, +) +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys + +from . import graph_puf55_route_attachment as attach + +require = attach.require +financial, recipient, canonical = attach.financial, attach.recipient, attach.canonical +physical, codec, population_ops = attach.physical, attach.codec, attach.population_ops + + +@dataclass(frozen=True) +class SurveyPuf55Run: + """Checked result values, not financial/weight/release admission evidence.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple[tuple[str, Path], ...] + receipt: bytes + + +def _registry(existing, donor): + require(type(existing) is codecs.SourceCodecRegistry, "UPSTREAM_CODEC_REGISTRY") + registry = codecs.SourceCodecRegistry() + for old in (existing, donor): + for name, loader in old.as_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register(name, loader) + for name, loader in old.as_bytes_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register_bytes(name, loader) + return registry + + +def _construct( + financial_run, donor_sources, *, fixture_definition, seed, n_estimators, zero_atol +): + """Construct actual declarations and bind actual source keys without running.""" + financial.check_atomic_survey_financial_run(financial_run) + require(len(financial_run.compiled.order) == 19, "UPSTREAM_NINETEEN_NODES") + require(type(seed) is int and 0 <= seed < 2**64, "SEED") + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + qualified = recipient.values.qualify_puf55_survey_recipients(financial_run) + donor = canonical.CanonicalPuf55DonorKernel( + seed=seed, fixture_definition=fixture_definition + ) + require( + type(donor_sources) is dict + and set(donor_sources) == {s.name for s in donor.source_refs}, + "DONOR_SOURCE_ROSTER", + ) + donor_paths = { + name: Path(path).resolve(strict=True) for name, path in donor_sources.items() + } + require(not set(donor_paths) & set(financial_run.sources), "SOURCE_NAME_COLLISION") + boundary = attach.Boundary( + financial_run, + qualified, + donor, + donor_paths, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + recipient_nodes = recipient.puf55_survey_recipient_nodes(qualified) + nodes = ( + *financial_run.compiled.graph.nodes, + *recipient_nodes, + boundary.donor_node, + boundary.nodes[0], + *(node for route in boundary.routes for node in (*route.fits, *route.applies)), + *boundary.nodes[1:], + ) + graph = replace( + financial_run.compiled.graph, + sources=(*financial_run.compiled.graph.sources, *donor.source_refs), + nodes=nodes, + ) + compiled = compile_graph(graph) + require(len(compiled.order) == 25 + 110 * len(boundary.routes), "NODE_COUNT") + order = compiled.order + require( + order.index(financial.financial.ATTACH_NODE) + < order.index(recipient.PROJECTION_NODE) + < order.index(recipient.MATRIX_NODE) + < order.index(attach.FILTER_NODE) + < order.index(attach.MASK_NODE) + < order.index(attach.ATTACH_NODE), + "EXTENSION_ORDER", + ) + # Private registries preserve the original issued run's retained containers. + kernels = KernelRegistry() + for kernel in financial_run.kernels.as_mapping().values(): + kernels.register(kernel) + for kernel in ( + recipient.Puf55SurveyRecipientProjectionKernel(financial_run), + recipient.Puf55SurveyRecipientMatrixKernel(financial_run), + donor, + attach.SurveyPuf55KeepAllKernel(boundary), + attach.SurveyPuf55MaskKernel(boundary), + attach.SurveyPuf55AttachKernel(boundary), + ): + require(kernel.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(kernel) + for kernel in (LegacyQRFTrainKernel(), LegacyQRFApplyMatrixKernel()): + if kernel.ref not in kernels.refs(): + kernels.register(kernel) + else: + require(type(kernels.get(kernel.ref)) is type(kernel), "LEGACY_KERNEL_TYPE") + store = ContentStore( + financial_run.store.root, + codecs=_registry(financial_run.store.codecs, donor.source_codecs), + ) + paths, sources = _source_paths_and_keys( + compiled, {**financial_run.sources, **donor_paths}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, sources) + # The preserved prefix must have the actual issued keys, not merely the + # same user-facing IDs. New donor sources are unused by the original nodes. + state = financial._run_entry(financial_run)[2] + require( + all(keys[n] == k for n, k in state.keys) + and all(implementations[n] == k for n, k in state.implementations), + "UPSTREAM_PRODUCER_KEYS", + ) + boundary.bind(compiled, store, kernels, paths, sources, keys, implementations) + boundary.borrow() + return boundary + + +def _heterogeneous_manifest_seals(manifest, compiled): + """Seal every attached version of this mixed-schema run, donor included. + + The survey helper stamps through ``source._frame_identity``, which requires + the six US_SCHEMA entity groups. This host attaches a tax_unit-only PUF + donor CREATE, so that helper cannot seal this manifest at all. The physical + seal is schema-aware and already seals exactly this donor frame elsewhere + (``canonical._frame_seal``). It records link *names* but never link bodies, + so a frame carrying link tables is refused here rather than sealed blind. + Reconstructed versions carry default owners and anchors derived from their + stored weights. This manifest-content seal does not recover independently + retained original owner/anchor/history evidence. + """ + seals = [] + for version in sorted(set(compiled.versions.values())): + frame = manifest.population(version) + require(not frame.links, "MANIFEST_SEAL_LINK_TABLES") + seals.append( + ( + version, + physical._population_stamp( + population_ops.Population.from_frame( + frame, version, mass_ledger=manifest.mass_ledger(version) + ) + ), + ) + ) + return tuple(seals) + + +def _check_replayed_survey_manifest(expected_manifest, actual_manifest, compiled): + """Compare an exact upstream survey roster across store representations. + + The caller keeps the original issued manifest and its physical lifetime + seals. These Frame/ledger views reconstruct the same default owners and + design anchors as the former manifest stamp; they do not recover true + execution ownership. Actual observed Populations are checked separately. + Manifest access may materialize lazy attachments, so the host brackets this + comparison with its retained-owner checks and keeps its final physical seal. + """ + versions = tuple(sorted(set(compiled.versions.values()))) + wanted = set(versions) + require( + expected_manifest.country + == actual_manifest.country + == compiled.graph.country + == "us", + "UPSTREAM_SURVEY_COUNTRY", + ) + require( + bool(versions) + and set(expected_manifest.nodes) == set(compiled.order) + and set(compiled.order) <= set(actual_manifest.nodes) + and set(expected_manifest.populations) == wanted + and set(expected_manifest.mass_ledgers) == wanted + and wanted <= set(actual_manifest.populations) + and wanted <= set(actual_manifest.mass_ledgers), + "UPSTREAM_SURVEY_ROSTER", + ) + for version in versions: + expected = expected_manifest.population(version) + actual = actual_manifest.population(version) + require(expected.schema == actual.schema == US_SCHEMA, "UPSTREAM_SURVEY_SCHEMA") + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + expected, version, mass_ledger=expected_manifest.mass_ledger(version) + ), + population_ops.Population.from_frame( + actual, version, mass_ledger=actual_manifest.mass_ledger(version) + ), + ) + + +def _loaded_values(boundary, manifest, loaded, node): + """Only construct typed inputs from the already-checked real manifest.""" + record = manifest.node(node.id) + return { + edge.name: artifact_edges.value_from_descriptor( + loaded[edge.producer, edge.artifact], + record.typed_artifacts["inputs"][edge.name], + ) + for edge in node.artifact_inputs + } + + +def _reconstruct(boundary, observed, values, result): + """Use original retained upstream and independently source-derived donor.""" + run = boundary.run + # The original issuer retains final views per structural version. Its + # existing nineteen-node keys/types are checked above; the complete retained + # final views below bind the unchanged prefix, without inventing a second + # upstream observer history or re-executing a preparation graph. + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.PROJECTION_NODE] + ) + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.MATRIX_NODE] + ) + physical.replay.same_replayed_population( + boundary.expected, observed[attach.FILTER_NODE] + ) + donor_frame, donors, donor_seal = boundary.canonical_donor(values) + require( + attach._donor_seal(donor_frame, donors) == donor_seal, "DONOR_RESULT_CHANGED" + ) + donor_node = boundary.donor_node + # CREATE assigns every loaded column, including linkage, to this version. + donor_population = population_ops.Population.from_frame(donor_frame, donor_node.id) + physical.replay.same_replayed_population(donor_population, observed[donor_node.id]) + for route in boundary.routes: + for node in route.fits: + physical.replay.same_replayed_population( + donor_population, observed[node.id] + ) + for node in route.applies: + # Ordinary fit/apply nodes own no Population cells. Compiler order + # can put the independent mask before any apply, so compare with + # the preceding state of this version, not an assumed fixed order. + require(not node.outputs and node.weights is None, "CHAIN_POPULATION_WRITE") + current = {} + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id == donor_node.id: + expected = donor_population + elif node_id == attach.FILTER_NODE: + expected = boundary.expected + elif node_id == attach.MASK_NODE: + expected = population_ops.patch( + current[version], node, attach._mask_result(boundary.expected.frame) + ) + elif node_id == attach.ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + return current[attach.FILTER_NODE] + + +def run_survey_puf55( + financial_run, + *, + donor_sources, + seed=578, + n_estimators=100, + zero_atol=1e-8, + fixture_definition=None, + resume="auto", +): + """Run one extension over an actual issued run; no pre-run or donor injection. + + ``fixture_definition`` is the canonical owner's explicit invented-source + route. It cannot claim packaged-source admission. A separate call with + ``resume='require'`` must reproduce every source/producer and all values. + """ + require(resume in ("auto", "require"), "RESUME") + boundary = _construct( + financial_run, + donor_sources, + fixture_definition=fixture_definition, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require( + all(manifest.node(n).hit for n in boundary.compiled.order), "REQUIRED_HITS" + ) + # Complete source-key/implementation/type ancestry before loading any + # trusted model or independently invoking the replay numerical verifier. + boundary.borrow() + loaded = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + # The mixed-manifest physical baseline includes donor and survey versions + # and is captured once. Cross-store comparison uses the still-issued + # upstream manifest as expected, retaining its original lifetime seals. + full_seals = _heterogeneous_manifest_seals(manifest, boundary.compiled) + boundary.pure() + _check_replayed_survey_manifest( + financial_run.manifest, manifest, financial_run.compiled + ) + boundary.pure() + values = _loaded_values(boundary, manifest, loaded, boundary.nodes[2]) + boundary.projections(values) + if boundary.computed is None: + result = boundary.finalize(values) + else: + boundary.pure() + result = boundary.computed[0] + require( + loaded[attach.ATTACH_NODE, "finalization"] == result.artifacts["finalization"], + "FINALIZATION_ARTIFACT", + ) + expected = _reconstruct(boundary, observed, values, result) + population = observed[attach.ATTACH_NODE] + physical.replay.same_replayed_population(expected, population) + receipt = codec.encode_json( + { + "protocol": attach.PROTOCOL, + "node_count": len(boundary.compiled.order), + "profiles": [r.profile.value for r in boundary.routes], + "financial_run_sha256": codec.sha(boundary.entry[1]), + "manifest_key": manifest.key, + "one_extension_execution": True, + "upstream_already_issued": True, + "complete_population_compared": True, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + output = SurveyPuf55Run( + financial_run, + population, + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + manifest_bytes = manifest.to_json() + expected_stamp = physical._population_stamp(expected) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + # Last reads include actual source owners/code, manifest store identities, + # and every returned Population. Nothing below the fence performs I/O. + boundary.borrow() + fresh = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes, + "LATE_ARTIFACT_CHANGED", + ) + boundary.pure() + require( + output.financial_run is financial_run + and output.population is population + and output.manifest is manifest + and output.compiled is boundary.compiled + and output.store is boundary.store + and output.kernels is boundary.kernels + and output.sources == boundary.paths + and output.receipt == receipt + and manifest.to_json() == manifest_bytes + and _heterogeneous_manifest_seals(manifest, boundary.compiled) == full_seals + and physical._population_stamp(expected) == expected_stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT_CHANGED", + ) + physical.replay.same_replayed_population(expected, population) + return output diff --git a/packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py b/packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py index 7a160046b..bb76c800f 100644 --- a/packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py +++ b/packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py @@ -30,7 +30,7 @@ from microcosm.graph.keys import opaque_artifact_key -def _original_sources(root, *, defect=None): +def _original_sources(root, *, defect=None, full_finalization_support=False): """Encode complete invented deliveries with real pinned source definitions.""" packaged = owner.raw.packaged_definition() rows = [] @@ -77,6 +77,10 @@ def _original_sources(root, *, defect=None): E03230="100", E87530="300", ) + if full_finalization_support and int(row["S006"]) > 0: + # Full-chain finalization needs positive weighted support for + # its configured capital-gain distribution tail bound. + row["E01100"] = str(100 + i) rows.append(row) if defect == "duplicate": rows[0]["RECID"] = rows[1]["RECID"] @@ -130,6 +134,87 @@ def _original_sources(root, *, defect=None): return definition, paths, (main, demo) +@pytest.mark.parametrize( + "case", ("default_unsupported", "supported", "positive_only_zero_weight") +) +def test_original_source_fixture_weighted_tail_support(tmp_path, case): + target = "non_sch_d_capital_gains" + support = owner.full.support + configuration = support.puf_tax_detail_tail_bound_quantiles_identity() + assert configuration == {target: 0.999} + assert owner.source.DIRECT_MAPPINGS[target] == "E01100" + assert all(target in profile.targets for profile in owner.numerical.PROFILES) + + baseline_definition, _, baseline_buffers = _original_sources(tmp_path / "default") + definition, paths, buffers = _original_sources( + tmp_path / "selected", full_finalization_support=case == "supported" + ) + for pin, payload in zip( + (definition.main, definition.demographic), buffers, strict=True + ): + assert paths[pin.source_name].read_bytes() == payload + assert pin.bytes == len(payload) + assert pin.sha256 == hashlib.sha256(payload).hexdigest() + if case != "supported": + # Explicit false keeps the default fixture's exact delivered bytes. + assert buffers == baseline_buffers + assert definition.canonical == baseline_definition.canonical + assert buffers[1] == baseline_buffers[1] + + baseline = owner.source.decode_full_puf_source( + *baseline_buffers, baseline_definition + ) + decoded = owner.source.decode_full_puf_source(*buffers, definition) + assert decoded.aggregate_tokens == baseline.aggregate_tokens + assert decoded.status.keys() == baseline.status.keys() + assert decoded.values.keys() == baseline.values.keys() + for name in decoded.status: + np.testing.assert_array_equal(decoded.status[name], baseline.status[name]) + for name in decoded.values: + if name != "E01100": + np.testing.assert_array_equal(decoded.values[name], baseline.values[name]) + np.testing.assert_array_equal( + decoded.values["E01100"][~decoded.ordinary], + baseline.values["E01100"][~baseline.ordinary], + ) + + observed, _, status = owner.source.observed_and_derived_return_columns(decoded) + np.testing.assert_array_equal(status["RECID"], np.arange(501000, 501064)) + expected_weights = np.array([0, *range(126, 189)], dtype=np.float64) / 100 + weights = status["S006"].astype(np.float64) / 100 + np.testing.assert_array_equal(weights, expected_weights) + values = observed[target].astype(np.float64, copy=True) + expected_values = ( + np.array([0, *range(101, 164)], dtype=np.float64) + if case == "supported" + else np.zeros(64, dtype=np.float64) + ) + np.testing.assert_array_equal(values, expected_values) + np.testing.assert_array_equal( + observed[target], decoded.values["E01100"][decoded.ordinary] + ) + if case == "positive_only_zero_weight": + # A detached negative numerical input; never alter source/donor bytes. + values[0] = 100 + assert (values > 0).any() + assert (weights[values > 0] == 0).all() + np.testing.assert_array_equal(observed[target], np.zeros(64)) + if case == "supported": + assert np.isfinite(values).all() + assert ((values > 0) & (weights > 0)).sum() == 63 + cap = support._weighted_positive_donor_quantile( + values, weights, configuration[target] + ) + assert np.isfinite(cap) and 0 < cap <= values.max() + else: + with pytest.raises( + ValueError, match=r"^PUF tail-bound donor has no positive donor support\.$" + ): + support._weighted_positive_donor_quantile( + values, weights, configuration[target] + ) + + def _context(kernel, paths): node = owner._node(kernel._definition, json.loads(kernel._state[2])) return KernelContext( diff --git a/packages/microcosm-build/tests/test_us_graph_survey_puf55.py b/packages/microcosm-build/tests/test_us_graph_survey_puf55.py new file mode 100644 index 000000000..6b481a5b2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_survey_puf55.py @@ -0,0 +1,855 @@ +"""Actual invented originals, one extension, two routes and required replay. + +The existing financial fixture is an actual issued postclone nineteen-node run. Original +PUF CSV construction is reused without running its CREATE fixture beforehand. +""" + +import sys +from _thread import get_ident +from contextlib import contextmanager +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_puf55_canonical_donor import _original_sources +from test_us_puf55_survey_recipients import recipient_financial_run # noqa: F401 + +from microcosm.build.us_runtime import graph_survey_puf55 as graph +from microcosm.graph import StructuralDelta +from microcosm.graph.keys import opaque_artifact_key + + +@pytest.fixture(scope="module") +def composed(recipient_financial_run, tmp_path_factory): # noqa: F811 + upstream = recipient_financial_run + root = tmp_path_factory.mktemp("survey_puf55_composition") + definition, paths, _ = _original_sources( + root / "original_puf", full_finalization_support=True + ) + boundaries, finalizations, contexts = [], [], [] + + # Code-local events preserve the exact captured objects without a Python + # profile callback on every call/return throughout the 245-node graph. + monitoring, tool_id = sys.monitoring, 4 + functions = ( + graph.attach.SurveyPuf55KeepAllKernel.run, + graph.attach.SurveyPuf55MaskKernel.run, + graph.attach.SurveyPuf55AttachKernel.run, + graph.attach.numerical._finalize_puf55_routes, + graph._construct, + ) + codes = tuple(function.__code__ for function in functions) + assert len({id(code) for code in codes}) == 5 + + def started(code, instruction_offset): + frame = sys._getframe(1) + assert frame.f_code is code + if code is codes[3]: + finalizations.append(frame.f_locals["frame"]) + else: + assert any(code is expected for expected in codes[:3]) + contexts.append(frame.f_locals["context"]) + + def returned(code, instruction_offset, result): + assert code is codes[4] + if result is not None: + boundaries.append(result) + + previous = sys.getprofile() + # Refuse an occupied slot; never clear or borrow another tool's state. + assert monitoring.get_tool(tool_id) is None + monitoring.use_tool_id(tool_id, "microcosm-postclone-puf55-object-observer") + try: + assert monitoring.get_events(tool_id) == 0 + monitoring.register_callback(tool_id, monitoring.events.PY_START, started) + monitoring.register_callback(tool_id, monitoring.events.PY_RETURN, returned) + for code in codes[:4]: + monitoring.set_local_events(tool_id, code, monitoring.events.PY_START) + monitoring.set_local_events(tool_id, codes[4], monitoring.events.PY_RETURN) + cold = graph.run_survey_puf55( + upstream.run, + donor_sources=paths, + fixture_definition=definition, + seed=578, + n_estimators=2, + zero_atol=0, + ) + warm = graph.run_survey_puf55( + upstream.run, + donor_sources=paths, + fixture_definition=definition, + seed=578, + n_estimators=2, + zero_atol=0, + resume="require", + ) + finally: + # Clear local events and callbacks even after setup/run failure, then + # release only this fixture's claimed tool ID. No global events used. + try: + # CPython 3.14.4 retains local-event getter bits after clear alone. + # Zero each of our exact local masks before clearing callbacks. + for code in codes: + monitoring.set_local_events(tool_id, code, 0) + monitoring.clear_tool_id(tool_id) + assert monitoring.get_events(tool_id) == 0 + assert all( + monitoring.get_local_events(tool_id, code) == 0 for code in codes + ) + finally: + monitoring.free_tool_id(tool_id) + assert monitoring.get_tool(tool_id) is None + assert sys.getprofile() is previous + assert all( + function.__code__ is code + for function, code in zip(functions, codes, strict=True) + ) + assert len(boundaries) == len(finalizations) == 2 + assert len(contexts) == 3 # Required replay calls no attachment kernels. + assert finalizations[0] is boundaries[0].expected.frame + assert finalizations[1] is boundaries[1].expected.frame + yield SimpleNamespace( + upstream=upstream, + cold=cold, + warm=warm, + boundary=boundaries[0], + definition=definition, + paths=paths, + contexts={context.node.id: context for context in contexts}, + ) + for boundary in boundaries: + boundary.pure() + graph.financial.check_atomic_survey_financial_run(upstream.run) + + +@contextmanager +def _observe_exact_code_events(trace, *, starts=(), returns=()): + """Test-only regular-Python events on this thread, preserving unwind returns. + + PY_UNWIND is global-only; ignore unlisted codes before acquiring frames. + Existing profilers and other monitoring tools remain untouched. + """ + monitoring, tool_id = sys.monitoring, 4 + functions = tuple(dict.fromkeys((*starts, *returns))) + codes = tuple(function.__code__ for function in functions) + start_codes = tuple(function.__code__ for function in starts) + return_codes = tuple(function.__code__ for function in returns) + assert functions and len({id(code) for code in codes}) == len(codes) + # No generators/coroutines: their resume/yield events need another contract. + assert not any(code.co_flags & (0x20 | 0x80 | 0x200) for code in codes) + thread_id = get_ident() + previous = sys.getprofile() + + def started(code, instruction_offset): + if get_ident() != thread_id: + return + frame = sys._getframe(1) + assert frame.f_code is code + assert any(code is selected for selected in start_codes) + trace(frame, "call", None) + + def returned(code, instruction_offset, result): + if get_ident() != thread_id: + return + frame = sys._getframe(1) + assert frame.f_code is code + assert any(code is selected for selected in return_codes) + trace(frame, "return", result) + + def unwound(code, instruction_offset, exception): + if get_ident() != thread_id or not any( + code is selected for selected in return_codes + ): + return + frame = sys._getframe(1) + assert frame.f_code is code + trace(frame, "return", None) + + assert monitoring.get_tool(tool_id) is None + monitoring.use_tool_id(tool_id, "microcosm-puf55-negative-exact-events") + try: + assert monitoring.get_events(tool_id) == 0 + assert all(monitoring.get_local_events(tool_id, code) == 0 for code in codes) + if start_codes: + assert ( + monitoring.register_callback( + tool_id, monitoring.events.PY_START, started + ) + is None + ) + if return_codes: + assert ( + monitoring.register_callback( + tool_id, monitoring.events.PY_RETURN, returned + ) + is None + ) + assert ( + monitoring.register_callback( + tool_id, monitoring.events.PY_UNWIND, unwound + ) + is None + ) + for code in codes: + mask = 0 + if any(code is selected for selected in start_codes): + mask |= monitoring.events.PY_START + if any(code is selected for selected in return_codes): + mask |= monitoring.events.PY_RETURN + monitoring.set_local_events(tool_id, code, mask) + if return_codes: + monitoring.set_events(tool_id, monitoring.events.PY_UNWIND) + yield + finally: + try: + # Explicit zeroing matches the earlier 3.14.4 cleanup correction. + monitoring.set_events(tool_id, 0) + for code in codes: + monitoring.set_local_events(tool_id, code, 0) + monitoring.clear_tool_id(tool_id) + assert monitoring.get_events(tool_id) == 0 + assert all( + monitoring.get_local_events(tool_id, code) == 0 for code in codes + ) + finally: + monitoring.free_tool_id(tool_id) + assert monitoring.get_tool(tool_id) is None + assert sys.getprofile() is previous + assert all( + function.__code__ is code + for function, code in zip(functions, codes, strict=True) + ) + + +def _values(case): + b = case.boundary + b.borrow() + loaded = graph.financial._artifacts( + case.cold.manifest, + b.compiled, + b.store, + b.kernels, + dict(b.keys), + dict(b.implementations), + ) + return graph._loaded_values(b, case.cold.manifest, loaded, b.nodes[2]) + + +def test_actual_two_route_extension_and_required_replay(composed): + case = composed + cold, warm, run, b = case.cold, case.warm, case.upstream.run, case.boundary + assert len(run.prefix.compiled.order) == 9 + assert len(run.compiled.order) == 19 + assert len(cold.compiled.order) == len(warm.compiled.order) == 245 + clone = graph.financial.atomic.clone + expand = clone.COMBINED_CLONE_NODE + claim = clone.COMBINED_CLONE_CLAIM_NODE + assign = cold.compiled.graph.node("geography.assign") + donor = cold.compiled.graph.node(graph.financial.financial.DONOR_NODE) + edge = graph.financial.financial._geography_edge() + order = cold.compiled.order + assert cold.compiled.graph.node(expand).structural is StructuralDelta.EXPAND + assert assign.population == expand + assert claim in cold.compiled.predecessors[assign.id] + assert ( + order.index(expand) + < order.index(claim) + < order.index(assign.id) + < order.index("geography.derive") + < order.index(edge.producer) + < order.index(donor.id) + < order.index(graph.financial.financial.ATTACH_NODE) + ) + assert graph.codec.decode_json(assign.params["definition"].encode())[ + "identity" + ] == [ + "survey_geography_origin_key", + "household_support_clone_index", + ] + assert edge in donor.artifact_inputs + assert edge.producer in cold.compiled.predecessors[donor.id] + assert donor.base == graph.financial.survey.CREATE_NODE + assert ( + run.manifest.population(donor.id).resolve_weights("person").kind + is graph.financial.values.WeightKind.DESIGN + ) + gate = cold.manifest.node(edge.producer) + assert gate.opaque_artifacts[edge.artifact] == opaque_artifact_key( + gate.key, edge.artifact + ) + assert cold.store.load_bytes(gate.opaque_artifacts[edge.artifact]) == ( + graph.codec.encode_json( + graph.codec.decode_json(run.projection)["atomic_geography"][ + "validation_receipt" + ] + ) + ) + assert run.prefix.clone_population is run.prefix.geography_population + assert "census_block_geoid" not in run.prefix.expanded_population.frame.table( + "household" + ) + assert "census_block_geoid" not in run.prefix.allocated_population.frame.table( + "household" + ) + households = run.prefix.geography_population.frame.table("household") + assert "census_block_geoid" in households + pd.testing.assert_frame_equal( + households, + cold.population.frame.table("household")[list(households)], + check_exact=True, + ) + assert cold.manifest.key == warm.manifest.key + assert tuple(r.profile for r in b.routes) == graph.attach.PROFILES + assert tuple(name for name, _ in b.qualified.matrices) == tuple( + p.value for p in graph.attach.PROFILES + ) + decoded = [ + graph.recipient.model_input.decode_recipient_matrix(payload) + for _, payload in b.qualified.matrices + ] + ids = [set(matrix.entity_ids.tolist()) for matrix in decoded] + table = b.expected.frame.table("tax_unit") + selected = graph.physical._masks(b.expected.frame)["tax_unit"] + assert ids[0] and ids[1] and not ids[0] & ids[1] + assert ids[0] | ids[1] == set(table.loc[selected, "tax_unit_id"]) + assert not (ids[0] | ids[1]) & set(table.loc[~selected, "tax_unit_id"]) + for matrix, route in zip(decoded, b.routes, strict=True): + assert tuple(matrix.features.columns) == route.profile.predictors + assert "employment_income_last_year" not in matrix.features + assert all(cold.manifest.node(n).hit for n in run.compiled.order) + assert all(record.hit for record in warm.manifest.nodes.values()) + assert len(cold.compiled.graph.sources) == 4 + assert cold.store.root == run.store.root and cold.store is not run.store + assert cold.kernels is not run.kernels + assert cold.store.codecs is not run.store.codecs + for node, key in graph.financial._run_entry(run)[2].keys: + assert cold.manifest.node(node).key == key + assert cold.compiled.order.index( + graph.financial.financial.ATTACH_NODE + ) < cold.compiled.order.index(graph.attach.FILTER_NODE) + assert all( + fit.population == graph.canonical.CANONICAL_DONOR_NODE + for route in b.routes + for fit in route.fits + ) + for route in b.routes: + assert tuple(route.fits[0].inputs[0].columns) == ( + *route.profile.predictors, + *route.profile.targets, + ) + matrix = next(e for e in route.applies[0].artifact_inputs if e.name == "matrix") + assert matrix.artifact == graph.recipient._NAMES[route.profile.value] + donor = cold.manifest.population(graph.canonical.CANONICAL_DONOR_NODE) + assert len(donor.table("tax_unit")) == 64 + assert donor.table("tax_unit").index.tolist() == list(range(501000, 501064)) + assert donor.table("tax_unit").tax_unit_id.tolist() == list(range(1, 65)) + assert donor.weights_for("tax_unit").values[0] == 0 + assert donor.weights_for("tax_unit").values[1:].tolist() == [ + (125 + i) / 100 for i in range(1, 64) + ] + graph.physical.replay.same_replayed_population(cold.population, warm.population) + assert graph.codec.decode_json(cold.receipt)["release_eligible"] is False + + +def test_direct_whole_cohort_oracle_all_outputs_and_nonowned_storage(composed): + case, b = composed, composed.boundary + values = _values(case) + _, donors, _ = b.canonical_donor(values) + draws = tuple(b.histories(values)) + # Separate direct finalizer oracle; neither the attachment's selected-column + # helper nor its retained computed KernelResult supplies expected values. + raw, _ = graph.attach.numerical.merge_puf55_route_draws( + b.expected.frame, + recipient_matrices=b.qualified.matrices, + route_draws=draws, + seed=b.seed, + ) + profile = graph.attach.PROFILES[0] + expected = graph.attach.full.support.finalize_us_puf_tax_detail_predictions( + b.expected.frame, + donors[0], + raw.copy(deep=True), + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, + tail_bound_diagnostics=[], + absent_cells=graph.attach.full.support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + before, after = b.expected.frame, case.cold.population.frame + masks = graph.physical._masks(before) + coordinates = {(o.entity, o.column) for o in b.nodes[2].outputs} + for entity in before.entities: + left, right = before.table(entity), after.table(entity) + assert left.index.identical(right.index) + for column in left: + owned = (entity, column) in coordinates + unchanged = ~masks[entity] if owned else np.ones(len(left), dtype=bool) + assert graph.population_ops.storage_equal( + left[column], right[column], unchanged + ) + for owned in b.nodes[2].outputs: + if owned.entity != entity: + continue + ids = before.schema.entity_id_column(entity) + actual = right.set_index(ids).loc[ + left.loc[masks[entity], ids], owned.column + ] + oracle = ( + expected.table(entity) + .set_index(ids) + .loc[actual.index, owned.column] + .astype(owned.dtype) + ) + pd.testing.assert_series_equal(actual, oracle, check_exact=True) + np.testing.assert_array_equal( + before.resolve_weights(entity).values, after.resolve_weights(entity).values + ) + assert not any( + o.column == "employment_income_last_year" for o in b.nodes[2].outputs + ) + assert not set(graph.attach.full.SURVEY_SS_COMPONENTS) & { + o.column for o in b.nodes[2].outputs + } + assert case.cold.population.mass_ledger == b.masked.mass_ledger + assert ( + case.cold.population.design_weights.keys() == b.expected.design_weights.keys() + ) + for entity in b.expected.design_weights: + np.testing.assert_array_equal( + case.cold.population.design_weights[entity], + b.expected.design_weights[entity], + ) + b.pure() + + +def test_wrong_actual_model_producer_is_refused_before_deserialization(composed): + b, values = composed.boundary, _values(composed) + name = "r0_model_000" + value = values[name] + wrong = "f" * 64 if value.producer_key != "f" * 64 else "e" * 64 + edge = next(e for e in b.nodes[2].artifact_inputs if e.name == name) + values[name] = replace( + value, producer_key=wrong, key=opaque_artifact_key(wrong, edge.artifact) + ) + calls = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__.__code__ + ): + calls.append(1) + + with _observe_exact_code_events( + trace, + starts=( + graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__, + ), + ): + with pytest.raises(ValueError, match="ARTIFACT_IDENTITY"): + b.finalize(values) + assert not calls + b.pure() + + +@pytest.mark.parametrize( + "mutation", ("receiving_after_finalizer", "selected_output_after_final_borrow") +) +def test_late_receiving_and_detached_output_mutation_refused(composed, mutation): + b, values = composed.boundary, _values(composed) + original_age = b.expected.frame.person.age.copy(deep=True) + fired = [] + + def trace(frame, event, arg): + if fired or event != "return": + return + if ( + mutation == "receiving_after_finalizer" + and frame.f_code is graph.attach.numerical._finalize_puf55_routes.__code__ + ): + # Mutate the retained DataFrame, not an extracted Series that may + # detach on write. Prove both the callee identity and changed owner. + assert frame.f_locals["frame"] is b.expected.frame + receiving = b.expected.frame.person + receiving.iloc[0, receiving.columns.get_loc("age")] = ( + int(original_age.iloc[0]) + 1 + ) + assert b.expected.frame.person.age.iloc[0] != original_age.iloc[0] + assert graph.physical._population_stamp(b.expected) != b.expected_stamp + fired.append(1) + elif ( + mutation == "selected_output_after_final_borrow" + and frame.f_code is graph.attach.Boundary.borrow.__code__ + ): + caller = frame.f_back + if ( + caller is not None + and caller.f_code is graph.attach.Boundary.finalize.__code__ + and "result" in caller.f_locals + ): + result = caller.f_locals["result"] + coordinate = next( + k + for k, s in result.columns.items() + if s.dtype == np.dtype("float64") + ) + changed = result.columns[coordinate].copy(deep=True) + changed.iloc[0] += 123 + result.columns[coordinate] = changed + fired.append(1) + + try: + with _observe_exact_code_events( + trace, + returns=( + ( + graph.attach.numerical._finalize_puf55_routes + if mutation == "receiving_after_finalizer" + else graph.attach.Boundary.borrow + ), + ), + ): + with pytest.raises(ValueError, match="CHANGED"): + b.finalize(values) + finally: + b.expected.frame.person["age"] = original_age + assert fired == [1] + b.pure() + + +def test_public_financial_dataclass_copy_cannot_issue_extension(composed): + run = replace(composed.upstream.run) + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + graph._construct( + run, + composed.paths, + fixture_definition=composed.definition, + seed=578, + n_estimators=2, + zero_atol=0, + ) + composed.boundary.pure() + + +@pytest.mark.parametrize("mutation", ("receipt", "artifact")) +def test_first_result_metadata_seal_binds_immutable_numerical_evidence( + composed, mutation +): + b = graph._construct( + composed.upstream.run, + composed.paths, + fixture_definition=composed.definition, + seed=578, + n_estimators=2, + zero_atol=0, + ) + assert b.computed is None + values = _values(composed) + expected_stamp = graph.physical._population_stamp(b.expected) + fired, finalizer_calls = [], [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.attach.full.support.finalize_us_puf_tax_detail_predictions.__code__ + ): + finalizer_calls.append(1) + if ( + event == "return" + and frame.f_code is graph.codec.encode_json.__code__ + and frame.f_back is not None + and frame.f_back.f_code is graph.attach.Boundary.finalize.__code__ + and "result" in frame.f_back.f_locals + and not fired + ): + parent = frame.f_back.f_locals + result = parent["result"] + evidence = parent["evidence"] + assert type(arg) is bytes and arg == evidence + assert b.computed is None + if mutation == "receipt": + old = result.receipt["recipient_rows"] + assert type(old) is int + result.receipt["recipient_rows"] = old + 1 + else: + assert result.artifacts == {"finalization": evidence} + result.artifacts["finalization"] = b"changed-after-encoder-return" + fired.append(1) + assert graph.physical._population_stamp(b.expected) == expected_stamp + + with _observe_exact_code_events( + trace, + starts=(graph.attach.full.support.finalize_us_puf_tax_detail_predictions,), + returns=(graph.codec.encode_json,), + ): + with pytest.raises( + ValueError, match="^SURVEY_PUF55_NUMERICAL_RECEIPT_BINDING$" + ): + b.finalize(values) + assert fired == finalizer_calls == [1] + assert b.computed is None + b.pure() + composed.boundary.pure() + + +@pytest.mark.parametrize("mutation", ("sources", "declaration", "route_matrix")) +def test_actual_context_declarations_and_route_matrix_are_exact(composed, mutation): + b = composed.boundary + original = composed.contexts[graph.attach.ATTACH_NODE] + if mutation == "sources": + paths = dict(original.sources) + name = next(iter(paths)) + paths[name] = next(p for p in paths.values() if p != paths[name]) + changed = replace(original, sources=paths) + reason = "CONTEXT_SOURCE_PATHS" + elif mutation == "declaration": + node = replace(original.node, params={**original.node.params, "seed": 579}) + changed = replace(original, node=node, params=node.params) + reason = "DECLARATION" + else: + artifacts = dict(original.artifacts) + first, second = (graph.recipient._NAMES[p.value] for p in graph.attach.PROFILES) + artifacts[first] = replace(artifacts[first], payload=artifacts[second].payload) + changed = replace(original, artifacts=artifacts) + reason = "RECIPIENT_MATRIX" + with pytest.raises(ValueError, match=reason): + b.context(changed, b.nodes[2], b.masked) + b.context(original, b.nodes[2], b.masked) + + +@pytest.mark.parametrize("mutation", ("payload", "producer", "mutable_payload")) +def test_typed_artifact_mutation_during_source_borrow_refused_before_decode( + composed, mutation +): + b, values = composed.boundary, _values(composed) + target = values["r0_model_000"] + original = (target.payload, target.producer_key, target.key) + fired, decodes = [], [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__.__code__ + ): + decodes.append(1) + if ( + event == "return" + and frame.f_code is graph.attach.Boundary.canonical_donor.__code__ + and frame.f_back is not None + and frame.f_back.f_code is graph.attach.Boundary.finalize.__code__ + and not fired + ): + if mutation == "payload": + object.__setattr__(target, "payload", b"changed typed model") + elif mutation == "producer": + wrong = "f" * 64 if target.producer_key != "f" * 64 else "e" * 64 + object.__setattr__(target, "producer_key", wrong) + object.__setattr__(target, "key", opaque_artifact_key(wrong, "model")) + else: + object.__setattr__(target, "payload", bytearray(target.payload)) + fired.append(1) + + try: + with _observe_exact_code_events( + trace, + starts=( + graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__, + ), + returns=(graph.attach.Boundary.canonical_donor,), + ): + with pytest.raises( + ValueError, match="ARTIFACT_(CHANGED_BEFORE_DECODE|VALUE_TYPE)" + ): + b.finalize(values) + finally: + for name, value in zip( + ("payload", "producer_key", "key"), original, strict=True + ): + object.__setattr__(target, name, value) + assert fired == [1] + assert not decodes + b.pure() + + +@pytest.mark.parametrize("mutation", ("capacity", "columns_metadata")) +def test_canonical_return_cannot_change_unmodeled_donor_surface(composed, mutation): + b, values = composed.boundary, _values(composed) + fired, decodes = [], [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__.__code__ + ): + decodes.append(1) + if ( + event == "return" + and frame.f_code is graph.attach.Boundary.canonical_donor.__code__ + and frame.f_back is not None + and frame.f_back.f_code is graph.attach.Boundary.finalize.__code__ + and not fired + ): + _, donors, _ = arg + for donor in donors: + if mutation == "capacity": + column = "puf_person_incidence_capacity" + before = donor.iloc[0][column] + donor.iloc[0, donor.columns.get_loc(column)] = before + 1 + assert donor.iloc[0][column] != before + else: + donor.columns.name = "changed_after_source_reconstruction" + fired.append(1) + + with _observe_exact_code_events( + trace, + starts=( + graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__, + ), + returns=(graph.attach.Boundary.canonical_donor,), + ): + with pytest.raises(ValueError, match="DONOR_RESULT_CHANGED"): + b.finalize(values) + assert fired == [1] + assert not decodes + b.pure() + + +def test_matching_public_producer_labels_do_not_authorize_changed_bytes(composed): + b, values = composed.boundary, _values(composed) + name = "r0_model_000" + values[name] = replace(values[name], payload=b"not the retained producer payload") + decodes = [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__.__code__ + ): + decodes.append(1) + + with _observe_exact_code_events( + trace, + starts=( + graph.physical.qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes.__func__, + ), + ): + with pytest.raises(ValueError, match="STORED_ARTIFACT_BYTES"): + b.finalize(values) + assert not decodes + b.pure() + + +@pytest.mark.parametrize( + "mutation", + ("candidate_return", "decoded_receipt_and_candidate", "kernel_result_return"), +) +def test_first_numerical_return_cannot_change_selected_candidate(composed, mutation): + # Construct a new actual-owner boundary over the already executed producers. + # It has never accepted a numerical result: repeated-result comparison must + # not mask this first-acceptance gap. No graph execution or fit is repeated. + b = graph._construct( + composed.upstream.run, + composed.paths, + fixture_definition=composed.definition, + seed=578, + n_estimators=2, + zero_atol=0, + ) + assert b.computed is None + values = _values(composed) + expected_stamp = graph.physical._population_stamp(b.expected) + fired, finalizer_calls = [], [] + + def trace(frame, event, arg): + if ( + event == "call" + and frame.f_code + is graph.attach.full.support.finalize_us_puf_tax_detail_predictions.__code__ + ): + finalizer_calls.append(1) + if ( + event == "return" + and frame.f_back is not None + and frame.f_back.f_code is graph.attach.Boundary.finalize.__code__ + and not fired + and ( + ( + mutation == "candidate_return" + and frame.f_code + is graph.attach.numerical._finalize_puf55_routes.__code__ + ) + or ( + mutation == "decoded_receipt_and_candidate" + and frame.f_code is graph.codec.decode_json.__code__ + and "candidate" in frame.f_back.f_locals + ) + or ( + mutation == "kernel_result_return" + and frame.f_code is graph.attach.KernelResult.__init__.__code__ + ) + ) + ): + if mutation == "kernel_result_return": + result = frame.f_locals["self"] + series = next( + v for v in result.columns.values() if v.dtype == np.dtype("float64") + ) + before = series.iloc[0] + assert np.isfinite(before) + series.iloc[0] = before + 123.0 + assert np.isfinite(series.iloc[0]) and series.iloc[0] != before + assert graph.physical._population_stamp(b.expected) == expected_stamp + fired.append(1) + return + if mutation == "candidate_return": + candidate, receipt = arg + assert type(receipt) is bytes + else: + candidate = frame.f_back.f_locals["candidate"] + assert arg["protocol"] == "microcosm.us.puf55-route-finalization.v2" + assert b.computed is None + owned = next(o for o in b.nodes[2].outputs if o.dtype == "float64") + mask = graph.physical._masks(b.expected.frame)[owned.entity] + table = candidate.table(owned.entity) + position = np.flatnonzero(mask)[0] + column = table.columns.get_loc(owned.column) + before = table.iloc[position, column] + assert np.isfinite(before) + table.iloc[position, column] = before + 123.0 + assert np.isfinite(table.iloc[position, column]) + assert table.iloc[position, column] != before + assert graph.physical._population_stamp(b.expected) == expected_stamp + if mutation == "decoded_receipt_and_candidate": + arg["candidate_frame_sha256"] = ( + graph.attach.numerical._candidate_frame_sha256(candidate) + ) + fired.append(1) + + with _observe_exact_code_events( + trace, + starts=(graph.attach.full.support.finalize_us_puf_tax_detail_predictions,), + returns=( + { + "candidate_return": graph.attach.numerical._finalize_puf55_routes, + "decoded_receipt_and_candidate": graph.codec.decode_json, + "kernel_result_return": graph.attach.KernelResult.__init__, + }[mutation], + ), + ): + code = { + "candidate_return": "NUMERICAL_OUTPUT_CHANGED", + "decoded_receipt_and_candidate": "NUMERICAL_RECEIPT_BINDING", + "kernel_result_return": "NUMERICAL_COLUMN_BINDING", + }[mutation] + with pytest.raises(ValueError, match="^SURVEY_PUF55_" + code + "$"): + b.finalize(values) + assert fired == finalizer_calls == [1] + assert b.computed is None + b.pure() + composed.boundary.pure() diff --git a/packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py b/packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py new file mode 100644 index 000000000..1d1453cdb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_puf55_heterogeneous_manifest_seals.py @@ -0,0 +1,571 @@ +"""Real tiny CREATE/store/replay controls; no financial fixture or fitting. + +These compare attached manifest contents. They issue no survey, donor, +calibration or release authority, and never call the full PUF55 host. +""" + +import hashlib +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_survey_puf55 as host +from microcosm.frame import ( + US_SCHEMA, + EntitySchema, + Frame, + LinkSpec, + WeightKind, + Weights, +) +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import load_frame_store +from microcosm.graph.population import token_for_dtype + +DONOR_VERSION = host.canonical.CANONICAL_DONOR_NODE + + +class InventedHeterogeneousSource(KernelBase): + """An actual CREATE loading only a supplied, owned frame-store source.""" + + ref = "test.puf55.heterogeneous_frame_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + frame = load_frame_store(context.sources[context.node.sources[0]]) + if context.node.params.get("noncanonical_null_backing", False): + assert frame.schema == US_SCHEMA + flag = frame.person["observed_flag"].array + assert bool(flag._mask[1]) + # A real CREATE result, before graph persistence: no production + # monkeypatch or normalized-away source-only probe. + flag._data[1] = True + assert bool(flag._data[1]) and pd.isna( + frame.person["observed_flag"].iloc[1] + ) + return KernelResult(frame=frame) + + +def _survey_frame(): + groups = np.array([1, 2], dtype=np.int64) + tables = { + entity: pd.DataFrame({US_SCHEMA.entity_id_column(entity): groups.copy()}) + for entity in US_SCHEMA.group_entities + } + tables["person"] = pd.DataFrame( + { + "person_id": np.array([1, 2, 3, 4], dtype=np.int64), + **{ + US_SCHEMA.membership_column(entity): np.repeat(groups, 2) + for entity in US_SCHEMA.group_entities + }, + "age": np.array([40, 12, 70, 20], dtype=np.int64), + "observed_flag": pd.array([True, pd.NA, False, True], dtype="boolean"), + } + ) + # Deliberately retain nonzero backing under a missing cell. The ordinary + # source codec may canonicalize it; the separate CREATE-result probe below + # reintroduces it only after that source decode, before graph persistence. + tables["person"]["observed_flag"].array._data[1] = True + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([2.0, 0.0]), WeightKind.DESIGN)}, + metadata={}, + ) + + +def _donor_frame(): + ids = np.array([1, 2, 3], dtype=np.int64) + return Frame( + { + "person": pd.DataFrame( + {"person_id": ids.copy(), "person_tax_unit_id": ids.copy()} + ), + "tax_unit": pd.DataFrame( + { + "tax_unit_id": ids.copy(), + "amount": np.array([10.0, -2.0, 0.0], dtype=np.float64), + "observed_count": pd.array([1, 0, 3], dtype="Int64"), + } + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.array([3.0, 1.0, 0.0]), WeightKind.DESIGN)}, + metadata={"fixture": "invented_donor", "revision": 1}, + ) + + +def _outputs(frame): + return tuple( + Owned(entity, column, token_for_dtype(frame.table(entity)[column].dtype)) + for entity in frame.entities + for column in frame.table(entity) + if column != frame.schema.entity_id_column(entity) + and not ( + entity == frame.schema.person_entity + and column + in {frame.schema.membership_column(g) for g in frame.schema.group_entities} + ) + ) + + +def _run_pair(root, donor, *, noncanonical_null_backing=False): + store = ContentStore(root, codecs={"frame-store": load_frame_store}) + frames = {"survey": _survey_frame(), DONOR_VERSION: donor} + sources, nodes, references = {}, [], [] + for name, frame in frames.items(): + source_name = "invented." + name + sources[source_name] = store.put_frame( + hashlib.sha256(("heterogeneous-source:" + name).encode()).hexdigest(), + frame, + ) + references.append(SourceRef(source_name, "frame-store")) + nodes.append( + Node( + name, + InventedHeterogeneousSource.ref, + sources=(source_name,), + structural=StructuralDelta.CREATE, + outputs=_outputs(frame), + params=( + {"noncanonical_null_backing": True} + if name == "survey" and noncanonical_null_backing + else {} + ), + ) + ) + graph = Graph("us", tuple(references), tuple(nodes)) + compiled = compile_graph(graph) + survey_compiled = compile_graph(Graph("us", (references[0],), (nodes[0],))) + registry = KernelRegistry() + registry.register(InventedHeterogeneousSource()) + cold = run_graph(compiled, sources=sources, store=store, kernels=registry) + warm = run_graph( + compiled, sources=sources, store=store, kernels=registry, resume="require" + ) + assert set(compiled.order) == {"survey", DONOR_VERSION} + assert all(not receipt.hit for receipt in cold.nodes.values()) + assert all(receipt.hit for receipt in warm.nodes.values()) + assert cold.key == warm.key + return SimpleNamespace( + compiled=compiled, + survey_compiled=survey_compiled, + cold=cold, + warm=warm, + ) + + +@pytest.fixture(scope="module") +def mixed_manifest(tmp_path_factory): + return _run_pair(tmp_path_factory.mktemp("heterogeneous-manifest"), _donor_frame()) + + +def _survey_manifest(manifest): + """Scope actual tiny CREATE receipts/attachments; no issuer is fabricated.""" + return replace( + manifest, + nodes={"survey": manifest.node("survey")}, + populations={"survey": manifest.population("survey")}, + mass_ledgers={"survey": manifest.mass_ledger("survey")}, + ) + + +def _copy_survey(frame): + return Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), frame.weights_for(entity).kind + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=dict(frame.metadata), + ) + + +def test_real_two_create_graph_has_distinct_schemas_and_required_hits(mixed_manifest): + case = mixed_manifest + for manifest in (case.cold, case.warm): + survey, donor = ( + manifest.population("survey"), + manifest.population(DONOR_VERSION), + ) + assert survey.schema == US_SCHEMA and len(survey.entities) == 6 + assert donor.schema == EntitySchema(group_entities=("tax_unit",)) + assert donor.entities == ("person", "tax_unit") + assert survey.n("person") == 4 and donor.n("person") == 3 + assert ( + manifest.mass_ledger("survey") == manifest.mass_ledger(DONOR_VERSION) == () + ) + assert survey.links == donor.links == () + assert survey.person["observed_flag"].isna().tolist() == [ + False, + True, + False, + False, + ] + + +def test_survey_only_helper_refuses_the_actual_mixed_roster(mixed_manifest): + for manifest in (mixed_manifest.cold, mixed_manifest.warm): + with pytest.raises(ValueError, match="^FRAME_TYPE$"): + host.financial._manifest_population_seals(manifest, mixed_manifest.compiled) + + +def test_heterogeneous_seal_includes_donor_and_is_store_stable(mixed_manifest): + case = mixed_manifest + seals = host._heterogeneous_manifest_seals(case.cold, case.compiled) + assert type(seals) is tuple + assert tuple(name for name, _ in seals) == tuple(sorted((DONOR_VERSION, "survey"))) + assert len(dict(seals)) == len(set(case.compiled.versions.values())) == 2 + # This independent existing donor helper is equivalent only for empty ledgers. + assert case.cold.mass_ledger(DONOR_VERSION) == () + assert dict(seals)[DONOR_VERSION] == host.canonical._frame_seal( + case.cold.population(DONOR_VERSION) + ) + assert seals == host._heterogeneous_manifest_seals(case.warm, case.compiled) + assert all(type(row) is tuple and all(type(v) is str for v in row) for row in seals) + with pytest.raises(TypeError): + seals[0][1] = "replacement" + # Both missing-content failures remain closed without a full host invocation. + without_donor = replace( + case.cold, populations={"survey": case.cold.population("survey")} + ) + with pytest.raises(KeyError, match="Population .* is not attached"): + host._heterogeneous_manifest_seals(without_donor, case.compiled) + without_ledger = replace(case.cold, mass_ledgers={"survey": ()}) + with pytest.raises(KeyError, match="Mass ledger .* is not attached"): + host._heterogeneous_manifest_seals(without_ledger, case.compiled) + + +def test_upstream_survey_roster_comparison_is_store_stable(mixed_manifest): + case = mixed_manifest + expected = _survey_manifest(case.cold) + assert tuple(expected.populations) == ("survey",) + assert ( + host._check_replayed_survey_manifest(expected, case.cold, case.survey_compiled) + is None + ) + assert ( + host._check_replayed_survey_manifest(expected, case.warm, case.survey_compiled) + is None + ) + full = dict(host._heterogeneous_manifest_seals(case.cold, case.compiled)) + subset = dict(host._heterogeneous_manifest_seals(case.cold, case.survey_compiled)) + assert tuple(subset) == ("survey",) + assert subset["survey"] == full["survey"] + # The two different seal algorithms are intentionally not cross-compared. + + +def test_survey_seal_replay_probe_retains_nonzero_missing_backing(tmp_path): + """A failure here is a real remaining cross-store seal blocker, not an xfail.""" + case = _run_pair( + tmp_path / "noncanonical-backing", + _donor_frame(), + noncanonical_null_backing=True, + ) + cold_flag = case.cold.population("survey").person["observed_flag"] + warm_flag = case.warm.population("survey").person["observed_flag"] + assert pd.isna(cold_flag.iloc[1]) and pd.isna(warm_flag.iloc[1]) + # The actual kernel asserted nonzero backing just before KernelResult. + # Canonicalization before cold-manifest retention is valid lifecycle behavior. + try: + host._check_replayed_survey_manifest( + _survey_manifest(case.cold), case.warm, case.survey_compiled + ) + except ValueError as error: + raise AssertionError( + "Upstream replay comparison failed for this valid nullable US CREATE; " + f"cold_missing_backing={bool(cold_flag.array._data[1])}, " + f"warm_missing_backing={bool(warm_flag.array._data[1])}" + ) from error + + +@pytest.mark.parametrize( + "change", ("value", "mask", "dtype", "column_order", "weight", "metadata") +) +def test_each_donor_change_alters_only_its_manifest_seal( + mixed_manifest, tmp_path, change +): + original = _donor_frame() + tables = {e: original.table(e).copy(deep=True) for e in original.entities} + weights = original.weights_for("tax_unit").values.copy() + metadata = dict(original.metadata) + table = tables["tax_unit"] + if change == "value": + table.loc[0, "amount"] = 11.0 + assert table.loc[0, "amount"] != original.table("tax_unit").loc[0, "amount"] + elif change == "mask": + table.loc[0, "observed_count"] = pd.NA + assert table["observed_count"].isna().tolist() == [True, False, False] + assert not original.table("tax_unit")["observed_count"].isna().any() + elif change == "dtype": + table["amount"] = table["amount"].astype("float32") + assert table["amount"].dtype == np.dtype("float32") + np.testing.assert_array_equal( + table["amount"], original.table("tax_unit")["amount"] + ) + elif change == "column_order": + tables["tax_unit"] = table.loc[:, ["tax_unit_id", "observed_count", "amount"]] + assert tuple(tables["tax_unit"]) != tuple(original.table("tax_unit")) + elif change == "weight": + weights[0] = 4.0 + assert weights[0] != original.weights_for("tax_unit").values[0] + else: + assert change == "metadata" + metadata["revision"] = 2 + assert metadata != dict(original.metadata) + changed = Frame( + tables, + original.schema, + {"tax_unit": Weights(weights, WeightKind.DESIGN)}, + original.strata.copy(deep=True), + metadata=metadata, + ) + # Every variation goes through actual source encoding, CREATE and required replay. + rerun = _run_pair(tmp_path / "changed-store", changed) + before = dict( + host._heterogeneous_manifest_seals(mixed_manifest.cold, mixed_manifest.compiled) + ) + after = dict(host._heterogeneous_manifest_seals(rerun.cold, rerun.compiled)) + assert set(before) == set(after) == {DONOR_VERSION, "survey"} + assert before[DONOR_VERSION] != after[DONOR_VERSION] + assert before["survey"] == after["survey"] + assert tuple(after.items()) == host._heterogeneous_manifest_seals( + rerun.warm, rerun.compiled + ) + for actual in (rerun.cold, rerun.warm): + assert ( + host._check_replayed_survey_manifest( + _survey_manifest(mixed_manifest.cold), + actual, + mixed_manifest.survey_compiled, + ) + is None + ) + + +def test_explicit_link_body_is_refused_before_content_sealing(mixed_manifest): + base = mixed_manifest.cold.population(DONOR_VERSION) + schema = EntitySchema( + group_entities=("tax_unit",), + links=(LinkSpec("claims", "person", "tax_unit"),), + ) + linked = Frame( + { + **{e: base.table(e).copy(deep=True) for e in base.entities}, + "claims": pd.DataFrame( + { + "person_id": np.array([1, 2], dtype=np.int64), + "tax_unit_id": np.array([2, 3], dtype=np.int64), + "share": np.array([0.25, 0.75], dtype=np.float64), + } + ), + }, + schema, + { + "tax_unit": Weights( + base.weights_for("tax_unit").values.copy(), WeightKind.DESIGN + ) + }, + base.strata.copy(deep=True), + metadata=dict(base.metadata), + ) + assert linked.links == ("claims",) and len(linked.link("claims")) == 2 + # A public descriptive manifest copy is sufficient for a negative helper input; + # its copied node receipts confer no new source or graph admission. + manifest = replace( + mixed_manifest.cold, + populations={ + "survey": mixed_manifest.cold.population("survey"), + DONOR_VERSION: linked, + }, + ) + with pytest.raises(ValueError, match="^SURVEY_PUF55_MANIFEST_SEAL_LINK_TABLES$"): + host._heterogeneous_manifest_seals(manifest, mixed_manifest.compiled) + + +@pytest.mark.parametrize( + "fault,reason", + ( + ("known_value", "SURVEY_POPULATION_REPLAY_NATIVE_BITS"), + ("mask", "SURVEY_POPULATION_REPLAY_MASKED_STORAGE"), + ("dtype", "SURVEY_POPULATION_REPLAY_SERIES_DTYPE_OR_LENGTH"), + ("column_order", "SURVEY_POPULATION_REPLAY_AXIS"), + ("column_roster", "SURVEY_POPULATION_REPLAY_AXIS"), + ("entity_roster", "SURVEY_PUF55_UPSTREAM_SURVEY_SCHEMA"), + ("weight", "SURVEY_POPULATION_REPLAY_WEIGHT_BYTES"), + ("metadata", "SURVEY_POPULATION_REPLAY_FRAME_CONTEXT"), + ( + "noncanonical_null_backing", + "SURVEY_POPULATION_REPLAY_NONCANONICAL_NULL_BACKING", + ), + ), +) +def test_replayed_survey_refuses_changed_actual_content(mixed_manifest, fault, reason): + # Start from actual required-store content. Only a detached descriptive + # negative input changes; the shared source/fixture manifest is untouched. + expected = _survey_manifest(mixed_manifest.warm) + original = expected.population("survey") + changed = _copy_survey(original) + tables = {entity: changed.table(entity) for entity in changed.entities} + weights = { + entity: changed.weights_for(entity) for entity in changed.weighted_entities + } + metadata = dict(changed.metadata) + person = tables["person"] + if fault == "known_value": + person.loc[0, "age"] += 1 + assert person.loc[0, "age"] != original.person.loc[0, "age"] + elif fault == "mask": + person.loc[0, "observed_flag"] = pd.NA + assert pd.isna(person.loc[0, "observed_flag"]) + assert not pd.isna(original.person.loc[0, "observed_flag"]) + elif fault == "dtype": + person["age"] = person["age"].astype("int32") + assert person["age"].dtype != original.person["age"].dtype + np.testing.assert_array_equal(person["age"], original.person["age"]) + elif fault == "column_order": + columns = [*person.columns[:-2], *reversed(person.columns[-2:])] + tables["person"] = person.loc[:, columns] + assert tuple(tables["person"]) != tuple(original.person) + elif fault == "column_roster": + tables["person"] = person.drop(columns=["age"]) + assert "age" not in tables["person"] and "age" in original.person + elif fault == "entity_roster": + changed = _donor_frame() + assert changed.schema != original.schema + elif fault == "weight": + values = original.weights_for("household").values.copy() + values[0] += 1 + weights["household"] = Weights(values, WeightKind.DESIGN) + assert values[0] != original.weights_for("household").values[0] + elif fault == "metadata": + metadata["invented_changed_metadata"] = True + assert metadata != dict(original.metadata) + else: + assert fault == "noncanonical_null_backing" + # The required-store side is the expected side here; a newly nonzero + # missing buffer in actual cannot use the one-way canonical-zero rule. + assert bool(original.person["observed_flag"].array._mask[1]) + assert not bool(original.person["observed_flag"].array._data[1]) + person["observed_flag"].array._data[1] = True + assert bool(person["observed_flag"].array._data[1]) + assert pd.isna(person["observed_flag"].iloc[1]) + if fault != "entity_roster": + changed = Frame( + tables, + changed.schema, + weights, + changed.strata.copy(deep=True), + mass_log=changed.mass_log, + metadata=metadata, + ) + actual = replace(expected, populations={"survey": changed}) + with pytest.raises(ValueError, match="^" + reason + "$"): + host._check_replayed_survey_manifest( + expected, actual, mixed_manifest.survey_compiled + ) + + +@pytest.mark.parametrize( + "fault", + ( + "expected_population", + "expected_ledger", + "actual_population", + "actual_ledger", + "compiled_roster", + ), +) +def test_replayed_survey_requires_exact_upstream_roster(mixed_manifest, fault): + expected = _survey_manifest(mixed_manifest.cold) + actual = mixed_manifest.warm # The legitimate extra donor version is allowed. + compiled = mixed_manifest.survey_compiled + if fault == "expected_population": + expected = replace(expected, populations={}) + elif fault == "expected_ledger": + expected = replace(expected, mass_ledgers={}) + elif fault == "actual_population": + actual = replace( + actual, populations={DONOR_VERSION: actual.population(DONOR_VERSION)} + ) + elif fault == "actual_ledger": + actual = replace( + actual, mass_ledgers={DONOR_VERSION: actual.mass_ledger(DONOR_VERSION)} + ) + else: + assert fault == "compiled_roster" + compiled = mixed_manifest.compiled + with pytest.raises(ValueError, match="^SURVEY_PUF55_UPSTREAM_SURVEY_ROSTER$"): + host._check_replayed_survey_manifest(expected, actual, compiled) + + +def test_replayed_survey_refuses_changed_mass_ledger(mixed_manifest): + expected = _survey_manifest(mixed_manifest.warm) + assert expected.mass_ledger("survey") == () + record = host.population_ops.MassRecord( + node_id="invented.extra", + operation="filter", + policy="preserve", + before_total=2.0, + after_total=2.0, + before_by_stratum=(), + after_by_stratum=(), + entity="household", + ) + actual = replace(expected, mass_ledgers={"survey": (record,)}) + assert actual.mass_ledger("survey") != expected.mass_ledger("survey") + with pytest.raises( + ValueError, match="^SURVEY_POPULATION_REPLAY_POPULATION_CONTEXT$" + ): + host._check_replayed_survey_manifest( + expected, actual, mixed_manifest.survey_compiled + ) + + +def test_full_manifest_physical_seal_still_detects_late_masked_bits(mixed_manifest): + expected = _survey_manifest(mixed_manifest.warm) + detached = _copy_survey(expected.population("survey")) + actual = replace( + mixed_manifest.warm, + populations={ + "survey": detached, + DONOR_VERSION: mixed_manifest.warm.population(DONOR_VERSION), + }, + ) + assert ( + host._check_replayed_survey_manifest( + expected, actual, mixed_manifest.survey_compiled + ) + is None + ) + before = host._heterogeneous_manifest_seals(actual, mixed_manifest.compiled) + flag = actual.population("survey").person["observed_flag"].array + assert bool(flag._mask[1]) and not bool(flag._data[1]) + flag._data[1] = True + assert bool(flag._data[1]) and bool(flag._mask[1]) + assert host._heterogeneous_manifest_seals(actual, mixed_manifest.compiled) != before From aea7b2e02623e9c2622283f30b0a6929506c1c7f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 08:54:48 -0400 Subject: [PATCH 107/255] Add declared US fiscal measurement graph stage --- .../us-declared-fiscal-measurement.added.md | 1 + .../us_runtime/graph_fiscal_measurement.py | 655 ++++++++++++++++++ .../tests/test_us_graph_fiscal_measurement.py | 463 +++++++++++++ 3 files changed, 1119 insertions(+) create mode 100644 changelog.d/us-declared-fiscal-measurement.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py create mode 100644 packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py diff --git a/changelog.d/us-declared-fiscal-measurement.added.md b/changelog.d/us-declared-fiscal-measurement.added.md new file mode 100644 index 000000000..58d36e459 --- /dev/null +++ b/changelog.d/us-declared-fiscal-measurement.added.md @@ -0,0 +1 @@ +Add a candidate-only US graph measurement stage using declared target bindings, household-aligned sparse measures, explicit national/state/district support checks, and complete input-closure checks for optional model evaluation. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py new file mode 100644 index 000000000..8d9ef4bfc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -0,0 +1,655 @@ +"""Declared fiscal measurements on one supplied US population, without fitting. + +V1 supports shared-interpreter value variables, sums and same-entity predicates. +Every target also has an explicit national/state/CD household scope. Specialized +providers, counterfactuals, band interpretation and monetary-binding receipts +are refused. This is not full fiscal-registry coverage or a population issuer. +The host owns source/target activation and complete-enrichment ancestry. + +Computed roots use the existing real US adapter only after its static closure +is explicitly present, with no missing cells or default allowlist. Measurements +and model outputs live only in private tables and a bounded CSR artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd +from scipy import sparse + +from microcosm.build import target_materialization +from microcosm.calibrate import TargetRegistry, TargetSpec, matrix +from microcosm.calibrate.hierarchy import CalibrationHierarchy, HierarchyGeography +from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame.adapters import policyengine_us +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.executor import _context_digest + +from . import asec_engine_evaluation as runtime + +MEASUREMENT_TYPE = ArtifactType("microcosm.us.declared_fiscal_measurement", 1) +PROTOCOL = "microcosm.us.declared-fiscal-measurement.v1" +MAX_BYTES = 64 * 1024**2 +MAX_DECLARATION_BYTES = 1024**2 +_BINDING_KEYS = {"value_variable", "value_expression", "filters", "from_entity"} +_LEVELS = {"national", "state", "congressional_district"} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_MEASUREMENT_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _json(value): + return canonical_json(value).decode() + + +def _registry_document(registry): + return {"country": registry.country, "specs": [asdict(s) for s in registry]} + + +def _registry(raw): + _require(set(raw) == {"country", "specs"}, "REGISTRY_FIELDS") + specs = [] + for item in raw["specs"]: + hierarchy = item.get("hierarchy") + specs.append( + TargetSpec( + **{ + **item, + "hierarchy": None + if hierarchy is None + else CalibrationHierarchy.from_dict(hierarchy), + } + ) + ) + result = TargetRegistry(specs, country=raw["country"]) + _require(result.country == "us" and len(result) > 0, "US_REGISTRY_REQUIRED") + _require(_json(_registry_document(result)) == _json(raw), "REGISTRY_CANONICAL") + return result + + +@dataclass(frozen=True) +class FiscalGeographyScope: + """An explicit interpretation of a target hierarchy's geographic scope. + + The national scope uses no column/value; state and CD scopes use the named + integer household geography columns. The host admits the boundary vintage. + """ + + geography: HierarchyGeography + column: str | None = None + value: int | None = None + + def __post_init__(self): + level = self.geography.level + _require(level in _LEVELS, "GEOGRAPHY_LEVEL") + expected = { + "national": None, + "state": "state_fips", + "congressional_district": "congressional_district_geoid", + }[level] + _require(self.column == expected, "GEOGRAPHY_COLUMN") + _require( + self.value is None if expected is None else type(self.value) is int, + "GEOGRAPHY_VALUE", + ) + + +def _scopes(raw): + scopes = tuple( + FiscalGeographyScope( + HierarchyGeography(**r["geography"]), r["column"], r["value"] + ) + for r in raw + ) + _require({s.geography.level for s in scopes} == _LEVELS, "ADVERTISED_LEVELS") + keys = [(s.geography.level, s.geography.id) for s in scopes] + _require(len(set(keys)) == len(keys), "DUPLICATE_SCOPE") + _require( + len({(s.column, s.value) for s in scopes}) == len(scopes), + "DUPLICATE_SCOPE_PREDICATE", + ) + _require( + sum(s.geography.level == "national" for s in scopes) == 1, "NATIONAL_SCOPE" + ) + return scopes + + +def _model_contract(roots): + if not roots: + return {"evaluation": "none", "roots": []} + identity = runtime.engine_runtime_identity() + index = policyengine_us.PolicyEngineUSVariableMetadataIndex() + contracts = [] + for root in roots: + closure = index.variable_dependency_closure(root) + contracts.append( + { + "closure": asdict(closure), + "entity": index.variable_metadata(root).entity, + "leaves": { + leaf: index.variable_metadata(leaf).entity + for leaf in closure.input_leaves + }, + } + ) + return {"evaluation": "real_baseline", "runtime": identity, "roots": contracts} + + +def fiscal_measurement_node( + registry: TargetRegistry, + *, + population: str, + input_columns: dict[str, tuple[str, ...]], + contract_targets: dict, + advertised_scopes: tuple[FiscalGeographyScope, ...], + geography_vintage: str, + period: int, + model_outputs: tuple[str, ...] = (), + node_id: str = "us.fiscal_measurement", +) -> Node: + """Declare a bounded measurement stage; supplied targets are not activated here. + + Use the same model_outputs on FiscalMeasurementKernel. Full US structural + person membership is retained through its required input slice. Unread + group tables are reconstructed as sorted IDs only, using Frame's exact + membership contract; none of their values is inferred or used. + Every required input leaf is checked against this declaration before model + evaluation. Neither this node nor its artifact admits a complete parent. + """ + _require(type(period) is int and period > 0, "PERIOD") + _require(type(geography_vintage) is str and bool(geography_vintage), "VINTAGE") + _require( + type(model_outputs) is tuple + and all(type(r) is str and r for r in model_outputs) + and len(set(model_outputs)) == len(model_outputs), + "MODEL_OUTPUTS", + ) + registry = _registry(_registry_document(registry)) + scopes = _scopes([asdict(s) for s in advertised_scopes]) + entities = (US_SCHEMA.person_entity, *US_SCHEMA.group_entities) + _require(set(input_columns) <= set(entities), "INPUT_ENTITIES") + columns = {entity: tuple(input_columns.get(entity, ())) for entity in entities} + _require(bool(columns["person"]), "DECLARED_PERSON_PROJECTION") + _require( + all(len(set(names)) == len(names) for names in columns.values()), + "DUPLICATE_INPUT", + ) + _require( + {s.column for s in scopes if s.column} <= set(columns["household"]), + "DECLARED_GEOGRAPHY", + ) + keys = {(s.geography.level, s.geography.id) for s in scopes} + target_scopes = set() + used_bindings = set() + measurements = {} + for spec in registry: + _require( + spec.period == period and spec.hierarchy is not None, + "TARGET_PERIOD_HIERARCHY", + ) + key = (spec.hierarchy.geography.level, spec.hierarchy.geography.id) + _require(key in keys, "TARGET_SCOPE") + target_scopes.add(key) + _require( + "monetary_binding" not in spec.metadata, "UNSUPPORTED_MONETARY_BINDING" + ) + name = spec.metadata.get("contract_target_id") + _require(name in contract_targets, "MISSING_BINDING") + used_bindings.add(name) + target = contract_targets[name] + _require( + set(target) == {"bindings"} and set(target["bindings"]) == {"policyengine"}, + "BINDING_SHAPE", + ) + binding = target["bindings"]["policyengine"] + coordinate = (spec.entity, spec.measure) + _require( + coordinate not in measurements + or measurements[coordinate] == _json(binding), + "CONFLICTING_MEASUREMENT_BINDINGS", + ) + measurements[coordinate] = _json(binding) + _require(set(binding) <= _BINDING_KEYS, "UNSUPPORTED_BINDING") + _require( + ("value_variable" in binding) != ("value_expression" in binding), + "VALUE_BINDING", + ) + _require( + binding.get("from_entity", spec.entity) == spec.entity, "BINDING_ENTITY" + ) + _require(spec.entity in entities, "TARGET_ENTITY") + _require( + spec.measure != f"{spec.entity}_id" + and not ( + spec.entity == "person" + and spec.measure in {f"person_{e}_id" for e in US_SCHEMA.group_entities} + ), + "STRUCTURAL_MEASURE_COLLISION", + ) + _require(spec.measure not in columns[spec.entity], "INPUT_MEASURE_COLLISION") + _require(spec.measure not in model_outputs, "MODEL_MEASURE_COLLISION") + for predicate in binding.get("filters", ()): + _require( + set(predicate) + in ({"variable", "operator", "value"}, {"variable", "equals"}), + "PREDICATE_FIELDS", + ) + _require(used_bindings == set(contract_targets), "UNUSED_BINDINGS") + _require(target_scopes == keys, "UNCOVERED_ADVERTISED_SCOPE") + model = _model_contract(model_outputs) + for contract in model["roots"]: + root = contract["closure"]["root"] + _require(root not in columns[contract["entity"]], "FORMULA_INPUT_COLLISION") + _require( + not ( + set(contract["closure"]["formula_nodes"]) + & {name for names in columns.values() for name in names} + ), + "FORMULA_INPUT_COLLISION", + ) + for leaf, entity in contract["leaves"].items(): + _require(leaf in columns[entity], "UNDECLARED_MODEL_LEAF:" + leaf) + params = { + "registry": _json(_registry_document(registry)), + "contract_targets": _json(contract_targets), + "advertised_scopes": _json([asdict(s) for s in scopes]), + "geography_vintage": geography_vintage, + "period": period, + "model_outputs": model_outputs, + "model_contract": _json(model), + } + _require(len(_json(params).encode()) <= MAX_DECLARATION_BYTES, "DECLARATION_BOUND") + return Node( + node_id, + FiscalMeasurementKernel.ref, + population=population, + inputs=tuple(Slice(e, columns[e]) for e in entities if columns[e]), + params=params, + artifact_outputs=(ArtifactOutput("measurement", MEASUREMENT_TYPE),), + ) + + +class _Adapter: + def __init__(self, frame): + self.tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + + def column(self, entity, variable): + if variable == f"{entity}_count": + return np.ones(len(self.tables[entity]), dtype=np.float64) + return self.tables[entity][variable].to_numpy(copy=True) + + def has_column(self, entity, variable): + return variable in self.tables[entity] + + def set_column(self, entity, variable, values): + _require(variable not in self.tables[entity], "MEASUREMENT_COLUMN_COLLISION") + array = np.asarray(values) + _require(array.shape == (len(self.tables[entity]),), "MEASUREMENT_ALIGNMENT") + self.tables[entity][variable] = array + + +def _array(array): + array = np.ascontiguousarray(array) + return {"dtype": array.dtype.str, "hex": array.tobytes().hex()} + + +def _read_array(raw, dtypes): + _require(type(raw) is dict and set(raw) == {"dtype", "hex"}, "ARRAY_FIELDS") + _require(raw["dtype"] in dtypes and type(raw["hex"]) is str, "ARRAY_DTYPE") + try: + values = np.frombuffer(bytes.fromhex(raw["hex"]), dtype=raw["dtype"]).copy() + except (ValueError, TypeError): + raise ValueError("FISCAL_MEASUREMENT_ARRAY_ENCODING") from None + _require(_array(values) == raw, "ARRAY_CANONICAL") + return values + + +@dataclass(frozen=True) +class FiscalMeasurement: + """Decoded numbers and declarations only; no source or release authority.""" + + document: dict + registry: TargetRegistry + household_ids: np.ndarray + matrix: sparse.csr_array + target_values: np.ndarray + + +def decode_fiscal_measurement(payload: bytes) -> FiscalMeasurement: + """Decode bounded, canonical numerical bytes without loading models/files.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD_BOUND") + raw = json.loads(payload) + _require(canonical_json(raw) == payload, "CANONICAL") + _require( + set(raw) + == { + "protocol", + "node", + "declaration", + "projection_sha256", + "household_ids", + "indptr", + "indices", + "data", + "target_values", + "support", + }, + "PAYLOAD_FIELDS", + ) + _require(raw["protocol"] == PROTOCOL, "PROTOCOL") + registry = _registry(json.loads(raw["declaration"]["registry"])) + ids = _read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + indptr = _read_array(raw["indptr"], {"= 0) + and indptr[-1] == len(indices) == len(data), + "CSR_SHAPE", + ) + _require( + np.all((indices >= 0) & (indices < len(ids))) and np.isfinite(data).all(), + "CSR_VALUES", + ) + problem = sparse.csr_array((data, indices, indptr), shape=(len(registry), len(ids))) + _require(problem.has_canonical_format and np.all(data != 0), "CSR_CANONICAL") + _require( + type(raw["support"]) is list and len(raw["support"]) == len(registry), + "SUPPORT_SHAPE", + ) + for spec, row in zip(registry, raw["support"], strict=True): + _require( + type(row) is dict + and set(row) + == { + "target", + "positive_weight_positive_rows", + "positive_weight_negative_rows", + }, + "SUPPORT_FIELDS", + ) + _require(row["target"] == spec.to_target().row_name, "SUPPORT_TARGET") + for key in ("positive_weight_positive_rows", "positive_weight_negative_rows"): + _require( + type(row[key]) is int and 0 <= row[key] <= len(ids), "SUPPORT_COUNT" + ) + return FiscalMeasurement(raw, registry, ids, problem, values) + + +def verify_fiscal_measurement(payload, *, context, expected_sha256): + """Bind a graph-selected artifact to its exact declared input projection. + + The host must obtain expected_sha256 from the matching actual graph record; + an arbitrary supplied hash cannot establish producer or parent authority. + """ + _require(_sha(payload) == expected_sha256, "ARTIFACT_DIGEST") + result = decode_fiscal_measurement(payload) + _require( + _json(result.document["node"]) == _json(normative(context.node)), + "NODE_BINDING", + ) + _require( + _json(result.document["declaration"]) == _json(dict(context.params)), + "DECLARATION_BINDING", + ) + _require( + result.document["projection_sha256"] == _context_digest(context).hex(), + "PROJECTION_BINDING", + ) + expected = context.tables["household"]["household_id"].to_numpy() + _require( + result.household_ids.dtype == expected.dtype + and np.array_equal(result.household_ids, expected), + "HOUSEHOLD_ALIGNMENT", + ) + masks = _geography_masks( + context.tables["household"], + _scopes(json.loads(context.params["advertised_scopes"])), + ) + weights = context.weights["household"].values + for i, spec in enumerate(result.registry): + row = result.matrix[i : i + 1] + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + _require(mask[row.indices].all(), "GEOGRAPHIC_ROW_SUPPORT") + _require( + _support(spec, row, weights) == result.document["support"][i], + "SUPPORT_BINDING", + ) + return result + + +def _geography_masks(household, scopes): + masks = {} + for column in ("state_fips", "congressional_district_geoid"): + values = household[column] + _require( + values.dtype.kind in "iu" and not values.isna().any(), "GEOGRAPHY_DTYPE" + ) + _require( + np.array_equal( + household.congressional_district_geoid.to_numpy() // 100, + household.state_fips.to_numpy(), + ), + "STATE_CD_MAPPING", + ) + for scope in scopes: + mask = ( + np.ones(len(household), dtype=bool) + if scope.column is None + else household[scope.column].to_numpy() == scope.value + ) + _require(mask.any(), "EMPTY_ADVERTISED_GEOGRAPHY:" + scope.geography.id) + masks[scope.geography.level, scope.geography.id] = mask + for level in ("state", "congressional_district"): + covered = np.sum( + [mask for (scope_level, _), mask in masks.items() if scope_level == level], + axis=0, + ) + _require(np.all(covered == 1), "UNCOVERED_POPULATION_GEOGRAPHY") + return masks + + +def _support(spec, row, weights): + positive = int(np.count_nonzero((row.data > 0) & (weights[row.indices] > 0))) + negative = int(np.count_nonzero((row.data < 0) & (weights[row.indices] > 0))) + _require( + spec.value == 0 or (positive > 0 if spec.value > 0 else negative > 0), + "UNSUPPORTED_TARGET:" + spec.name, + ) + return { + "target": spec.to_target().row_name, + "positive_weight_positive_rows": positive, + "positive_weight_negative_rows": negative, + } + + +class FiscalMeasurementKernel(KernelBase): + """Materialize declared measures and compile the real household CSR system.""" + + ref = "us.declared_fiscal_measurement@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + dependencies=("numpy", "pandas", "scipy"), + ) + + def __init__(self, *, model_outputs=()): + self.model_outputs = model_outputs + + def implementation_hash(self): + return _sha( + canonical_json( + { + "implementation": source_hash( + sys.modules[__name__], + target_materialization, + matrix, + TargetSpec, + Frame, + policyengine_us, + runtime, + _context_digest, + dependencies=self.capabilities.dependencies, + ), + "model": _model_contract(self.model_outputs), + } + ) + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + _require(params["model_outputs"] == self.model_outputs, "KERNEL_MODEL_OUTPUTS") + registry = _registry(json.loads(params["registry"])) + scopes = _scopes(json.loads(params["advertised_scopes"])) + expected = fiscal_measurement_node( + registry, + population=context.node.population, + input_columns={s.entity: s.columns for s in context.node.inputs}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=scopes, + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=self.model_outputs, + node_id=context.node.id, + ) + _require(context.node.normative() == expected.normative(), "NODE_DECLARATION") + projection = _context_digest(context).hex() + tables = {e: t.copy(deep=True) for e, t in context.tables.items()} + for entity in US_SCHEMA.group_entities: + if entity not in tables: + tables[entity] = pd.DataFrame( + { + f"{entity}_id": np.unique( + tables["person"][f"person_{entity}_id"].to_numpy() + ), + } + ) + frame = Frame( + tables, + US_SCHEMA, + {"household": context.weights["household"]}, + strata=context.strata.copy(), + ) + adapter = _Adapter(frame) + model = json.loads(params["model_contract"]) + if self.model_outputs: + for contract in model["roots"]: + for leaf, entity in contract["leaves"].items(): + values = frame.table(entity)[leaf] + _require(not values.isna().any(), "MISSING_MODEL_INPUT:" + leaf) + if pd.api.types.is_numeric_dtype(values.dtype): + _require( + np.isfinite(values.to_numpy()).all(), + "NONFINITE_MODEL_INPUT", + ) + outputs = policyengine_us.PolicyEngineUSEngine().materialize( + frame, self.model_outputs, params["period"] + ) + _require(set(outputs) == set(self.model_outputs), "MODEL_RESULT_ROSTER") + for contract in model["roots"]: + output = np.asarray(outputs[contract["closure"]["root"]]) + _require( + output.dtype.kind in "biuf" and np.isfinite(output).all(), + "MODEL_RESULT_VALUES", + ) + adapter.set_column( + contract["entity"], + contract["closure"]["root"], + outputs[contract["closure"]["root"]], + ) + prepared = target_materialization.materialize_target_bindings( + adapter, + registry, + json.loads(params["contract_targets"]), + period=params["period"], + ) + _require( + not prepared.skipped, "UNMATERIALIZED_TARGETS:" + str(prepared.skipped) + ) + work = Frame( + adapter.tables, US_SCHEMA, {"household": context.weights["household"]} + ) + problem = matrix.build_constraint_matrix( + work, registry.to_target_set(), weight_entity="household" + ) + _require( + not problem.skipped + and problem.names == tuple(s.to_target().row_name for s in registry), + "TARGET_ALIGNMENT", + ) + household = frame.table("household") + masks = _geography_masks(household, scopes) + rows, support = [], [] + incoming = problem.initial_weights.values + for i, spec in enumerate(registry): + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + row = problem.matrix[i : i + 1].multiply(mask).tocsr() + row.eliminate_zeros() + rows.append(row) + support.append(_support(spec, row, incoming)) + csr = sparse.vstack(rows, format="csr") + _require(csr.nnz * 48 + frame.n("household") * 16 < MAX_BYTES, "MATRIX_BOUND") + payload = canonical_json( + { + "protocol": PROTOCOL, + "node": normative(context.node), + "declaration": dict(params), + "projection_sha256": projection, + "household_ids": _array(household.household_id.to_numpy()), + "indptr": _array(csr.indptr.astype("", "value": 5}], + } + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(context(arguments)).artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [60, 60, 80]) + + +@pytest.mark.parametrize( + "change,reason", + [ + ("binding", "UNSUPPORTED_BINDING"), + ("coverage", "ADVERTISED_LEVELS"), + ("period", "TARGET_PERIOD_HIERARCHY"), + ("collision", "INPUT_MEASURE_COLLISION"), + ("unknown", "UNMATERIALIZED_TARGETS"), + ("empty_area", "EMPTY_ADVERTISED_GEOGRAPHY"), + ("zero_support", "UNSUPPORTED_TARGET"), + ("zero_weight", "UNSUPPORTED_TARGET"), + ], +) +def test_declared_surface_and_advertised_support_fail_closed(change, reason): + arguments, value = declaration(), frame() + if change == "binding": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "kind" + ] = "input_substitution_counterfactual" + elif change == "coverage": + arguments["advertised_scopes"] = arguments["advertised_scopes"][:3] + elif change == "period": + arguments["period"] = 2025 + elif change == "collision": + arguments["input_columns"]["person"] += ("measure_0",) + elif change == "unknown": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "value_variable" + ] = "not_produced" + elif change == "empty_area": + value.table("household").loc[2, "congressional_district_geoid"] = 3699 + elif change == "zero_support": + arguments["contract_targets"]["invented_5"]["bindings"]["policyengine"][ + "filters" + ] = [{"variable": "amount", "operator": ">", "value": 999}] + elif change == "zero_weight": + value = Frame( + {e: value.table(e) for e in value.entities}, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 0.0]), WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match=reason): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +def test_artifact_digest_projection_and_order_tamper_refusals(): + c = context() + payload = stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + digest = hashlib.sha256(payload).hexdigest() + changed = json.loads(payload) + changed["data"]["hex"] = np.full(9, 99.0, dtype=" Date: Sat, 12 Sep 2026 09:04:26 -0400 Subject: [PATCH 108/255] Fix fiscal measurement arithmetic, predicate inputs and cache identity --- ...ement-predicate-and-math-identity.fixed.md | 1 + .../microcosm/build/target_materialization.py | 4 + .../us_runtime/graph_fiscal_measurement.py | 28 ++++++- .../tests/test_target_materialization.py | 25 ++++++ .../tests/test_us_graph_fiscal_measurement.py | 76 +++++++++++++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md new file mode 100644 index 000000000..364840b61 --- /dev/null +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -0,0 +1 @@ +Bind fiscal measurement caches to target arithmetic, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 2629e4507..bc5f899e6 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -716,6 +716,10 @@ def _expression(adapter: Any, entity: str, expression: str) -> np.ndarray: total: np.ndarray | None = None for part in parts: values = _column(adapter, entity, part) + # NumPy adds bool arrays with logical-OR semantics. A sum of indicator + # terms counts their True values, so widen only boolean operands. + if values.dtype.kind == "b": + values = values.astype(np.int64) total = values.copy() if total is None else total + values if total is None: raise ValueError(f"unsupported value_expression {expression!r}") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py index 8d9ef4bfc..f4328c264 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -24,6 +24,7 @@ from microcosm.build import target_materialization from microcosm.calibrate import TargetRegistry, TargetSpec, matrix +from microcosm.calibrate import target as target_math from microcosm.calibrate.hierarchy import CalibrationHierarchy, HierarchyGeography from microcosm.frame import US_SCHEMA, Frame from microcosm.frame.adapters import policyengine_us @@ -306,6 +307,14 @@ def column(self, entity, variable): def has_column(self, entity, variable): return variable in self.tables[entity] + def require_known_predicate(self, entity, variable): + # A missing comparison input is unknown, even when NumPy would turn + # the comparison into False or treat NaN as a nonzero target mask. + _require( + not pd.isna(self.column(entity, variable)).any(), + "MISSING_PREDICATE_INPUT:" + entity + "." + variable, + ) + def set_column(self, entity, variable, values): _require(variable not in self.tables[entity], "MEASUREMENT_COLUMN_COLLISION") array = np.asarray(values) @@ -515,6 +524,7 @@ def implementation_hash(self): sys.modules[__name__], target_materialization, matrix, + target_math, TargetSpec, Frame, policyengine_us, @@ -588,15 +598,31 @@ def run(self, context: KernelContext) -> KernelResult: contract["closure"]["root"], outputs[contract["closure"]["root"]], ) + bindings = json.loads(params["contract_targets"]) + predicates = { + (spec.entity, predicate["variable"]) + for spec in registry + for predicate in bindings[spec.metadata["contract_target_id"]]["bindings"][ + "policyengine" + ].get("filters", ()) + } + for entity, variable in sorted(predicates): + adapter.require_known_predicate(entity, variable) prepared = target_materialization.materialize_target_bindings( adapter, registry, - json.loads(params["contract_targets"]), + bindings, period=params["period"], ) _require( not prepared.skipped, "UNMATERIALIZED_TARGETS:" + str(prepared.skipped) ) + # TargetSpec.filter is evaluated by the CSR compiler, independently + # of contract binding predicates, and may use a prepared measure. + for entity, variable in sorted( + {(s.entity, s.filter) for s in registry if s.filter is not None} + ): + adapter.require_known_predicate(entity, variable) work = Frame( adapter.tables, US_SCHEMA, {"household": context.weights["household"]} ) diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 4c81e6173..7b1592e3b 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -158,6 +158,31 @@ def test_prepared_column_path_materializes_filtered_values(): ] +@pytest.mark.parametrize( + "left,right,expected", + [ + ([True, False, True], [True, False, False], [2.0, 0.0, 1.0]), + ([True, False, True], [2, 3, 4], [3.0, 3.0, 5.0]), + ([2, 3, 4], [True, False, True], [3.0, 3.0, 5.0]), + ([1, 2, 3], [4, 5, 6], [5.0, 7.0, 9.0]), + ([0.25, -0.5, 2.0], [0.5, 1.5, -1.0], [0.75, 1.0, 1.0]), + ], +) +def test_shared_expression_uses_arithmetic_for_boolean_terms(left, right, expected): + adapter = StubAdapter() + adapter.tables["person"]["left"] = np.asarray(left) + adapter.tables["person"]["right"] = np.asarray(right) + registry = _resolution_registry("sum") + result = materialize_target_bindings( + adapter, + registry, + {"sum": {"bindings": {"policyengine": {"value_expression": "left + right"}}}}, + period=2025, + ) + assert result.skipped == () + np.testing.assert_array_equal(adapter.tables["person"]["sum_measure"], expected) + + @pytest.mark.parametrize("fact_period", [2024, 2025]) @pytest.mark.parametrize("require_matching_fact_period", [None, False, True]) def test_existing_measure_respects_fact_guard_at_default_measurement_period( diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py index 24dcb951e..268bfa2c6 100644 --- a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py @@ -11,6 +11,7 @@ from microcosm.build.us_runtime import graph_fiscal_measurement as stage from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate import target as target_math from microcosm.calibrate.hierarchy import ( CalibrationHierarchy, HierarchyCategory, @@ -206,6 +207,81 @@ def test_actual_shared_interpreter_sum_and_filter(): np.testing.assert_array_equal(measured.matrix.toarray()[0], [60, 60, 80]) +def test_boolean_indicator_expression_is_an_arithmetic_sum(): + arguments = declaration() + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = {"value_expression": "keep + keep"} + value = frame() + value.person["keep"] = [True, False, True, True] + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [2, 2, 2]) + + +@pytest.mark.parametrize("kind", ["float", "nullable_boolean"]) +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_missing_consumed_predicate_is_refused(kind, location): + arguments, value = declaration(), frame() + variable = "amount" if kind == "float" else "keep" + if kind == "float": + value.person.loc[1, variable] = np.nan + predicate = {"variable": variable, "operator": ">", "value": 5} + else: + value.person[variable] = pd.Series([True, pd.NA, True, True], dtype="boolean") + predicate = {"variable": variable, "equals": True} + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [predicate] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter=variable) for s in arguments["registry"]], country="us" + ) + with pytest.raises( + ValueError, match="MISSING_PREDICATE_INPUT:person\\." + variable + ): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_known_false_predicates_and_unconsumed_missing_inputs_are_preserved(location): + arguments, value = declaration(), frame() + value.person["keep"] = [True, False, True, True] + value.person.loc[0, "amount"] = np.nan # This measure does not read amount. + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [ + {"variable": "keep", "equals": True} + ] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter="keep") for s in arguments["registry"]], country="us" + ) + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [1, 1, 1]) + + +def test_implementation_identity_includes_target_arithmetic_source( + tmp_path, monkeypatch +): + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + source = Path(target_math.__file__).read_text() + original = " return values * filter_mask\n" + assert source.count(original) == 1 + changed.write_text( + source.replace(original, " return 2.0 * values * filter_mask\n") + ) + monkeypatch.setattr(target_math, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + @pytest.mark.parametrize( "change,reason", [ From 852967cac8a8e3256e5fd0ebfab8d5391f3f1291 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:09:12 -0400 Subject: [PATCH 109/255] Bind fiscal engine table transforms into cache identity --- ...measurement-predicate-and-math-identity.fixed.md | 2 +- .../build/us_runtime/graph_fiscal_measurement.py | 3 +++ .../tests/test_us_graph_fiscal_measurement.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md index 364840b61..8f4a31c03 100644 --- a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -1 +1 @@ -Bind fiscal measurement caches to target arithmetic, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. +Bind fiscal measurement caches to target arithmetic and engine table materialization, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py index f4328c264..6b6ce6bde 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -528,6 +528,9 @@ def implementation_hash(self): TargetSpec, Frame, policyengine_us, + # The adapter delegates table/weight materialization + # to this helper's separate defining module. + policyengine_us.engine_tables, runtime, _context_digest, dependencies=self.capabilities.dependencies, diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py index 268bfa2c6..ad80a0f09 100644 --- a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py @@ -19,6 +19,7 @@ HierarchyNode, ) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import materialize as frame_materialize from microcosm.graph import ( Capabilities, ContentStore, @@ -282,6 +283,18 @@ def test_implementation_identity_includes_target_arithmetic_source( assert stage.FiscalMeasurementKernel().implementation_hash() != before +def test_implementation_identity_includes_engine_table_transform(tmp_path, monkeypatch): + assert stage.policyengine_us.engine_tables is frame_materialize.engine_tables + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_engine_tables.py" + source = Path(frame_materialize.__file__).read_text() + original = " ).values\n" + assert source.count(original) == 1 + changed.write_text(source.replace(original, " ).values * 2.0\n")) + monkeypatch.setattr(frame_materialize, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + @pytest.mark.parametrize( "change,reason", [ From ceae455e703e423c3c81baf7a56cc36e50bc46f1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:13:12 -0400 Subject: [PATCH 110/255] Update US release path with PUF and measurement progress --- docs/us-uk-release-path.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index a75930fc9..4a38ab152 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -42,9 +42,9 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | -| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; complete PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF host passes invented cold/replay controls, with native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | | Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict replay is running; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | -| Calibration | Solver controls pass; fully enriched native national/CD solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Calibration | Declared national/state/CD measurement stage and solver controls pass; dense graph integration and fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | | Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | @@ -68,6 +68,31 @@ consolidated against the amended main interface. These software changes still need their own native candidate verification; the frozen pilot does not certify the newer source. +The maintained PUF55 host is now adopted. Its 31 small controls and two actual +full-host controls pass on invented inputs: both conditioning routes, 110 fits +and 110 applications, cold execution, required replay, whole-population +preservation and numerical finalizer equality. The full-host run took 742 +seconds and 0.74 GB peak RSS. Those 64 invented donors do not establish the +runtime or quality of the full native donor. A separate current-source native +packet uses a fresh writable store and the actual retained financial run. +See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). + +The new fiscal measurement stage passes all 66 interpreter/stage checks on the +combined integration, including an actual US-engine dividend calculation. It +uses declared target bindings, household-aligned CSR measurements and explicit +national/state/CD masks. Missing consumed predicates refuse, indicator sums +are numeric, and local calculation helpers participate in cache identity. +Specialized unsupported bindings refuse; this is not full fiscal-registry +coverage. The dense grouped weight solve is being integrated separately. + +Complete-population ancestry remains an explicit requirement. The PUF host +checks its output before return but does not issue a persistent authority token +for its public dataclass. Later calibration and export must check the actual +retained output; they cannot treat its receipt as an admission mechanism. The +minimal invented fixture's 61 present input names out of 161 are diagnostic +only, with unresolved original-channel values. Native coverage will be measured +on the new candidate rather than inferred from this fixture. + The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale and release verification. The existing age-development runner is a separate From e01955780626aa08a55e86d45df5501d922b3608 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:35:32 -0400 Subject: [PATCH 111/255] Record passing native financial required replay --- docs/us-uk-release-path.md | 25 ++++---- ...us-financial19-required-replay-20260912.md | 60 +++++++++++++++++++ 2 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 experiments/us-financial19-required-replay-20260912.md diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 4a38ab152..3aa736b25 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -43,7 +43,7 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | | Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF host passes invented cold/replay controls, with native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | -| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict replay is running; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Declared national/state/CD measurement stage and solver controls pass; dense graph integration and fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | @@ -58,15 +58,20 @@ source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. This supersedes the earlier assignment-before-cloning milestone for ordering; it does not establish full PUF enrichment or national/CD calibration. -On 12 September, a strict required replay of that frozen source was launched -with a fresh output directory and a read-only retained store. It must verify -all source bytes again, reuse every node, and reproduce complete artifact -content, excluding only operational timing and cache-hit fields. Its outcome -is pending. Meanwhile the maintained graph's observer-mutation defect has been -fixed and independently reviewed, and the shared graph changes are being -consolidated against the amended main interface. These software changes still -need their own native candidate verification; the frozen pilot does not -certify the newer source. +On 12 September, a strict required replay of that frozen source passed with a +fresh output directory and a read-only retained store. It verified the source +bytes again, reused all nine prefix and nineteen financial-graph nodes, and +reproduced the four non-manifest exports byte for byte. The portable manifest +also matched after excluding only operational timing and cache-hit fields. +The closed run took 3,981 seconds and peaked at 10.99 GB RSS; source, resource, +native-input, owned-file and thread-control checks remained unchanged, with no +unexpected access refusals. See the +[replay record](../experiments/us-financial19-required-replay-20260912.md). +Meanwhile the maintained graph's observer-mutation defect has been fixed and +independently reviewed, and the shared graph changes are being consolidated +against the amended main interface. Those software changes still need their +own native candidate verification; this replay certifies only its frozen +financial stage, not the newer source or a release. The maintained PUF55 host is now adopted. Its 31 small controls and two actual full-host controls pass on invented inputs: both conditioning routes, 110 fits diff --git a/experiments/us-financial19-required-replay-20260912.md b/experiments/us-financial19-required-replay-20260912.md new file mode 100644 index 000000000..f0865222b --- /dev/null +++ b/experiments/us-financial19-required-replay-20260912.md @@ -0,0 +1,60 @@ +# Native financial graph required replay — 12 September 2026 + +The frozen nineteen-node financial graph passed a separate required-cache +replay on real ACS/ASEC inputs. This accepts replay of the earlier financial +pilot, not PUF enrichment, a calibration, a current-source population or release. + +## Exact scope + +- Source: `2ca11c85a0f710b8fa495e363243b940f7eabae2`. +- One collected test passed; outer guard result `0`. +- Request: source fraction `1/1000`, survey/geography seed `20260908`, two + trees, demographic conditioning enabled, `resume="require"`. +- All nine prefix nodes and all nineteen financial-graph nodes were cache hits. +- The graph, preparation, projection and recipient-matrix exports matched the + cold run byte for byte. Portable manifest content matched after removing only + start/end times and per-node timing/cache-hit fields. The complete artifact + report also matched, excluding its cache-hit and manifest-file identity fields. +- Source, resources, native inputs, owned files, baseline exports and thread + controls remained unchanged. There were no unexpected access refusals. +- Elapsed: `3981.09802937496` seconds; CPU: `3944.4073700000004` seconds; + peak RSS: `10988191744` bytes. Declared limits were 7,200 CPU seconds, + 10,800 elapsed seconds and 32 GiB peak RSS, with one numerical worker. + +The source population remains 1,584 households / 3,464 people, expanded to +3,168 households / 6,928 people before block assignment. The final stage carries +seven financial fields. The same frozen source produced the 11 September cold +pilot; no broader coverage follows from replay. + +## Evidence identities + +| Evidence | SHA256 | +| --- | --- | +| Closed replay guard receipt | `71d6981518acad0ad0fd199f906c01ccfa867796aaf053cf6fa5f5a3df1193ae` | +| Final replay manifest | `c8a1b7715bae740c183aa592fca718849e35edf6722328c1de3d83f0cacafa77` | +| Cold baseline manifest | `0d745678802d8dae16750f57ebe779644237e1b2416f1cb78edf4c59d2163f0e` | +| Reviewed guard | `5a2bcd05665644a86ff2f3fc35a2e7e751d759412380dce07193f9b0c6f3c670` | +| Manifest content key, both runs | `134ab4ea39b434066e48258be345999425a6d23f3b5016680f457fd357be9edf` | + +Retained local evidence is in +`PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run`: +`us-native-postclone-financial19-required-20260912.json`, +`native-financial-manifest.json`, and the reviewed harness and helper. +The coordinator's metadata postcheck is +`PolicyEngine/_recovered/scratch-backup/893/codex-takeover-20260912/native-required-replay-postcheck.json`. +It rechecked receipt consistency, exact owned-file hashes, reported bounds, +node hits and export identities without reading native data or generated Frame +payloads again. The actual payload comparison and final source/output checks ran +inside the reviewed native harness before its successful guard closure. + +The guard's expected optional dependency probes remain recorded separately from +unexpected refusals. The run enabled no engine, network, child processes or +automatic retry. Original cold and failed-attempt evidence remains retained. + +## Next acceptance + +The maintained integration includes later runtime repairs and the restored +PUF55 host. Its native candidate needs a fresh cold execution, required replay, +complete input coverage and full Frame export/readback on that identified source. +Calibration and complete-population acceptance follow separately. This replay +does not authorize reuse of the serialized receipt as a live population issuer. From d07eb7ebc8be4e107fa9cd3c28e695b4e3bd7730 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:13:22 -0400 Subject: [PATCH 112/255] Add constrained dense fiscal calibration graph stage --- .../us-fiscal-dense-calibration.added.md | 1 + .../graph_fiscal_dense_calibration.py | 602 ++++++++++++++++++ .../test_us_graph_fiscal_dense_calibration.py | 473 ++++++++++++++ 3 files changed, 1076 insertions(+) create mode 100644 changelog.d/us-fiscal-dense-calibration.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py create mode 100644 packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py diff --git a/changelog.d/us-fiscal-dense-calibration.added.md b/changelog.d/us-fiscal-dense-calibration.added.md new file mode 100644 index 000000000..497b5415f --- /dev/null +++ b/changelog.d/us-fiscal-dense-calibration.added.md @@ -0,0 +1 @@ +Add a numerical fiscal graph calibration stage using existing grouped Adam, explicit original weight and origin bounds, exact household alignment, and target and origin diagnostics without claiming source or release admission. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py new file mode 100644 index 000000000..1c16e0cc8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py @@ -0,0 +1,602 @@ +"""Numerical dense fiscal calibration; complete-parent admission stays external. + +The caller supplies original DESIGN anchors and origin-derived absolute caps. +This module transports and checks those numbers but cannot authenticate their +source or the complete Population. The host must compare original anchors and +admit the complete parent and successor on cold execution, replay and return. + +One existing grouped Adam/free-mass solve preserves every initial zero exactly. +There is no pruning, restart, repeated calibration or row-cap relaxation. CSR +rows enter the existing public Target callable API one at a time: O(N) temporary +row memory, plus sparse storage, rather than a dense target-by-household table. +The existing solver chooses sparse/dense Torch storage for the resulting matrix; +its capped scaled-MAPE objective uses float32 and accepted weights use float64. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.calibrate import TargetSet, calibrate, diagnostics_payload, group_bounds +from microcosm.calibrate import kernels as calibration_kernels +from microcosm.calibrate import target as target_module +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.keys import opaque_artifact_key + +from . import graph_fiscal_measurement as measurement +from .demographic_calibration_graph import DIAGNOSTICS_TYPE + +BOUNDS_TYPE = ArtifactType("microcosm.us.fiscal_calibration_numeric_bounds", 1) +BOUNDS_PROTOCOL = "microcosm.us.fiscal-calibration-numeric-bounds.v1" +ORIGIN_TYPE = ArtifactType("microcosm.us.fiscal_calibration_origin_diagnostics", 1) +MAX_BYTES = 64 * 1024**2 +_FIELDS = { + "protocol", + "population", + "budget_sha256", + "household_ids", + "original_design", + "incoming", + "group_indices", + "group_upper", + "row_upper", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_CALIBRATION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def _same_array(a, b): + return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes() + + +@dataclass(frozen=True) +class FiscalCalibrationBounds: + """Decoded numerical references only; never an issued sampling budget.""" + + document: dict + household_ids: np.ndarray + original_design: Weights + incoming: Weights + grouped: group_bounds.GroupedUpperBounds + row_upper: np.ndarray + + +def decode_fiscal_calibration_bounds(payload: bytes) -> FiscalCalibrationBounds: + """Read bounded, canonical typed-ID and exact float64 reference arrays.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "BOUNDS_PAYLOAD") + try: + raw = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("FISCAL_CALIBRATION_BOUNDS_JSON") from None + _require(type(raw) is dict and set(raw) == _FIELDS, "BOUNDS_FIELDS") + _require(canonical_json(raw) == payload, "BOUNDS_CANONICAL") + _require(raw["protocol"] == BOUNDS_PROTOCOL, "BOUNDS_PROTOCOL") + _require(_digest(raw["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(raw["population"]) is str and 0 < len(raw["population"]) <= 256, + "POPULATION", + ) + ids = measurement._read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + vectors = { + name: measurement._read_array(raw[name], {"= 0) for v in vectors.values()), + "REFERENCE_VALUES", + ) + _require( + all( + len(vectors[n]) == len(ids) + for n in ("original_design", "incoming", "row_upper") + ), + "REFERENCE_ALIGNMENT", + ) + _require(np.any(vectors["incoming"] > 0), "EMPTY_POSITIVE_SUPPORT") + groups = group_bounds.GroupedUpperBounds( + ids, + measurement._read_array(raw["group_indices"], {" bytes: + """Freeze explicitly supplied original references without issuing authority. + + ``row_upper`` is already the original source-reference cap, not an inferred + multiple of incoming or cloned DESIGN values. Its provenance and arithmetic + belong to the original budget owner. Full uint64 IDs and row order survive. + """ + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN, + "DESIGN_KIND", + ) + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + type(grouped_upper_bounds) is group_bounds.GroupedUpperBounds, "GROUP_TYPE" + ) + _require( + type(household_ids) is np.ndarray + and household_ids.dtype.str in {" 0), + "FIXED_ZERO_SUPPORT", + ) + + +def verify_fiscal_calibration_bounds( + payload, + *, + household_ids, + incoming_weights, + original_design_weights, +) -> FiscalCalibrationBounds: + """Compare numeric bytes to a host's live references; not source admission. + + The host must obtain original_design_weights from retained Population + anchors and separately authenticate the original group and row caps. + """ + bounds = decode_fiscal_calibration_bounds(payload) + _require(_same_array(bounds.household_ids, household_ids), "HOUSEHOLD_ALIGNMENT") + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + _same_array(bounds.incoming.values, incoming_weights.values), "INCOMING_ANCHOR" + ) + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN + and _same_array(bounds.original_design.values, original_design_weights.values), + "DESIGN_ANCHOR", + ) + return bounds + + +def _measurement_node(raw): + params = raw["params"] + result = measurement.fiscal_measurement_node( + measurement._registry(json.loads(params["registry"])), + population=raw["population"], + node_id=raw["id"], + input_columns={s["entity"]: tuple(s["columns"]) for s in raw["inputs"]}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=measurement._scopes(json.loads(params["advertised_scopes"])), + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=tuple(params["model_outputs"]), + ) + _require( + canonical_json(normative(result)) == canonical_json(raw), "MEASUREMENT_NODE" + ) + return result + + +def fiscal_dense_calibration_node( + *, + measurement_node: Node, + bounds_node: str, + epochs: int = 256, + learning_rate: float = 0.02, + node_id: str = "us.fiscal_dense_calibration", +) -> Node: + """Declare one weight-only, fixed-zero grouped solve on the measured parent.""" + _require(type(epochs) is int and 1 <= epochs <= 10000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + raw = canonical_json(normative(measurement_node)) + _require(len(raw) <= 2 * measurement.MAX_DECLARATION_BYTES, "DECLARATION_SIZE") + verified = _measurement_node(json.loads(raw)) + return Node( + node_id, + FiscalDenseCalibrationKernel.ref, + base=verified.population, + inputs=verified.inputs, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + "measurement_node": raw.decode(), + "epochs": epochs, + "learning_rate": learning_rate, + "method": "adam", + "seed": 0, + "grouped_preserve_zeros": True, + "l0_lambda": 0.0, + "l1_lambda": 0.0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "weight_anchor": "explicit_original_sampling_references", + "cap_enforcement": "numeric_group_projection_and_final_row_refusal", + }, + artifact_inputs=( + ArtifactInput( + "measurement", verified.id, "measurement", measurement.MEASUREMENT_TYPE + ), + ArtifactInput("bounds", bounds_node, "numeric_bounds", BOUNDS_TYPE), + ), + artifact_outputs=( + ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE), + ArtifactOutput("origin_diagnostics", ORIGIN_TYPE), + ), + ) + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output), + "ARTIFACT_EDGE", + ) + return value.payload + + +class FiscalDenseCalibrationKernel(KernelBase): + """Use the existing solver and independently check the compiled measurement.""" + + ref = "us.fiscal_dense_calibration@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + measurement, + group_bounds, + target_module, + measurement.TargetSpec, + measurement.CalibrationHierarchy, + calibration_kernels.diagnostics_module.assemble_target_loss_attribution, + dependencies=self.capabilities.dependencies, + ), + "solver": calibration_kernels.CALIBRATE_ADAM.implementation_hash(), + } + ) + ) + + def run(self, context): + params = context.params + _require( + tuple(a.name for a in context.node.artifact_inputs) + == ("measurement", "bounds"), + "ALIASES", + ) + measured_node = _measurement_node(json.loads(params["measurement_node"])) + expected = fiscal_dense_calibration_node( + measurement_node=measured_node, + bounds_node=context.node.artifact_inputs[1].producer, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + node_id=context.node.id, + ) + _require( + normative(context.node) == normative(expected) + and dict(params) == dict(expected.params), + "DECLARATION", + ) + _require( + context.weights["household"].kind is WeightKind.IMPORTANCE, + "IMPORTANCE_REQUIRED", + ) + _require(not context.sources, "UNDECLARED_SOURCE") + raw_measurement = _artifact( + context, "measurement", measurement.MEASUREMENT_TYPE, "measurement" + ) + measured_context = replace( + context, node=measured_node, params=measured_node.params, artifacts={} + ) + measured = measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + raw_bounds = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + bounds = decode_fiscal_calibration_bounds(raw_bounds) + _require( + bounds.document["population"] == context.node.base, "BOUNDS_POPULATION" + ) + _require( + _same_array(bounds.household_ids, measured.household_ids), + "HOUSEHOLD_ALIGNMENT", + ) + _require( + _same_array(bounds.incoming.values, context.weights["household"].values), + "INCOMING_ANCHOR", + ) + table = pd.DataFrame({"household_id": measured.household_ids.copy()}) + work = calibration_kernels._frame_from_context( + replace( + context, + tables={"household": table}, + weights={"household": bounds.incoming}, + ), + "household", + ) + targets = [] + for i, spec in enumerate(measured.registry): + # The public callable API materializes only this one sparse row. + def row(_frame, position=i): + return measured.matrix[position : position + 1].toarray().ravel() + + targets.append( + replace(spec.to_target(), entity="household", measure=row, filter=None) + ) + result = calibrate( + work, + TargetSet(targets), + weight_entity="household", + method="adam", + seed=0, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + mass="free", + max_weight_ratio=None, + target_records=None, + l0_lambda=0.0, + l1_lambda=0.0, + l2_lambda=0.0, + l2_anchor="initial", + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + ) + _require( + not result.skipped + and result.problem.names + == tuple(s.to_target().row_name for s in measured.registry), + "TARGET_ALIGNMENT", + ) + _require( + _same_array(result.problem.target_vector, measured.target_values) + and result.problem.matrix.shape == measured.matrix.shape + and all( + _same_array( + getattr(result.problem.matrix, k), getattr(measured.matrix, k) + ) + for k in ("indptr", "indices", "data") + ), + "MATRIX_IDENTITY", + ) + _require( + _same_array(result.initial_weights, bounds.incoming.values), + "SOLVER_INITIAL", + ) + weights = result.frame.weights_for("household") + _require(weights.kind is WeightKind.CALIBRATED, "CALIBRATED_KIND") + check_fiscal_calibration_weights(bounds, weights.values) + accepted = weights.values.tobytes() + _require( + np.isfinite(result.loss_trajectory).all() + and np.isfinite(result.final_loss), + "NONFINITE_LOSS", + ) + initial_estimates = measured.matrix @ bounds.incoming.values + final_estimates = measured.matrix @ weights.values + _require( + np.isfinite(initial_estimates).all() + and np.isfinite(final_estimates).all() + and np.isfinite(final_estimates - measured.target_values).all(), + "NONFINITE_RESIDUAL", + ) + _require( + _same_array( + np.asarray([d.initial_estimate for d in result.diagnostics]), + initial_estimates, + ) + and _same_array( + np.asarray([d.final_estimate for d in result.diagnostics]), + final_estimates, + ), + "DIAGNOSTIC_ESTIMATES", + ) + build = { + "measurement_sha256": _sha(raw_measurement), + "numeric_bounds_sha256": _sha(raw_bounds), + "budget_sha256": bounds.document["budget_sha256"], + "original_design_sha256": _sha(bounds.original_design.values.tobytes()), + "incoming_sha256": _sha(bounds.incoming.values.tobytes()), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "matrix_matches_measurement": True, + "numeric_measurement_entity": "household", + "original_measurement_entities": { + s.name: s.entity for s in measured.registry + }, + "weight_anchor": params["weight_anchor"], + "cap_enforcement": params["cap_enforcement"], + } + diagnostics = canonical_json( + diagnostics_payload( + result, + target_registry=measured.registry, + build=build, + ) + ) + origin = canonical_json( + { + "protocol": "microcosm.us.fiscal-calibration-origin-diagnostics.v1", + **build, + "household_ids": measurement._array(bounds.household_ids), + "accepted_weights": measurement._array(weights.values), + "group_totals": measurement._array( + bounds.grouped.totals(weights.values) + ), + "group_upper": measurement._array(bounds.grouped.absolute_bounds), + "group_diagnostics": bounds.grouped.diagnostics( + weights.values, + result.options["grouped_upper_bounds"][ + "last_corrected_group_count" + ], + ), + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming.values == 0)), + "active_rows": int(np.count_nonzero(weights.values > 0)), + "binding_row_caps": int( + np.count_nonzero(weights.values == bounds.row_upper) + ), + "all_row_caps_satisfied": True, + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + } + ) + _require( + len(diagnostics) <= MAX_BYTES and len(origin) <= MAX_BYTES, + "DIAGNOSTIC_SIZE", + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": diagnostics, "origin_diagnostics": origin}, + receipt={ + **build, + "diagnostics_sha256": _sha(diagnostics), + "origin_diagnostics_sha256": _sha(origin), + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + "scope": "one_numeric_dense_fiscal_solve_no_ancestry_admission", + }, + ) + check_fiscal_calibration_weights( + decode_fiscal_calibration_bounds(raw_bounds), weights.values + ) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + _same_array( + work.table("household").household_id.to_numpy(), bounds.household_ids + ) + and _same_array( + result.frame.table("household").household_id.to_numpy(), + bounds.household_ids, + ), + "FINAL_HOUSEHOLD_ALIGNMENT", + ) + measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + return output diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py new file mode 100644 index 000000000..7f69dec3f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_dense_calibration.py @@ -0,0 +1,473 @@ +"""Invented common-frame fiscal solve through existing grouped Adam and graph.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_fiscal_measurement import InventedSource, context, declaration, frame + +from microcosm.build.us_runtime import graph_fiscal_dense_calibration as stage +from microcosm.build.us_runtime import graph_fiscal_measurement as measurement +from microcosm.calibrate import TargetRegistry, group_bounds +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactOutput, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Numeric, + NumericScope, + Owned, + Slice, + SourceRef, + StoreMissError, + StructuralDelta, + WeightTransition, + compile_graph, + run_graph, +) +from microcosm.graph.keys import opaque_artifact_key + + +def setup(*, incoming=None, row_upper=None, ids=None): + value = frame() + if ids is not None: + old = value.table("household").household_id.to_numpy().copy() + mapping = dict(zip(old.tolist(), ids.tolist(), strict=True)) + value.table("household")["household_id"] = ids + value.table("person")["person_household_id"] = np.array( + [mapping[v] for v in value.person.person_household_id], dtype=ids.dtype + ) + incoming = np.array([1.0, 2.0, 3.0] if incoming is None else incoming) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(incoming, WeightKind.IMPORTANCE)}, + ) + args = declaration() + # Exact common-frame targets correspond to desired weights [2, 2, 4]. + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [10, 6, 4, 4, 2, 4], strict=True) + ], + country="us", + ) + if incoming[1] == 0: + args["registry"] = TargetRegistry( + [ + replace(s, value=v) + for s, v in zip(args["registry"], [8, 4, 4, 4, 0, 4], strict=True) + ], + country="us", + ) + mc = context(args, value) + measured = measurement.FiscalMeasurementKernel().run(mc).artifacts["measurement"] + hh_ids = value.table("household").household_id.to_numpy() + groups = group_bounds.GroupedUpperBounds(hh_ids, np.array([0, 0, 1]), [6.0, 5.0]) + bounds = stage.encode_fiscal_calibration_bounds( + population=mc.node.population, + household_ids=hh_ids, + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + incoming_weights=value.weights_for("household"), + grouped_upper_bounds=groups, + row_upper=np.array([6.0, 6.0, 5.0] if row_upper is None else row_upper), + budget_sha256=hashlib.sha256(b"invented budget; not an issuer").hexdigest(), + ) + node = stage.fiscal_dense_calibration_node( + measurement_node=mc.node, + bounds_node="invented.bounds", + epochs=120, + learning_rate=0.03, + ) + artifacts = {} + for name, payload, type_, output in ( + ("measurement", measured, measurement.MEASUREMENT_TYPE, "measurement"), + ("bounds", bounds, stage.BOUNDS_TYPE, "numeric_bounds"), + ): + key = hashlib.sha256(name.encode()).hexdigest() + artifacts[name] = ArtifactValue( + payload, + type_, + opaque_artifact_key(key, output), + key, + NumericScope(Numeric.BITWISE), + ) + return value, mc, replace(mc, node=node, params=node.params, artifacts=artifacts) + + +def test_real_grouped_solver_rebuilds_exact_matrix_and_residuals(monkeypatch): + value, mc, c = setup() + captured = [] + original_calibrate = stage.calibrate + + def record(*args, **kwargs): + result = original_calibrate(*args, **kwargs) + captured.append(result) + return result + + monkeypatch.setattr(stage, "calibrate", record) + result = stage.FiscalDenseCalibrationKernel().run(c) + solved = captured[0] + expected = measurement.decode_fiscal_measurement(c.artifacts["measurement"].payload) + np.testing.assert_array_equal( + solved.problem.matrix.toarray(), expected.matrix.toarray() + ) + assert solved.problem.names == tuple( + s.to_target().row_name for s in expected.registry + ) + assert solved.problem.target_vector.tobytes() == expected.target_values.tobytes() + assert solved.final_loss < solved.initial_loss + assert result.weights.kind is WeightKind.CALIBRATED + assert result.frame is None and not result.columns + assert result.receipt["release_eligible"] is False + assert result.receipt["source_admission"] == "required_from_complete_parent_owner" + diagnostic = json.loads(result.artifacts["diagnostics"]) + assert diagnostic["schema_version"] == 8 + origin = json.loads(result.artifacts["origin_diagnostics"]) + achieved = expected.matrix @ result.weights.values + for row, actual in zip(diagnostic["targets"], achieved, strict=True): + assert row["final_estimate"] == actual + assert origin["accepted_weights"] == measurement._array(result.weights.values) + assert origin["group_totals"] == measurement._array( + np.array([result.weights.values[:2].sum(), result.weights.values[2]]) + ) + assert origin["fixed_zero_rows"] == 0 + assert diagnostic["build"]["matrix_matches_measurement"] is True + # The direct kernel receives observations only, never a writable parent. + for entity in value.entities: + if entity in mc.tables: + pd.testing.assert_frame_equal(c.tables[entity], mc.tables[entity]) + assert ( + value.weights_for("household").values.tobytes() + == np.array([1.0, 2.0, 3.0]).tobytes() + ) + + +def test_uint64_order_original_design_and_strict_zero_support(): + ids = np.array([2**63, 2**63 + 1, 2**64 - 1], dtype=np.uint64) + _, _, c = setup(incoming=[1.0, -0.0, 3.0], ids=ids) + bounds = stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN), + ) + assert bounds.household_ids.dtype == ids.dtype + assert ( + bounds.original_design.values.tobytes() == np.array([8.0, 9.0, 10.0]).tobytes() + ) + output = stage.FiscalDenseCalibrationKernel().run(c) + assert output.weights.values[1:2].tobytes() == np.array([-0.0]).tobytes() + assert np.all(output.weights.values[[0, 2]] > 0) + stage.check_fiscal_calibration_weights(bounds, output.weights.values) + assert json.loads(output.artifacts["origin_diagnostics"])["fixed_zero_rows"] == 1 + with pytest.raises(ValueError, match="DESIGN_ANCHOR"): + stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights( + np.array([8.0, 8.0, 10.0]), WeightKind.DESIGN + ), + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("epochs", 0), + ("epochs", True), + ("epochs", 1.5), + ("learning_rate", float("nan")), + ("learning_rate", float("inf")), + ("learning_rate", 0.0), + ("learning_rate", True), + ], +) +def test_solver_settings_refuse_nonfinite_or_undeclared_modes(field, value): + _, mc, _ = setup() + with pytest.raises(ValueError): + stage.fiscal_dense_calibration_node( + measurement_node=mc.node, bounds_node="bounds", **{field: value} + ) + + +def test_misaligned_measurement_bounds_and_calibrated_parent_refuse_before_solve( + monkeypatch, +): + _, _, c = setup() + + def forbidden(*a, **kw): + pytest.fail("solver ran before admission") + + monkeypatch.setattr(stage, "calibrate", forbidden) + with pytest.raises(ValueError, match="IMPORTANCE"): + stage.FiscalDenseCalibrationKernel().run( + replace( + c, + weights={ + "household": Weights( + np.array([1.0, 2.0, 3.0]), WeightKind.CALIBRATED + ) + }, + ) + ) + changed = dict(c.tables) + changed["person"] = c.tables["person"].copy() + changed["person"]["amount"] += 1 + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + stage.FiscalDenseCalibrationKernel().run(replace(c, tables=changed)) + raw = json.loads(c.artifacts["bounds"].payload) + raw["household_ids"] = measurement._array(np.array([30, 20, 10], dtype=np.int64)) + artifacts = dict(c.artifacts) + artifacts["bounds"] = replace( + artifacts["bounds"], payload=measurement.canonical_json(raw) + ) + with pytest.raises(ValueError, match="HOUSEHOLD_ALIGNMENT"): + stage.FiscalDenseCalibrationKernel().run(replace(c, artifacts=artifacts)) + + +def test_group_and_row_caps_refuse_without_clipping(): + _, _, c = setup(row_upper=[1.0, 6.0, 5.0]) + with pytest.raises(ValueError, match="ROW_REFERENCE_CAP"): + stage.FiscalDenseCalibrationKernel().run(c) + bounds = stage.decode_fiscal_calibration_bounds(c.artifacts["bounds"].payload) + with pytest.raises(ValueError, match="frozen absolute bounds"): + stage.check_fiscal_calibration_weights(bounds, np.array([5.0, 2.0, 3.0])) + + +@pytest.mark.parametrize( + "change,reason", + [ + ("names", "TARGET_ALIGNMENT"), + ("matrix", "MATRIX_IDENTITY"), + ("diagnostic", "DIAGNOSTIC_ESTIMATES"), + ], +) +def test_solver_return_with_wrong_matrix_or_target_order_is_refused( + monkeypatch, change, reason +): + _, _, c = setup() + actual = stage.calibrate + + def changed(*a, **kw): + r = actual(*a, **kw) + if change == "names": + return replace( + r, problem=replace(r.problem, names=tuple(reversed(r.problem.names))) + ) + if change == "matrix": + matrix = r.problem.matrix.copy() + matrix.data[0] += 1 + return replace(r, problem=replace(r.problem, matrix=matrix)) + return replace( + r, + diagnostics=( + replace(r.diagnostics[0], final_estimate=999.0), + *r.diagnostics[1:], + ), + ) + + monkeypatch.setattr(stage, "calibrate", changed) + with pytest.raises(ValueError, match=reason): + stage.FiscalDenseCalibrationKernel().run(c) + + +class InventedImportance(KernelBase): + ref = "test.fiscal_dense_importance@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult( + weights=Weights(np.array(context.params["incoming"]), WeightKind.IMPORTANCE) + ) + + +class InventedBounds(KernelBase): + ref = "test.fiscal_dense_bounds@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + payload = context.sources["bounds"].read_bytes() + stage.decode_fiscal_calibration_bounds(payload) + return KernelResult(artifacts={"numeric_bounds": payload}) + + +def graph_fixture(tmp_path): + value, mc, c = setup(incoming=[1.0, 0.0, 3.0]) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN)}, + ) + create = Node( + "invented.design", + InventedSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "amount", "float64"), + Owned("person", "keep", "bool"), + Owned("tax_unit", "tax_unit_amount", "float64"), + Owned("household", "state_fips", "int64"), + Owned("household", "congressional_district_geoid", "int64"), + ), + ) + importance = Node( + "invented", + InventedImportance.ref, + base=create.id, + structural=StructuralDelta.REWEIGHT, + inputs=(Slice("household", ("state_fips",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + params={"incoming": (1.0, 0.0, 3.0)}, + ) + bounds = Node( + "invented.bounds", + InventedBounds.ref, + population=importance.id, + sources=("bounds",), + artifact_outputs=(ArtifactOutput("numeric_bounds", stage.BOUNDS_TYPE),), + ) + nodes = (create, importance, bounds, mc.node, c.node) + sources = (SourceRef("fixture", "frame-store"), SourceRef("bounds", "raw-bytes-v1")) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedImportance(), + InventedBounds(), + measurement.FiscalMeasurementKernel(), + stage.FiscalDenseCalibrationKernel(), + ): + kernels.register(kernel) + store = ContentStore(tmp_path / "store") + path = store.put_frame( + hashlib.sha256(b"invented fiscal dense fixture").hexdigest(), value + ) + bound_path = tmp_path / "bounds.json" + bound_path.write_bytes(c.artifacts["bounds"].payload) + arguments = { + "store": store, + "kernels": kernels, + "sources": {"fixture": path, "bounds": bound_path}, + } + return value, sources, nodes, arguments + + +def test_real_graph_cold_required_replay_preserves_parent_and_original_anchors( + tmp_path, +): + original, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + anchors = {} + + def observe(node_id, population): + anchors[node_id] = population.design_weights["household"].tobytes() + + cold = run_graph(compiled, **arguments, _population_observer=observe) + cold_anchors = dict(anchors) + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=observe + ) + assert cold.key == warm.key and all(n.store_hit for n in warm.nodes.values()) + before = warm.population("invented") + after = warm.population(nodes[-1].id) + # The graph performs precisely DESIGN -> IMPORTANCE -> CALIBRATED. + assert after.weights_for("household").kind is WeightKind.CALIBRATED + assert before.weights_for("household").kind is WeightKind.IMPORTANCE + for entity in original.entities: + pd.testing.assert_frame_equal(before.table(entity), original.table(entity)) + pd.testing.assert_frame_equal(after.table(entity), original.table(entity)) + assert after.weights_for("household").values[1] == 0 + assert ( + cold.population(nodes[-1].id).weights_for("household").values.tobytes() + == after.weights_for("household").values.tobytes() + ) + # Detached observer snapshots expose original anchors without owner authority. + assert anchors == cold_anchors + assert anchors[nodes[-1].id] == np.array([8.0, 9.0, 10.0]).tobytes() + payload = arguments["store"].load_bytes( + warm.node(nodes[-1].id).opaque_artifacts["origin_diagnostics"] + ) + assert json.loads(payload)["fixed_zero_rows"] == 1 + # A repeated CALIBRATED -> CALIBRATED stage is refused before another solve. + new_measure = replace(nodes[-2], id="repeat.measurement", population=nodes[-1].id) + repeated = stage.fiscal_dense_calibration_node( + measurement_node=new_measure, + bounds_node=nodes[2].id, + node_id="repeat.calibration", + ) + compiled_repeat = compile_graph( + Graph("us", sources, (*nodes, new_measure, repeated)) + ) + with pytest.raises( + NodeRejectedError, match="(?i)(importance|calibrated|transition|forward)" + ): + run_graph(compiled_repeat, **arguments) + + +def test_real_graph_replay_refuses_changed_bounds_and_target_declaration(tmp_path): + _, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + run_graph(compiled, **arguments) + changed_measure = measurement.fiscal_measurement_node(**declaration()) + changed_solve = stage.fiscal_dense_calibration_node( + measurement_node=changed_measure, bounds_node=nodes[2].id + ) + changed = compile_graph( + Graph("us", sources, (*nodes[:3], changed_measure, changed_solve)) + ) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(changed, **arguments, resume="require") + path = arguments["sources"]["bounds"] + raw = json.loads(path.read_bytes()) + raw["group_upper"] = measurement._array(np.array([5.5, 5.0])) + path.write_bytes(measurement.canonical_json(raw)) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(compiled, **arguments, resume="require") + + +def test_bound_codec_refuses_noncanonical_values_and_wrong_types(): + _, _, c = setup() + payload = c.artifacts["bounds"].payload + for field, value in ( + ("incoming", np.array([1.0, float("nan"), 3.0])), + ("group_indices", np.array([0, 0, 9], dtype=np.int64)), + ("original_design", np.array([8.0, 9.0])), + ("household_ids", np.array([10, 10, 30], dtype=np.int64)), + ): + raw = json.loads(payload) + raw[field] = measurement._array(value) + with pytest.raises(ValueError): + stage.decode_fiscal_calibration_bounds(measurement.canonical_json(raw)) + with pytest.raises(ValueError, match="CANONICAL"): + stage.decode_fiscal_calibration_bounds(payload + b"\n") + + +def test_implementation_identity_binds_existing_solver_and_target_math( + monkeypatch, tmp_path +): + # Point source identity at changed synthetic bytes without editing package source. + baseline = stage.FiscalDenseCalibrationKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + changed.write_text("# invented changed target arithmetic\n") + monkeypatch.setattr(stage.target_module, "__file__", str(changed)) + assert stage.FiscalDenseCalibrationKernel().implementation_hash() != baseline From 99543c3bf9c49d9ee953aa1e8241f00f6b086506 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:42:12 -0400 Subject: [PATCH 113/255] Document integrated dense calibration validation --- docs/us-uk-release-path.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 3aa736b25..18162ad3a 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -44,7 +44,7 @@ common source population alone does not guarantee identical estimates. | --- | --- | --- | | Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF host passes invented cold/replay controls, with native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | | Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | -| Calibration | Declared national/state/CD measurement stage and solver controls pass; dense graph integration and fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | | Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | | Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | @@ -88,7 +88,18 @@ uses declared target bindings, household-aligned CSR measurements and explicit national/state/CD masks. Missing consumed predicates refuse, indicator sums are numeric, and local calculation helpers participate in cache identity. Specialized unsupported bindings refuse; this is not full fiscal-registry -coverage. The dense grouped weight solve is being integrated separately. +coverage. + +The dense grouped calibration stage is now integrated. Independent review of +its numerical adapter found no actionable defects, and all 167 selected +calibration tests passed. The combined integration also passed all 42 tests +for the measurement and dense stages together. The adapter uses the existing +grouped Adam solver, keeps initially zero weights fixed, enforces group bounds +and refuses a result exceeding the declared original row caps. It returns +calibrated weights and schema-8 diagnostics. Numeric bound arrays do not prove +their source: the complete-population host must authenticate the original +sampling references and retained parent before and after use. Native fit, +full-size memory and calibration acceptance remain unestablished. Complete-population ancestry remains an explicit requirement. The PUF host checks its output before return but does not issue a persistent authority token From eb1f1caa15111984459289a54140d1bb62be5557 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:44:49 -0400 Subject: [PATCH 114/255] Open the #908 lane journal for the per-target snapshot slice Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshots.md | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 PROGRESS-908-target-snapshots.md diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md new file mode 100644 index 000000000..affcd9cb1 --- /dev/null +++ b/PROGRESS-908-target-snapshots.md @@ -0,0 +1,38 @@ +# microcosm#908 — per-target calibration estimate snapshots (bounded slice 1) + +Lane journal for branch `calibration-target-snapshots-908-20260912`. +Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. + +## State + +In progress. Bounded first slice only — see "Out of scope for this slice". + +## Scope of this slice + +1. A shared, leaf-level typed snapshot codec in `microcosm-calibrate`: + schema name + version, aggregate-only per-target rows, ordered target + identity digest, finite-value and ordering validation, signed relative + error with the repo's existing zero-target convention. +2. Real emission from the actual Adam solver loop in + `microcosm.calibrate.solve`, at a configurable bounded cadence with an + explicit every-epoch option, opt-in (off by default). +3. Honest iterate labelling: current / best_retained / selected, with the + selected snapshot taken from the weights the solver actually returns. +4. Atomic local `latest.json` plus immutable, bounded history chunks. +5. A synthetic-only benchmark at US/UK-sized dimensions (invented matrices). + +## Out of scope for this slice (remaining #908 acceptance scope) + +- Version-2 staging upload / remote publication (PR896's country host). +- Calibration Diagnostics UI (different repository, by the issue's own text). +- Real UK/US native calibration measurements — this host is forbidden native + microdata, engine runs and installs, so no real-run cadence default is + announced here. + +## Done + +- (nothing yet) + +## Next + +- Read scouts, write failing tests, implement, bench, commit. From d91b8996c0bcd8bd0f11bab1342a315092e042f3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:56:08 -0400 Subject: [PATCH 115/255] Emit per-target calibration estimate snapshots from the solver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #908's first bounded slice. The solver already materializes the complete target-estimate vector once per epoch and drops it: only the scalar loss reaches progress_callback, and the per-target picture appears only after the run, in calibration_diagnostics.json. microcosm.calibrate.target_snapshots is the shared, leaf-level codec for publishing that vector mid-run. A snapshot is aggregate-only by construction — one row per calibration target, no weights, no record index, and validation refuses an unknown top-level key or a record-level key name at any depth, so the context seam cannot smuggle record data. The prohibited names match the version-2 staging content policy's, so a payload that passes here is not rejected later at publication. Iterate labelling is honest rather than convenient. `current` is the epoch's iterate; `best_retained` is attached only when those values ARE the incumbent best AND the optimizer is retaining one at all (gated, mass-conserved and L2 runs retain none, and say so rather than nominating a newest-is-best); `selected` is taken from the weights calibrate returns, off the same float64 estimates the final diagnostics use. In-loop snapshots are float32-derived and record that, matching the existing selection receipt's float32 honesty. Emission reads `estimate.detach()` — the tensor the epoch's loss was already computed from. It never re-runs the forward pass, which would redraw the hard-concrete gates' noise and move the RNG stream. With the observer off nothing runs; with it on the weights and trajectory are unchanged, which a test pins directly. Identity survives multi-phase runs and sparse-selection searches: each budget probe carries its own (iteration, iters, l0_lambda), l0_selection and post_l0_refit carry their phase, and one monotone sequence counter shared across every phase view recovers true emission order where epoch numbers restart. targets_sha256 digests the ordered (name, target value) list, so reordering or revaluing breaks it. The local store replaces latest.json atomically (temp file, fsync, os.replace, parent fsync) so a concurrent dashboard read never sees a partial document, and writes each snapshot once to an immutable history chunk with a bounded retention whose drops are recorded, not silent. Opt-in throughout; no production cadence default is announced, because no native calibration was measured. Co-Authored-By: Claude Opus 5 --- .../908-calibration-target-snapshots.added.md | 1 + .../src/microcosm/calibrate/__init__.py | 20 + .../src/microcosm/calibrate/solve.py | 105 ++- .../microcosm/calibrate/target_snapshots.py | 710 ++++++++++++++++++ .../tests/test_target_snapshots.py | 474 ++++++++++++ 5 files changed, 1309 insertions(+), 1 deletion(-) create mode 100644 changelog.d/908-calibration-target-snapshots.added.md create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py create mode 100644 packages/microcosm-calibrate/tests/test_target_snapshots.py diff --git a/changelog.d/908-calibration-target-snapshots.added.md b/changelog.d/908-calibration-target-snapshots.added.md new file mode 100644 index 000000000..127a2b385 --- /dev/null +++ b/changelog.d/908-calibration-target-snapshots.added.md @@ -0,0 +1 @@ +Shared microcosm-calibrate can emit aggregate-only per-target estimate snapshots from real solver iterations at a configurable bounded or every-epoch cadence, with a versioned schema, stable ordered target identity digest, honest current/best-retained/selected iterate labelling, and an atomic latest-snapshot plus immutable bounded history store (opt-in; off by default). diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py index ccd544f5a..a467486c6 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py @@ -143,6 +143,17 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: Target, TargetSet, ) +from microcosm.calibrate.target_snapshots import ( # noqa: E402 - after compat gate + EVERY_EPOCH, + TARGET_SNAPSHOT_ITERATES, + TARGET_SNAPSHOT_SCHEMA, + TARGET_SNAPSHOT_SCHEMA_VERSION, + TargetSnapshotCadence, + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + validate_target_snapshot, +) from microcosm.calibrate.variable_labels import ( # noqa: E402 - after compat gate CALIBRATION_VARIABLE_LABELS_BY_COUNTRY, US_CALIBRATION_VARIABLE_LABELS, @@ -156,11 +167,15 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "CALIBRATION_PROVIDER_LABELS_BY_COUNTRY", "CALIBRATION_VARIABLE_LABELS_BY_COUNTRY", "CONSERVE_MASS", + "EVERY_EPOCH", "FREE_MASS", "TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE", "TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE", "TARGET_LOSS_ATTRIBUTION_WARNING_CODES", "TARGET_LOSS_BASIS_HASH_ALGORITHM", + "TARGET_SNAPSHOT_ITERATES", + "TARGET_SNAPSHOT_SCHEMA", + "TARGET_SNAPSHOT_SCHEMA_VERSION", "CalibrationProblem", "CalibrationHierarchy", "CalibrationHierarchySeed", @@ -176,6 +191,10 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "TargetDiagnostic", "TargetRegistry", "TargetSet", + "TargetSnapshotCadence", + "TargetSnapshotError", + "TargetSnapshotObserver", + "TargetSnapshotWriter", "TargetSpec", "US_CALIBRATION_PROVIDER_LABELS", "US_CALIBRATION_VARIABLE_LABELS", @@ -196,6 +215,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "score_targets", "select_exact_k", "specs_from_pe_surface", + "validate_target_snapshot", "write_calibration_diagnostics", "__version__", ] diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index 4bda636c1..67a258e8b 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -91,6 +91,13 @@ build_constraint_matrix, ) from microcosm.calibrate.target import TargetSet +from microcosm.calibrate.target_snapshots import ( + ITERATE_BEST_RETAINED, + ITERATE_CURRENT, + ITERATE_SELECTED, + BoundTargetSnapshots, + TargetSnapshotObserver, +) from microcosm.frame import Frame, MassChange, WeightKind, Weights __all__ = [ @@ -787,6 +794,7 @@ def _optimize( temperature: float, progress_callback: Callable[[dict[str, object]], None] | None = None, progress_context: Mapping[str, object] | None = None, + snapshots: BoundTargetSnapshots | None = None, return_gate_open_probabilities: bool = False, selection_receipt: dict[str, object] | None = None, ) -> tuple[np.ndarray, np.ndarray] | tuple[np.ndarray, np.ndarray, np.ndarray | None]: @@ -913,6 +921,30 @@ def _optimize( "loss": trajectory[epoch], } ) + if snapshots is not None and snapshots.emits(epoch + 1, epochs): + # The estimate tensor this epoch's loss was computed from, detached + # rather than recomputed: re-running the forward pass would redraw + # the hard-concrete gates' noise and move the RNG stream, changing + # the run. The current values are labelled best_retained only when + # they ARE the incumbent best iterate. + retained = retain_best and best_log_w is not None + snapshots.emit( + estimate.detach().numpy(), + epoch=epoch + 1, + epochs=epochs, + iterate=( + ITERATE_BEST_RETAINED + if retained and best_epoch == epoch + else ITERATE_CURRENT + ), + precision="float32", + loss=trajectory[epoch], + best_retained={ + "available": bool(retain_best), + "epoch": (best_epoch + 1) if retained else None, + "loss": best_loss if retained else None, + }, + ) total_loss.backward() optimizer.step() @@ -1020,6 +1052,7 @@ def _optimize_proximal( prune_atol: float, progress_callback: Callable[[dict[str, object]], None] | None = None, progress_context: Mapping[str, object] | None = None, + snapshots: BoundTargetSnapshots | None = None, ) -> tuple[np.ndarray, np.ndarray]: """Proximal gradient (ISTA-style) on weight ratios -- the L1 selection path. @@ -1075,6 +1108,17 @@ def _optimize_proximal( "loss": trajectory[epoch], } ) + if snapshots is not None and snapshots.emits(epoch + 1, epochs): + # The proximal path retains no best iterate (its selection is the + # closing state), so every in-loop snapshot here is `current`. + snapshots.emit( + estimate.detach().numpy(), + epoch=epoch + 1, + epochs=epochs, + iterate=ITERATE_CURRENT, + precision="float32", + loss=trajectory[epoch], + ) loss.backward() with torch.no_grad(): grad = ratio.grad @@ -1141,6 +1185,7 @@ def _search_l0_lambda_for_budget( prune_atol: float, initial_lambda: float | None, progress_callback: Callable[[dict[str, object]], None] | None = None, + snapshots: BoundTargetSnapshots | None = None, budget_iters: int = _DEFAULT_BUDGET_ITERS, return_gate_open_probabilities: bool = False, budget_basis: str = BUDGET_BASIS_NONZERO_COUNT, @@ -1270,6 +1315,17 @@ def evaluate( "budget_iters": budget_iters, "l0_lambda": lam, }, + # Each probe is a full reseeded optimization whose epoch numbering + # restarts, so its snapshots carry the probe's own identity. + "snapshots": ( + None + if snapshots is None + else snapshots.with_search( + budget_iteration=evaluation, + budget_iters=budget_iters, + l0_lambda=lam, + ) + ), } if gate_initialization is not None: optimize_kwargs["gate_initialization"] = gate_initialization @@ -1614,6 +1670,7 @@ def calibrate( target_loss_cap: float = _DEFAULT_TARGET_LOSS_CAP, warm_start_weights: np.ndarray | None = None, progress_callback: Callable[[dict[str, object]], None] | None = None, + target_snapshots: TargetSnapshotObserver | None = None, ) -> CalibrationResult: """Calibrate ``weight_entity``'s weights to ``targets`` over ``frame``. @@ -1731,6 +1788,14 @@ def calibrate( "calibration_epoch", "epoch": 1, "epochs": 256, "loss": ...}``. Build drivers use this to publish staging telemetry; calibration results are unchanged. + target_snapshots: Optional + :class:`~microcosm.calibrate.target_snapshots.TargetSnapshotObserver` + receiving aggregate-only per-target estimate snapshots at its own + cadence while the run is in flight (microcosm#908). Off by default; + enabling it does not change the weights, the trajectory, or the + RNG stream — in-loop snapshots read the estimate tensor the epoch's + loss was already computed from. The closing ``selected`` snapshot + describes the weights calibration actually returns. Returns: A :class:`CalibrationResult` with the calibrated frame, per-target @@ -1883,6 +1948,11 @@ def calibrate( ) ) problem = build_constraint_matrix(frame, targets, weight_entity) + snapshots = ( + None + if target_snapshots is None + else target_snapshots.bind(names=problem.names, targets=problem.target_vector) + ) initial = problem.initial_weights w0 = initial.values prune_atol = _PRUNE_REL_ATOL * float(np.mean(w0)) @@ -1958,6 +2028,7 @@ def calibrate( l1_lambda=l1_lambda, prune_atol=prune_atol, progress_callback=progress_callback, + snapshots=snapshots, ) n_nonzero = int((final_weights > prune_atol).sum()) if n_nonzero == 0: @@ -2004,6 +2075,7 @@ def calibrate( prune_atol=prune_atol, initial_lambda=(l0_lambda if l0_lambda > 0.0 else None), progress_callback=progress_callback, + snapshots=snapshots, budget_iters=budget_iters, budget_basis=budget_basis, feasible_draw_pi_hi=feasible_draw_pi_hi, @@ -2037,6 +2109,7 @@ def calibrate( init_mean=init_mean, temperature=temperature, progress_callback=progress_callback, + snapshots=snapshots, return_gate_open_probabilities=True, selection_receipt=iterate_selection_receipt, **( @@ -2075,13 +2148,28 @@ def calibrate( # capped weighted-MAPE loss the optimizer minimizes, evaluated after the closing # mass/cap projections — so final_loss describes # what calibrate returns, not the trajectory's pre-projection tail. + final_estimates = problem.estimates(final_weights) closing_loss = relative_error_loss( - problem.estimates(final_weights), + final_estimates, problem.target_vector, target_loss_weights=target_loss_weights_np, target_loss_scales=target_loss_scales_np, target_loss_cap=target_loss_cap, ) + if snapshots is not None: + # The selected snapshot describes the weights calibration RETURNS, + # read off the same float64 estimates the final diagnostics use — not + # the last in-loop iterate, which the optimizer may have discarded in + # favour of an earlier better one or changed by a closing projection. + snapshots.emit( + final_estimates, + epoch=epochs, + epochs=epochs, + iterate=ITERATE_SELECTED, + precision="float64", + loss=float(closing_loss), + selection=dict(iterate_selection_receipt) or None, + ) effective_target_loss_weights = ( np.ones(problem.target_vector.shape, dtype=np.float64) if target_loss_weights_np is None @@ -2315,6 +2403,16 @@ def callback(event: dict[str, object]) -> None: return callback +def _phase_snapshots( + target_snapshots: TargetSnapshotObserver | None, + phase: str, +) -> TargetSnapshotObserver | None: + """The ``_phase_callback`` analogue for the snapshot observer.""" + if target_snapshots is None: + return None + return target_snapshots.with_phase(phase) + + def _selected_person_mask( frame: Frame, weight_entity: str, @@ -2403,6 +2501,7 @@ def refit_l0_selection( target_loss_scales: np.ndarray | None = None, target_loss_cap: float = _DEFAULT_TARGET_LOSS_CAP, progress_callback: Callable[[dict[str, object]], None] | None = None, + target_snapshots: TargetSnapshotObserver | None = None, ) -> L0RefitResult: """Refit ordinary calibration on a frozen support from an L0 solve. @@ -2581,6 +2680,7 @@ def refit_l0_selection( target_loss_scales=target_loss_scales, target_loss_cap=target_loss_cap, progress_callback=progress_callback, + target_snapshots=target_snapshots, ) return L0RefitResult( selection=selection, @@ -2617,6 +2717,7 @@ def calibrate_l0_refit( target_loss_cap: float = _DEFAULT_TARGET_LOSS_CAP, warm_start_weights: np.ndarray | None = None, progress_callback: Callable[[dict[str, object]], None] | None = None, + target_snapshots: TargetSnapshotObserver | None = None, ) -> L0RefitResult: """Select a sparse support with L0 gates, then refit ordinary calibration. @@ -2690,6 +2791,7 @@ def calibrate_l0_refit( target_loss_cap=target_loss_cap, warm_start_weights=warm_start_weights, progress_callback=_phase_callback(progress_callback, "l0_selection"), + target_snapshots=_phase_snapshots(target_snapshots, "l0_selection"), ) return refit_l0_selection( frame, @@ -2712,4 +2814,5 @@ def calibrate_l0_refit( target_loss_scales=target_loss_scales, target_loss_cap=target_loss_cap, progress_callback=_phase_callback(progress_callback, "post_l0_refit"), + target_snapshots=_phase_snapshots(target_snapshots, "post_l0_refit"), ) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py new file mode 100644 index 000000000..807a034ee --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -0,0 +1,710 @@ +"""Per-target estimate snapshots from an in-progress calibration (microcosm#908). + +The solver already materializes the complete target-estimate vector once per +epoch (``estimate = A @ w`` inside :func:`~microcosm.calibrate.solve._optimize`) +and then drops it: only the scalar loss reaches ``progress_callback``, and the +complete per-target picture appears only after the run finishes, in +``calibration_diagnostics.json``. This module is the shared, aggregate-only +codec that lets a run publish that vector *while it is still running*, without +changing what the optimizer computes. + +Three promises the codec is built to keep. + +**Aggregate-only.** A snapshot carries one row per calibration *target* and +nothing else. There is no weight vector, no record index, and no household, +person, benefit-unit or source-record identifier; :func:`validate_target_snapshot` +refuses a payload that carries an unknown top-level key or a record-level key +name anywhere inside it, so smuggling record data through the ``context`` seam +fails loudly rather than silently shipping. + +**Honest iterate labelling.** ``current`` means "the weights the optimizer held +at this epoch". ``best_retained`` means "these values are the incumbent best +iterate's, and the optimizer is actually retaining a best" — it is never +attached to an iterate merely because it is the newest. ``selected`` means "the +weights the solver actually returned". A run whose optimizer retains no best at +all (gated, mass-conserved, or L2-penalized runs; see ``retain_best`` in +:mod:`~microcosm.calibrate.solve`) reports ``best_retained.available: false`` +rather than nominating one. + +**Stable identity.** ``targets_sha256`` digests the ordered ``(row name, target +value)`` list, so a consumer can tell that two snapshots describe the same +ordered target list, and reordering or revaluing the targets breaks the digest. + +Values sampled inside the optimization loop come off a ``float32`` tensor and +say so (``precision: "float32"``); the closing ``selected`` snapshot is taken +from the same ``float64`` path the final diagnostics use +(``precision: "float64"``). The solver's existing receipt already makes this +distinction (``selected_loss_float32``), and a snapshot must not quietly imply +more precision than it has. + +The store keeps a ``latest.json`` replaced atomically (temporary file in the +same directory, ``fsync``, ``os.replace``, parent ``fsync`` — the idiom +``microcosm.build.logbook_adoption.atomic_write_json`` already uses, reproduced +here because the calibrate shard may not depend on the build shard) so a +concurrent dashboard read never sees a partially written document, plus +write-once history chunks under ``history/`` with a bounded retention whose +drops are recorded rather than silent. + +This module is a leaf: it imports only the standard library and numpy, so +:mod:`microcosm.calibrate.solve` can import it without a cycle. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import uuid +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np + +__all__ = [ + "EVERY_EPOCH", + "ITERATE_BEST_RETAINED", + "ITERATE_CURRENT", + "ITERATE_SELECTED", + "LATEST_SNAPSHOT_FILENAME", + "RECORD_LEVEL_KEY_NAMES", + "SNAPSHOT_HISTORY_DIRNAME", + "SNAPSHOT_HISTORY_INDEX_FILENAME", + "TARGET_SNAPSHOT_ITERATES", + "TARGET_SNAPSHOT_SCHEMA", + "TARGET_SNAPSHOT_SCHEMA_VERSION", + "BoundTargetSnapshots", + "TargetSnapshotCadence", + "TargetSnapshotError", + "TargetSnapshotObserver", + "TargetSnapshotWriter", + "signed_relative_error", + "target_identity_digest", + "validate_target_snapshot", +] + +#: Schema identity. Consumers key their readers on the pair; bump the version +#: with any shape change, exactly as the diagnostics payload does. +TARGET_SNAPSHOT_SCHEMA = "microcosm.calibration.target-snapshot" +TARGET_SNAPSHOT_SCHEMA_VERSION = 1 + +#: Which weights a snapshot's values were read off. +ITERATE_CURRENT = "current" +ITERATE_BEST_RETAINED = "best_retained" +ITERATE_SELECTED = "selected" +TARGET_SNAPSHOT_ITERATES = ( + ITERATE_CURRENT, + ITERATE_BEST_RETAINED, + ITERATE_SELECTED, +) + +#: The cadence that emits on every epoch. +EVERY_EPOCH = 1 + +LATEST_SNAPSHOT_FILENAME = "latest.json" +SNAPSHOT_HISTORY_DIRNAME = "history" +SNAPSHOT_HISTORY_INDEX_FILENAME = "history_index.json" + +#: Record-level key names a snapshot may never carry, at any depth. The names +#: match the prohibited content keys the version-2 staging content policy +#: enforces on upload (microcosm#896), so a payload that passes here is not +#: rejected later at the publication boundary; the check is repeated in the +#: shared codec so a purely local snapshot store is bound by the same rule. +RECORD_LEVEL_KEY_NAMES = frozenset( + { + "benunit_id", + "household_id", + "person_id", + "raw_record", + "raw_records", + "row_id", + "source_value", + "source_values", + } +) + +_TOP_LEVEL_KEYS = frozenset( + { + "schema", + "schema_version", + "run_id", + "candidate_id", + "created_at", + "sequence", + "phase", + "epoch", + "epochs", + "search", + "iterate", + "precision", + "loss", + "best_retained", + "selection", + "targets_sha256", + "n_targets", + "context", + "targets", + } +) +_TARGET_ROW_KEYS = frozenset({"index", "name", "target", "estimate", "relative_error"}) + +_PRECISIONS = ("float32", "float64") + +#: Tolerance for the relative-error consistency check. The stored value is a +#: float64 recomputation of a float32-derived estimate on the in-loop path, so +#: the check is a consistency guard, not a bit-equality assertion. +_RELATIVE_ERROR_RTOL = 1e-9 + + +class TargetSnapshotError(ValueError): + """A snapshot payload violates the versioned contract.""" + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +def _finite_float(value: object, *, what: str) -> float: + try: + result = float(value) # type: ignore[arg-type] + except (TypeError, ValueError) as error: + raise TargetSnapshotError( + f"{what} must be a real number, got {value!r}." + ) from (error) + if not math.isfinite(result): + raise TargetSnapshotError(f"{what} must be finite, got {value!r}.") + return result + + +def _canonical_json_bytes(value: object) -> bytes: + """Canonical bytes for a digest: sorted keys, no spaces, no NaN.""" + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def signed_relative_error(estimate: float, target: float) -> float: + """The signed miss of ``estimate`` against ``target``. + + ``(estimate - target) / target``, except that a zero target has no + relative form, so the signed *absolute* miss ``estimate - target`` is + reported instead. This is exactly the convention + :class:`~microcosm.calibrate.solve.TargetDiagnostic` documents and + ``_target_diagnostics`` implements, so a mid-run snapshot and the run's + final diagnostics never disagree about what a target's error means. + """ + estimate = _finite_float(estimate, what="estimate") + target = _finite_float(target, what="target") + if target == 0.0: + return estimate - target + return (estimate - target) / target + + +def target_identity_digest( + names: Sequence[str], targets: Sequence[float] | np.ndarray +) -> str: + """A sha256 over the ordered ``(name, target value)`` list. + + Order-sensitive by construction: the digest covers the list, not a set, so + a consumer comparing two snapshots' digests is comparing the exact ordered + target identity the rows are aligned to. + """ + names = tuple(str(name) for name in names) + values = [ + _finite_float(value, what="target value") for value in np.asarray(targets) + ] + if len(names) != len(values): + raise TargetSnapshotError( + f"names and targets must align: {len(names)} names, {len(values)} targets." + ) + if not names: + raise TargetSnapshotError("a calibration snapshot needs at least one target.") + if len(set(names)) != len(names): + raise TargetSnapshotError("target row names must be unique.") + if any(not name for name in names): + raise TargetSnapshotError("target row names must be non-empty.") + return hashlib.sha256( + _canonical_json_bytes( + [[name, value] for name, value in zip(names, values, strict=True)] + ) + ).hexdigest() + + +def _reject_record_level_keys(value: object, *, where: str) -> None: + """Refuse a record-level key name anywhere in ``value``.""" + if isinstance(value, Mapping): + for key, item in value.items(): + if str(key).lower() in RECORD_LEVEL_KEY_NAMES: + raise TargetSnapshotError( + f"{where} carries the record-level key {key!r}; calibration " + "target snapshots are aggregate-only." + ) + _reject_record_level_keys(item, where=where) + elif isinstance(value, (list, tuple)): + for item in value: + _reject_record_level_keys(item, where=where) + + +@dataclass(frozen=True) +class TargetSnapshotCadence: + """How often a run emits a per-target snapshot. + + ``every`` epochs, plus the first and last epoch of the stretch so a + consumer always sees where a phase started and where it ended. + :data:`EVERY_EPOCH` is the explicit every-epoch mode. + """ + + every: int = 25 + include_first: bool = True + include_last: bool = True + + def __post_init__(self) -> None: + if isinstance(self.every, bool) or not isinstance(self.every, int): + raise ValueError(f"every must be an integer, got {self.every!r}.") + if self.every <= 0: + raise ValueError(f"every must be positive, got {self.every!r}.") + + def emits(self, epoch: int, epochs: int) -> bool: + """Whether the 1-based ``epoch`` of an ``epochs``-long stretch emits.""" + if epoch <= 0: + return False + if self.include_first and epoch == 1: + return True + if self.include_last and epoch == epochs: + return True + return epoch % self.every == 0 + + +@dataclass(frozen=True) +class TargetSnapshotObserver: + """Caller-side snapshot configuration: where snapshots go and how often. + + The observer knows nothing about the compiled target list — a caller + cannot, before calibration compiles it. :meth:`bind` attaches the ordered + target identity once the solver has it, producing the + :class:`BoundTargetSnapshots` the solver emits through. + + ``sink`` follows the existing ``progress_callback`` contract: it receives a + freshly built dictionary on every call (so mutating a delivered payload can + never reach the next one or the optimizer), and an exception it raises + propagates rather than being swallowed — the solver does not guard + ``progress_callback`` either. + """ + + sink: Callable[[dict[str, object]], None] + run_id: str + candidate_id: str | None = None + cadence: TargetSnapshotCadence = field(default_factory=TargetSnapshotCadence) + context: Mapping[str, object] = field(default_factory=dict) + clock: Callable[[], datetime] = _utc_now + phase: str | None = None + #: One monotone emission counter per observer, shared by every phase view + #: and every bind. A multi-phase run's epoch numbers restart, so `sequence` + #: is what recovers the true emission order across the whole run. + sequence_counter: list[int] = field( + default_factory=lambda: [0], repr=False, compare=False + ) + + def with_phase(self, phase: str) -> TargetSnapshotObserver: + """A view labelling its snapshots ``phase``, sharing this counter.""" + return replace(self, phase=str(phase)) + + def bind( + self, + *, + names: Sequence[str], + targets: Sequence[float] | np.ndarray, + ) -> BoundTargetSnapshots: + """Bind this observer to a compiled problem's ordered targets.""" + names = tuple(str(name) for name in names) + values = tuple( + _finite_float(value, what="target value") for value in np.asarray(targets) + ) + return BoundTargetSnapshots( + observer=self, + names=names, + targets=values, + digest=target_identity_digest(names, values), + counter=self.sequence_counter, + phase=self.phase, + ) + + +@dataclass(frozen=True) +class BoundTargetSnapshots: + """An observer bound to one compiled target list, with phase/search identity. + + ``phase`` and ``search`` are the identity a multi-phase run and a + sparse-selection search need: ``(phase, search iteration, epoch, iterate)`` + identifies a snapshot even though epoch numbers restart in every phase and + in every budget-search probe. ``sequence`` is a monotone counter shared by + all derived views of one bound observer, so the emission order is + recoverable even when the identity tuple is not ordered. + """ + + observer: TargetSnapshotObserver + names: tuple[str, ...] + targets: tuple[float, ...] + digest: str + counter: list[int] + phase: str | None = None + search: Mapping[str, object] | None = None + + def with_phase(self, phase: str) -> BoundTargetSnapshots: + """A view whose snapshots are labelled with ``phase``.""" + return replace(self, phase=str(phase)) + + def with_search(self, **search: object) -> BoundTargetSnapshots: + """A view whose snapshots carry a sparse-selection search identity.""" + return replace(self, search=dict(search)) + + @property + def cadence(self) -> TargetSnapshotCadence: + return self.observer.cadence + + def emits(self, epoch: int, epochs: int) -> bool: + return self.cadence.emits(epoch, epochs) + + def snapshot( + self, + estimates: np.ndarray, + *, + epoch: int, + epochs: int, + iterate: str, + precision: str = "float32", + loss: float | None = None, + best_retained: Mapping[str, object] | None = None, + selection: Mapping[str, object] | None = None, + ) -> dict[str, object]: + """Build (and validate) one snapshot payload. Does not call the sink.""" + if iterate not in TARGET_SNAPSHOT_ITERATES: + raise TargetSnapshotError( + f"iterate must be one of {TARGET_SNAPSHOT_ITERATES}, got {iterate!r}." + ) + if precision not in _PRECISIONS: + raise TargetSnapshotError( + f"precision must be one of {_PRECISIONS}, got {precision!r}." + ) + values = np.asarray(estimates, dtype=np.float64).reshape(-1) + if values.shape[0] != len(self.names): + raise TargetSnapshotError( + "estimates must align with the compiled targets: got " + f"{values.shape[0]}, expected {len(self.names)}." + ) + rows: list[dict[str, object]] = [] + for index, (name, target) in enumerate( + zip(self.names, self.targets, strict=True) + ): + estimate = _finite_float( + values[index], what=f"estimate for target {name!r}" + ) + rows.append( + { + "index": index, + "name": name, + "target": target, + "estimate": estimate, + "relative_error": signed_relative_error(estimate, target), + } + ) + self.counter[0] += 1 + payload: dict[str, object] = { + "schema": TARGET_SNAPSHOT_SCHEMA, + "schema_version": TARGET_SNAPSHOT_SCHEMA_VERSION, + "run_id": str(self.observer.run_id), + "candidate_id": ( + None + if self.observer.candidate_id is None + else str(self.observer.candidate_id) + ), + "created_at": self.observer.clock().isoformat(), + "sequence": self.counter[0], + "phase": self.phase, + "epoch": int(epoch), + "epochs": int(epochs), + "search": None if self.search is None else dict(self.search), + "iterate": iterate, + "precision": precision, + "loss": None if loss is None else _finite_float(loss, what="loss"), + "best_retained": ( + {"available": False, "epoch": None, "loss": None} + if best_retained is None + else dict(best_retained) + ), + "selection": None if selection is None else dict(selection), + "targets_sha256": self.digest, + "n_targets": len(self.names), + "context": dict(self.observer.context), + "targets": rows, + } + validate_target_snapshot(payload) + return payload + + def emit(self, estimates: np.ndarray, **kwargs: Any) -> dict[str, object] | None: + """Build a snapshot and hand it to the sink. Returns what was sent.""" + payload = self.snapshot(estimates, **kwargs) + self.observer.sink(payload) + return payload + + +def validate_target_snapshot(payload: Mapping[str, object]) -> None: + """Raise :class:`TargetSnapshotError` unless ``payload`` obeys the schema.""" + if not isinstance(payload, Mapping): + raise TargetSnapshotError(f"a snapshot must be a mapping, got {type(payload)}.") + keys = set(payload) + unknown = sorted(keys - _TOP_LEVEL_KEYS) + if unknown: + raise TargetSnapshotError( + f"snapshot carries unknown top-level keys: {unknown}." + ) + missing = sorted(_TOP_LEVEL_KEYS - keys) + if missing: + raise TargetSnapshotError(f"snapshot is missing required keys: {missing}.") + if payload["schema"] != TARGET_SNAPSHOT_SCHEMA: + raise TargetSnapshotError( + f"schema must be {TARGET_SNAPSHOT_SCHEMA!r}, got {payload['schema']!r}." + ) + if payload["schema_version"] != TARGET_SNAPSHOT_SCHEMA_VERSION: + raise TargetSnapshotError( + f"schema_version must be {TARGET_SNAPSHOT_SCHEMA_VERSION}, " + f"got {payload['schema_version']!r}." + ) + if payload["iterate"] not in TARGET_SNAPSHOT_ITERATES: + raise TargetSnapshotError( + f"iterate must be one of {TARGET_SNAPSHOT_ITERATES}, " + f"got {payload['iterate']!r}." + ) + if payload["precision"] not in _PRECISIONS: + raise TargetSnapshotError( + f"precision must be one of {_PRECISIONS}, got {payload['precision']!r}." + ) + if not isinstance(payload["run_id"], str) or not payload["run_id"]: + raise TargetSnapshotError("run_id must be a non-empty string.") + for key in ("sequence", "epoch", "epochs", "n_targets"): + value = payload[key] + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise TargetSnapshotError(f"{key} must be a non-negative integer.") + if payload["loss"] is not None: + _finite_float(payload["loss"], what="loss") + best = payload["best_retained"] + if not isinstance(best, Mapping) or "available" not in best: + raise TargetSnapshotError("best_retained must record whether one is available.") + if not isinstance(best["available"], bool): + raise TargetSnapshotError("best_retained.available must be a boolean.") + if not best["available"] and payload["iterate"] == ITERATE_BEST_RETAINED: + raise TargetSnapshotError( + "a snapshot cannot be labelled best_retained on a run that retains no " + "best iterate." + ) + + rows = payload["targets"] + if not isinstance(rows, Sequence) or isinstance(rows, (str, bytes)): + raise TargetSnapshotError("targets must be a sequence of rows.") + if len(rows) != payload["n_targets"]: + raise TargetSnapshotError( + f"n_targets is {payload['n_targets']} but {len(rows)} rows are present." + ) + names: list[str] = [] + values: list[float] = [] + for position, row in enumerate(rows): + if not isinstance(row, Mapping): + raise TargetSnapshotError(f"target row {position} is not a mapping.") + row_unknown = sorted(set(row) - _TARGET_ROW_KEYS) + if row_unknown: + raise TargetSnapshotError( + f"target row {position} carries unknown keys: {row_unknown}." + ) + row_missing = sorted(_TARGET_ROW_KEYS - set(row)) + if row_missing: + raise TargetSnapshotError( + f"target row {position} is missing keys: {row_missing}." + ) + if row["index"] != position: + raise TargetSnapshotError( + f"target rows must be in compiled order: row at position {position} " + f"declares index {row['index']!r}." + ) + name = row["name"] + if not isinstance(name, str) or not name: + raise TargetSnapshotError(f"target row {position} needs a non-empty name.") + target = _finite_float(row["target"], what=f"target for {name!r}") + estimate = _finite_float(row["estimate"], what=f"estimate for {name!r}") + stored = _finite_float( + row["relative_error"], what=f"relative_error for {name!r}" + ) + expected = signed_relative_error(estimate, target) + if not math.isclose( + stored, expected, rel_tol=_RELATIVE_ERROR_RTOL, abs_tol=0.0 + ): + raise TargetSnapshotError( + f"relative_error for {name!r} is {stored!r}; the signed relative " + f"error of {estimate!r} against {target!r} is {expected!r}." + ) + names.append(name) + values.append(target) + if len(set(names)) != len(names): + raise TargetSnapshotError("target row names must be unique within a snapshot.") + digest = target_identity_digest(names, values) + if payload["targets_sha256"] != digest: + raise TargetSnapshotError( + "targets_sha256 does not digest this snapshot's ordered target " + f"identity: declared {payload['targets_sha256']!r}, computed {digest!r}." + ) + for key in ("context", "search", "selection", "best_retained"): + _reject_record_level_keys(payload[key], where=f"snapshot {key}") + _reject_record_level_keys(payload["targets"], where="snapshot targets") + # A last strictness check: the payload must be strict JSON, so a consumer + # never receives NaN/Infinity tokens no parser outside Python accepts. + _canonical_json_bytes(payload) + + +def _fsync_parent_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path.parent, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _atomic_write_json(path: Path, payload: Mapping[str, object]) -> None: + """Replace ``path`` atomically: a reader sees the old or the new document.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + with temporary.open("w", encoding="utf-8") as stream: + stream.write(json.dumps(payload, indent=1, sort_keys=True, allow_nan=False)) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + _fsync_parent_directory(path) + finally: + temporary.unlink(missing_ok=True) + + +@dataclass +class TargetSnapshotWriter: + """A local snapshot store: atomic ``latest.json`` plus bounded history. + + ``latest.json`` is replaced atomically, so a dashboard polling it either + reads the previous complete snapshot or the new one — never a truncated + document. Each snapshot is also written once to + ``history/.json``; those chunks are immutable, so a reader that + has opened one is never racing a rewrite of it. + + ``history_limit`` bounds what the store *retains* (the oldest chunks are + dropped once the limit is exceeded); ``history_index.json`` records what is + retained and how many were dropped, so a bounded history is visibly + bounded rather than silently lossy. ``history_limit=None`` retains + everything. + """ + + directory: Path + history_limit: int | None = 256 + _dropped: int = field(default=0, init=False, repr=False) + + def __post_init__(self) -> None: + if self.history_limit is not None and ( + isinstance(self.history_limit, bool) + or not isinstance(self.history_limit, int) + or self.history_limit <= 0 + ): + raise ValueError( + f"history_limit must be a positive integer or None, " + f"got {self.history_limit!r}." + ) + self.directory = Path(self.directory) + self.directory.mkdir(parents=True, exist_ok=True) + self.history_directory.mkdir(parents=True, exist_ok=True) + + @property + def history_directory(self) -> Path: + return self.directory / SNAPSHOT_HISTORY_DIRNAME + + @property + def latest_path(self) -> Path: + return self.directory / LATEST_SNAPSHOT_FILENAME + + @property + def index_path(self) -> Path: + return self.directory / SNAPSHOT_HISTORY_INDEX_FILENAME + + def retained(self) -> tuple[str, ...]: + """The retained history chunk filenames, oldest first.""" + return tuple( + sorted(path.name for path in self.history_directory.glob("*.json")) + ) + + def _write_chunk(self, path: Path, payload: Mapping[str, object]) -> None: + """Write a history chunk exactly once; refuse to overwrite one.""" + try: + with path.open("x", encoding="utf-8") as stream: + stream.write( + json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) + ) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError as error: + raise TargetSnapshotError( + f"history chunks are immutable; {path.name} already exists." + ) from error + _fsync_parent_directory(path) + + def __call__(self, payload: Mapping[str, object]) -> None: + validate_target_snapshot(payload) + sequence = int(payload["sequence"]) + self._write_chunk(self.history_directory / f"{sequence:08d}.json", payload) + self._prune() + _atomic_write_json(self.latest_path, dict(payload)) + retained = self.retained() + _atomic_write_json( + self.index_path, + { + "schema": TARGET_SNAPSHOT_SCHEMA, + "schema_version": TARGET_SNAPSHOT_SCHEMA_VERSION, + "run_id": payload["run_id"], + "history_limit": self.history_limit, + "retained": list(retained), + "first_retained_sequence": ( + int(Path(retained[0]).stem) if retained else None + ), + "last_sequence": sequence, + "dropped": self._dropped, + }, + ) + + def _prune(self) -> int: + """Drop the oldest chunks past ``history_limit``; return how many.""" + if self.history_limit is None: + return 0 + chunks = self.retained() + excess = len(chunks) - self.history_limit + if excess <= 0: + return 0 + for name in chunks[:excess]: + (self.history_directory / name).unlink(missing_ok=True) + self._dropped += excess + return excess + + +def bounded_cadence(every: int) -> TargetSnapshotCadence: + """A bounded cadence emitting every ``every`` epochs (plus first and last).""" + return TargetSnapshotCadence(every=every) + + +def iter_history(directory: Path | str) -> Iterable[dict[str, object]]: + """Read a store's retained history chunks in sequence order.""" + history = Path(directory) / SNAPSHOT_HISTORY_DIRNAME + for path in sorted(history.glob("*.json")): + payload = json.loads(path.read_text(encoding="utf-8")) + validate_target_snapshot(payload) + yield payload diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py new file mode 100644 index 000000000..15db55a5a --- /dev/null +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -0,0 +1,474 @@ +"""Per-target calibration estimate snapshots (microcosm#908, slice 1). + +Covers the shared codec (schema identity, ordering, finiteness, signed +relative error with the zero-target convention, aggregate-only refusal), +the cadence, the atomic/immutable local store, and the solver seam: +emission from real Adam iterations, honest current/best_retained/selected +labelling, multi-phase and budget-search identity, and byte-identical +optimizer results with the observer off and on. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +import numpy as np +import pytest + +from microcosm.calibrate.solve import CONSERVE_MASS, calibrate, calibrate_l0_refit +from microcosm.calibrate.target import Target, TargetSet +from microcosm.calibrate.target_snapshots import ( + EVERY_EPOCH, + ITERATE_BEST_RETAINED, + ITERATE_CURRENT, + ITERATE_SELECTED, + LATEST_SNAPSHOT_FILENAME, + TARGET_SNAPSHOT_SCHEMA, + TARGET_SNAPSHOT_SCHEMA_VERSION, + TargetSnapshotCadence, + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + signed_relative_error, + target_identity_digest, + validate_target_snapshot, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + +#: One person per household, so household weights are the calibrated vector. +_SCHEMA = EntitySchema(group_entities=("household",)) + +#: A tiny invented frame: four households with an income and an adult count. +_INCOME = np.array([100.0, 200.0, 300.0, 400.0]) +_ADULTS = np.array([1.0, 2.0, 1.0, 2.0]) +_WEIGHTS = np.array([10.0, 10.0, 10.0, 10.0]) + + +def _frame() -> Frame: + import pandas as pd + + household_ids = np.arange(len(_WEIGHTS), dtype="int64") + person = pd.DataFrame( + { + "person_id": household_ids, + "person_household_id": household_ids, + } + ) + household = pd.DataFrame( + { + "household_id": household_ids, + "income": _INCOME, + "adults": _ADULTS, + } + ) + return Frame( + {"person": person, "household": household}, + _SCHEMA, + {"household": Weights(values=_WEIGHTS.copy(), kind=WeightKind.DESIGN)}, + ) + + +def _targets() -> TargetSet: + """Targets deliberately off the frame's aggregates, so the solver has to move.""" + return TargetSet( + [ + Target( + name="income", + period=2024, + entity="household", + measure="income", + value=12000.0, + ), + Target( + name="adults", + period=2024, + entity="household", + measure="adults", + value=50.0, + ), + ] + ) + + +def _collect(**observer_kwargs) -> tuple[list[dict], TargetSnapshotObserver]: + seen: list[dict] = [] + observer = TargetSnapshotObserver( + sink=seen.append, run_id="run-1", **observer_kwargs + ) + return seen, observer + + +# -------------------------------------------------------------------------- +# Codec: identity, ordering, finiteness, zero-target semantics +# -------------------------------------------------------------------------- + + +def test_target_identity_digest_is_order_sensitive_and_value_sensitive(): + names = ("a@2024", "b@2024") + base = target_identity_digest(names, (1.0, 2.0)) + assert base == target_identity_digest(names, (1.0, 2.0)) + assert base != target_identity_digest(("b@2024", "a@2024"), (2.0, 1.0)) + assert base != target_identity_digest(names, (1.0, 2.5)) + assert len(base) == 64 + + +def test_target_identity_digest_refuses_duplicate_or_non_finite(): + with pytest.raises(TargetSnapshotError): + target_identity_digest(("a", "a"), (1.0, 2.0)) + with pytest.raises(TargetSnapshotError): + target_identity_digest(("a", "b"), (1.0, float("nan"))) + with pytest.raises(TargetSnapshotError): + target_identity_digest(("a", "b"), (1.0,)) + + +def test_signed_relative_error_matches_the_solver_zero_target_convention(): + assert signed_relative_error(90.0, 100.0) == pytest.approx(-0.1) + assert signed_relative_error(110.0, 100.0) == pytest.approx(0.1) + # Zero target: the relative form is undefined, so the solver's diagnostics + # report the signed absolute miss. The snapshot must agree exactly. + assert signed_relative_error(3.0, 0.0) == pytest.approx(3.0) + assert signed_relative_error(-3.0, 0.0) == pytest.approx(-3.0) + + +def test_validate_rejects_reordered_rows_and_wrong_digest(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + payload = observer.bind( + names=("a@2024", "b@2024"), targets=np.array([10.0, 20.0]) + ).snapshot(np.array([9.0, 21.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT) + validate_target_snapshot(payload) + + reordered = json.loads(json.dumps(payload)) + reordered["targets"] = list(reversed(reordered["targets"])) + with pytest.raises(TargetSnapshotError, match="index"): + validate_target_snapshot(reordered) + + wrong_digest = json.loads(json.dumps(payload)) + wrong_digest["targets_sha256"] = "0" * 64 + with pytest.raises(TargetSnapshotError, match="targets_sha256"): + validate_target_snapshot(wrong_digest) + + +def test_validate_rejects_non_finite_and_inconsistent_relative_error(): + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + payload = bound.snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + broken = json.loads(json.dumps(payload)) + broken["targets"][0]["relative_error"] = 0.5 + with pytest.raises(TargetSnapshotError, match="relative_error"): + validate_target_snapshot(broken) + + with pytest.raises(TargetSnapshotError, match="finite"): + bound.snapshot( + np.array([float("inf")]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + + +def test_validate_refuses_record_level_identifiers_in_context(): + _, observer = _collect(context={"household_id": 4}) + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + with pytest.raises(TargetSnapshotError, match="aggregate-only"): + bound.snapshot(np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT) + + +def test_validate_refuses_unknown_top_level_keys(): + _, observer = _collect() + payload = observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + smuggled = json.loads(json.dumps(payload)) + smuggled["household_weights"] = [1.0, 2.0] + with pytest.raises(TargetSnapshotError, match="unknown"): + validate_target_snapshot(smuggled) + + +def test_schema_identity_is_pinned(): + _, observer = _collect() + payload = observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + assert payload["schema"] == TARGET_SNAPSHOT_SCHEMA + assert payload["schema_version"] == TARGET_SNAPSHOT_SCHEMA_VERSION + assert payload["iterate"] in { + ITERATE_CURRENT, + ITERATE_BEST_RETAINED, + ITERATE_SELECTED, + } + + +# -------------------------------------------------------------------------- +# Cadence +# -------------------------------------------------------------------------- + + +def test_cadence_every_epoch_emits_every_epoch(): + cadence = TargetSnapshotCadence(every=EVERY_EPOCH) + assert [cadence.emits(e, 5) for e in range(1, 6)] == [True] * 5 + + +def test_bounded_cadence_emits_first_multiples_and_last(): + cadence = TargetSnapshotCadence(every=4) + assert [cadence.emits(e, 10) for e in range(1, 11)] == [ + True, + False, + False, + True, + False, + False, + False, + True, + False, + True, + ] + + +def test_cadence_rejects_non_positive_every(): + with pytest.raises(ValueError): + TargetSnapshotCadence(every=0) + + +# -------------------------------------------------------------------------- +# Local store: atomic latest, immutable bounded history +# -------------------------------------------------------------------------- + + +def _payloads(n: int) -> list[dict]: + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + return [ + bound.snapshot( + np.array([9.0 + i]), epoch=i + 1, epochs=n, iterate=ITERATE_CURRENT + ) + for i in range(n) + ] + + +def test_writer_replaces_latest_atomically_and_never_leaves_partial_json(tmp_path): + writer = TargetSnapshotWriter(tmp_path) + for payload in _payloads(3): + writer(payload) + text = (tmp_path / LATEST_SNAPSHOT_FILENAME).read_text() + validate_target_snapshot(json.loads(text)) + assert json.loads((tmp_path / LATEST_SNAPSHOT_FILENAME).read_text())["epoch"] == 3 + # No temporary files survive a completed write. + assert sorted(p.name for p in tmp_path.iterdir()) == sorted( + [LATEST_SNAPSHOT_FILENAME, "history", "history_index.json"] + ) + + +def test_writer_history_chunks_are_write_once(tmp_path): + writer = TargetSnapshotWriter(tmp_path) + payloads = _payloads(2) + writer(payloads[0]) + chunk = next((tmp_path / "history").iterdir()) + with pytest.raises(TargetSnapshotError, match="immutable"): + writer._write_chunk(chunk, payloads[0]) + + +def test_writer_bounds_retained_history_and_records_what_it_dropped(tmp_path): + writer = TargetSnapshotWriter(tmp_path, history_limit=2) + for payload in _payloads(5): + writer(payload) + chunks = sorted(p.name for p in (tmp_path / "history").iterdir()) + assert len(chunks) == 2 + index = json.loads((tmp_path / "history_index.json").read_text()) + assert index["retained"] == chunks + assert index["dropped"] == 3 + assert index["last_sequence"] == 5 + # The retained chunks are the most recent ones and still validate. + for name in chunks: + validate_target_snapshot(json.loads((tmp_path / "history" / name).read_text())) + + +# -------------------------------------------------------------------------- +# Solver seam +# -------------------------------------------------------------------------- + + +def test_observer_off_is_the_default_and_changes_nothing(): + baseline = calibrate(_frame(), _targets(), epochs=20, seed=0) + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + observed = calibrate( + _frame(), _targets(), epochs=20, seed=0, target_snapshots=observer + ) + np.testing.assert_array_equal( + baseline.weights, + observed.weights, + ) + np.testing.assert_array_equal(baseline.loss_trajectory, observed.loss_trajectory) + assert seen, "an enabled observer must actually receive snapshots" + + +def test_every_epoch_emits_one_snapshot_per_epoch_plus_the_selected_one(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + calibrate(_frame(), _targets(), epochs=6, seed=0, target_snapshots=observer) + epochs = [s["epoch"] for s in seen if s["iterate"] != ITERATE_SELECTED] + assert epochs == [1, 2, 3, 4, 5, 6] + selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED] + assert len(selected) == 1 + for snapshot in seen: + validate_target_snapshot(snapshot) + + +def test_bounded_cadence_emits_fewer_snapshots_than_every_epoch(): + dense, dense_observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + calibrate(_frame(), _targets(), epochs=40, seed=0, target_snapshots=dense_observer) + sparse, sparse_observer = _collect(cadence=TargetSnapshotCadence(every=10)) + calibrate(_frame(), _targets(), epochs=40, seed=0, target_snapshots=sparse_observer) + assert len(sparse) < len(dense) + assert [s["epoch"] for s in sparse if s["iterate"] != ITERATE_SELECTED] == [ + 1, + 10, + 20, + 30, + 40, + ] + + +def test_snapshot_values_are_the_real_iterate_estimates(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), _targets(), epochs=30, seed=0, target_snapshots=observer + ) + selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED][-1] + by_name = {d.name: d for d in result.diagnostics} + for row in selected["targets"]: + diagnostic = by_name[row["name"]] + assert row["estimate"] == pytest.approx(diagnostic.final_estimate, rel=1e-9) + assert row["target"] == pytest.approx(diagnostic.target, rel=1e-9) + assert row["relative_error"] == pytest.approx( + diagnostic.relative_error, rel=1e-9 + ) + + +def test_current_iterates_are_never_labelled_best_when_they_are_not(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + calibrate(_frame(), _targets(), epochs=30, seed=0, target_snapshots=observer) + epoch_snapshots = [s for s in seen if s["iterate"] != ITERATE_SELECTED] + best_so_far = math.inf + for snapshot in epoch_snapshots: + loss = snapshot["loss"] + is_incumbent = loss < best_so_far + best_so_far = min(best_so_far, loss) + expected = ITERATE_BEST_RETAINED if is_incumbent else ITERATE_CURRENT + assert snapshot["iterate"] == expected, ( + f"epoch {snapshot['epoch']} labelled {snapshot['iterate']}" + ) + assert snapshot["best_retained"]["available"] is True + assert any(s["iterate"] == ITERATE_BEST_RETAINED for s in epoch_snapshots) + + +def test_a_run_that_retains_no_best_says_so_instead_of_claiming_one(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + calibrate( + _frame(), + _targets(), + epochs=10, + seed=0, + mass=CONSERVE_MASS, + target_snapshots=observer, + ) + epoch_snapshots = [s for s in seen if s["iterate"] != ITERATE_SELECTED] + assert epoch_snapshots + assert all(s["iterate"] == ITERATE_CURRENT for s in epoch_snapshots) + assert all(s["best_retained"]["available"] is False for s in epoch_snapshots) + + +def test_multi_phase_and_search_identity_is_unambiguous(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=5)) + calibrate_l0_refit( + _frame(), + _targets(), + epochs=10, + refit_epochs=10, + target_records=2, + budget_iters=2, + seed=0, + target_snapshots=observer, + ) + phases = {s["phase"] for s in seen} + assert {"l0_selection", "post_l0_refit"} <= phases + searched = [s for s in seen if s["search"] is not None] + assert searched, "budget-search probes must carry their search identity" + assert {s["search"]["budget_iteration"] for s in searched} >= {1, 2} + # (phase, search iteration, epoch, iterate) must identify a snapshot. + keys = [ + ( + s["phase"], + None if s["search"] is None else s["search"]["budget_iteration"], + s["epoch"], + s["iterate"], + s["sequence"], + ) + for s in seen + ] + assert len({k[:4] for k in keys}) == len(keys) + assert [k[4] for k in keys] == sorted(k[4] for k in keys) + for snapshot in seen: + validate_target_snapshot(snapshot) + + +def test_sink_exceptions_follow_the_progress_callback_contract(): + def boom(_payload): + raise RuntimeError("sink down") + + observer = TargetSnapshotObserver( + sink=boom, run_id="run-1", cadence=TargetSnapshotCadence(every=EVERY_EPOCH) + ) + with pytest.raises(RuntimeError, match="sink down"): + calibrate(_frame(), _targets(), epochs=4, seed=0, target_snapshots=observer) + + +def test_sink_mutation_cannot_reach_the_next_snapshot(): + seen: list[dict] = [] + + def mutating(payload): + payload["targets"].clear() + payload["run_id"] = "mutated" + seen.append(json.loads(json.dumps(payload))) + + observer = TargetSnapshotObserver( + sink=mutating, + run_id="run-1", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + ) + kept: list[dict] = [] + observer_two = TargetSnapshotObserver( + sink=kept.append, + run_id="run-1", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + ) + mutated_result = calibrate( + _frame(), _targets(), epochs=5, seed=0, target_snapshots=observer + ) + clean_result = calibrate( + _frame(), _targets(), epochs=5, seed=0, target_snapshots=observer_two + ) + np.testing.assert_array_equal( + mutated_result.weights, + clean_result.weights, + ) + assert all(len(s["targets"]) == 0 for s in seen) + assert all(len(s["targets"]) == 2 for s in kept) + + +def test_snapshots_carry_no_record_level_vectors(tmp_path: Path): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), _targets(), epochs=5, seed=0, target_snapshots=observer + ) + n_records = int(result.weights.shape[0]) + for snapshot in seen: + text = json.dumps(snapshot) + assert "household_id" not in text + assert "household_weight" not in text + for value in snapshot.values(): + assert not ( + isinstance(value, list) + and len(value) == n_records + and value + and isinstance(value[0], float) + ) From 46a64cfb80d5eb0cc72c889085873828e9d1c5fc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 09:58:27 -0400 Subject: [PATCH 116/255] Measure the snapshot path on synthetic matrices, and label it synthetic #908 asks for representative UK and US size and runtime measurements. This host is not permitted native microdata, engine runs or uploads, so this measures what it honestly can: invented target matrices and an invented seeded frame, at stated dimension points, inside one numeric thread, a 120 s CPU bound and a recorded peak RSS. What it establishes: emission cost and serialized size are linear in target count at ~177 bytes per target row; a 10,000-target snapshot is ~1.7 MB; and the returned weights are bitwise equal to the observer-off run at every cadence. What it deliberately does not establish: a production cadence default. The benchmarked solve costs 33 ms, so per-snapshot cost dominates it by construction and the 19x every-epoch ratio is an artifact of a cheap solve, not a forecast for a real calibration. The receipt says so, and the shipped default stays off. Co-Authored-By: Claude Opus 5 --- .../908-target-snapshot-bench-receipt.json | 84 ++++++ .../908-target-snapshot-bench-receipts.md | 63 ++++ experiments/908_target_snapshot_bench.py | 277 ++++++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 experiments/908-target-snapshot-bench-receipt.json create mode 100644 experiments/908-target-snapshot-bench-receipts.md create mode 100644 experiments/908_target_snapshot_bench.py diff --git a/experiments/908-target-snapshot-bench-receipt.json b/experiments/908-target-snapshot-bench-receipt.json new file mode 100644 index 000000000..d6dc03578 --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipt.json @@ -0,0 +1,84 @@ +{ + "label": "SYNTHETIC \u2014 invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": [ + 500, + 2000, + 5000, + 10000 + ], + "solver_records": 10000, + "solver_targets": 100, + "solver_epochs": 300 + }, + "bounds": { + "numeric_threads": 1, + "cpu_seconds_limit_applied": 120, + "memory_bytes_limit_applied": null, + "peak_rss_bytes": 289226752, + "native_inputs_read": 0 + }, + "codec_costs": [ + { + "n_targets": 500, + "build_and_validate_ms": 1.687, + "serialize_ms": 0.507, + "store_write_ms": 3.385, + "serialized_bytes": 88602, + "bytes_per_target": 177.2 + }, + { + "n_targets": 2000, + "build_and_validate_ms": 7.345, + "serialize_ms": 2.038, + "store_write_ms": 12.424, + "serialized_bytes": 353872, + "bytes_per_target": 176.9 + }, + { + "n_targets": 5000, + "build_and_validate_ms": 16.641, + "serialize_ms": 4.912, + "store_write_ms": 26.369, + "serialized_bytes": 885576, + "bytes_per_target": 177.1 + }, + { + "n_targets": 10000, + "build_and_validate_ms": 33.189, + "serialize_ms": 9.833, + "store_write_ms": 53.045, + "serialized_bytes": 1771910, + "bytes_per_target": 177.2 + } + ], + "solver_overhead": [ + { + "cadence": "observer_off", + "best_of_3_seconds": 0.0326, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": null, + "chunks_retained_on_disk": 0, + "retained_bytes_on_disk": 0 + }, + { + "cadence": "bounded_every_25", + "best_of_3_seconds": 0.0555, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 14, + "chunks_retained_on_disk": 14, + "retained_bytes_on_disk": 230903 + }, + { + "cadence": "every_epoch", + "best_of_3_seconds": 0.603, + "closing_loss": 2.7968459435242565e-07, + "weights_bitwise_equal_to_observer_off": true, + "snapshots_emitted": 301, + "chunks_retained_on_disk": 256, + "retained_bytes_on_disk": 4214808 + } + ] +} diff --git a/experiments/908-target-snapshot-bench-receipts.md b/experiments/908-target-snapshot-bench-receipts.md new file mode 100644 index 000000000..211f6691c --- /dev/null +++ b/experiments/908-target-snapshot-bench-receipts.md @@ -0,0 +1,63 @@ +# microcosm#908 — per-target snapshot cost, SYNTHETIC measurement + +**Everything below is synthetic.** The numbers come from invented target +matrices and an invented seeded frame produced by +[`908_target_snapshot_bench.py`](908_target_snapshot_bench.py). No native +microdata was read, no country engine ran, and nothing was uploaded. These are +**not** US or UK calibration runtimes and **not** upload times. The dimension +points were chosen to bracket an order of magnitude; they are not measured from +the real US or UK target registries. + +Run 2026-09-12 on macOS 26.6.2 / arm64, one numeric thread, CPU rlimit 120 s, +peak RSS 0.29 GiB (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the +script records the limit it actually applied and the peak RSS it reached). +Machine-readable copy: [`908-target-snapshot-bench-receipt.json`](908-target-snapshot-bench-receipt.json). + +## Codec and local-store cost per snapshot, by target count + +Build+validate is the full emission cost (`snapshot()` validates what it +builds). Store write is one immutable history chunk plus the atomic `latest.json` +replacement plus the index rewrite. + +| targets | build+validate (ms) | serialize (ms) | store write (ms) | serialized bytes | bytes/target | +|--------:|--------------------:|---------------:|-----------------:|-----------------:|-------------:| +| 500 | 1.69 | 0.51 | 3.39 | 88,602 | 177.2 | +| 2,000 | 7.35 | 2.04 | 12.42 | 353,872 | 176.9 | +| 5,000 | 16.64 | 4.91 | 26.37 | 885,576 | 177.1 | +| 10,000 | 33.19 | 9.83 | 53.05 | 1,771,910 | 177.2 | + +Both cost and size are linear in target count at ~177 bytes per target row +(indent=1, sorted keys). A single 10,000-target snapshot is ~1.7 MB. + +## Solver overhead at three cadences + +Invented frame: 10,000 households, 100 targets, 300 epochs, seed 0. One +discarded warm-up run, then best of three timed runs per cadence. + +| cadence | best-of-3 (s) | snapshots emitted | chunks retained | retained bytes | weights bitwise equal to observer-off | +|---|---:|---:|---:|---:|---| +| observer off | 0.0326 | — | 0 | 0 | yes (reference) | +| bounded, every 25 | 0.0555 | 14 | 14 | 230,903 | yes | +| every epoch | 0.6030 | 301 | 256 | 4,214,808 | yes | + +The closing loss is identical to all printed digits across all three +(`2.7968459435242565e-07`), and the returned weight vectors are bitwise equal. + +## What this does and does not license + +It does establish that emission is linear, that ~177 bytes/target is the wire +cost, and that turning the observer on does not change the optimizer's result. + +It does **not** license a production cadence default. This solve costs 33 ms +total, so per-snapshot cost dominates it by construction and the 19x +every-epoch ratio above is an artifact of a trivially cheap solve — a real +calibration's epoch is far more expensive, so the same absolute per-snapshot +cost would be a much smaller fraction of it. Choosing the production default +needs the native UK and US measurements #908 asks for, which this host is not +permitted to run. The shipped default is therefore *off*; a caller that opts in +picks its own cadence. + +The 5 MiB per-remote-file ceiling in the version-2 staging content policy is a +separate, unmeasured constraint here: at ~177 bytes/target a single snapshot +stays well inside it, but an accumulated history does not, which is why the +store chunks history rather than growing one document. diff --git a/experiments/908_target_snapshot_bench.py b/experiments/908_target_snapshot_bench.py new file mode 100644 index 000000000..c48d02785 --- /dev/null +++ b/experiments/908_target_snapshot_bench.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""SYNTHETIC benchmark for the #908 per-target snapshot path. + +Every number this script produces comes from **invented** target matrices and +an **invented** frame built from a seeded RNG. Nothing here reads native +microdata, runs a country engine, or touches a network. The dimension points +below were chosen to bracket an order of magnitude; they are NOT measured from +the real US or UK calibration target registries, and the timings are NOT US or +UK calibration times or upload times. Read them as "what does the codec and the +local store cost per target row and per epoch", nothing more. + +Run (from the repo root, inside the isolated lane interpreter):: + + python experiments/908_target_snapshot_bench.py + +Bounded on purpose: one numeric thread, a 2 GiB address-space cap and a 120 s +CPU-time cap are set before torch loads, so the script cannot quietly grow into +a machine-sized job. +""" + +from __future__ import annotations + +import json +import os +import resource +import sys +import tempfile +import time +from pathlib import Path + +_MEMORY_BYTES = 2 * 1024**3 + + +def _bound_resource(which: int, value: int) -> int | None: + """Apply a soft rlimit where the platform allows it; report what stuck.""" + soft, hard = resource.getrlimit(which) + if hard != resource.RLIM_INFINITY and value > hard: + return None + try: + resource.setrlimit(which, (value, hard)) + except (ValueError, OSError): + return None + return value + + +_CPU_LIMIT = _bound_resource(resource.RLIMIT_CPU, 120) +_MEMORY_LIMIT = _bound_resource(resource.RLIMIT_AS, _MEMORY_BYTES) +if _MEMORY_LIMIT is None: + # macOS refuses RLIMIT_AS here; RLIMIT_DATA is the portable fallback. + _MEMORY_LIMIT = _bound_resource(resource.RLIMIT_DATA, _MEMORY_BYTES) +for _variable in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): + os.environ.setdefault(_variable, "1") + +import numpy as np # noqa: E402 - after the resource and thread bounds +import pandas as pd # noqa: E402 - after the resource and thread bounds +import torch # noqa: E402 - after the resource and thread bounds + +from microcosm.calibrate import ( # noqa: E402 - after the resource bounds + EVERY_EPOCH, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) +from microcosm.frame import ( # noqa: E402 - after the resource bounds + EntitySchema, + Frame, + WeightKind, + Weights, +) + +torch.set_num_threads(1) + +#: Invented target-count dimension points for the codec measurement. +TARGET_COUNTS = (500, 2_000, 5_000, 10_000) + +#: Invented solver dimension point: households x targets x epochs. +SOLVER_RECORDS = 10_000 +SOLVER_TARGETS = 100 +SOLVER_EPOCHS = 300 + +#: Timed repeats per cadence; the table reports the best (least noisy) run. +REPEATS = 3 + +_SCHEMA = EntitySchema(group_entities=("household",)) + + +def _bound(n_targets: int): + """A bound observer over ``n_targets`` invented targets, sinking to nowhere.""" + rng = np.random.default_rng(0) + names = tuple(f"synthetic_target_{index:06d}@2024" for index in range(n_targets)) + targets = rng.lognormal(12.0, 1.0, n_targets) + observer = TargetSnapshotObserver(sink=lambda payload: None, run_id="synthetic") + return observer.bind(names=names, targets=targets), rng.lognormal( + 12.0, 1.0, n_targets + ) + + +def codec_costs() -> list[dict[str, object]]: + """Build/validate/serialize/write cost per snapshot, by target count.""" + rows: list[dict[str, object]] = [] + for n_targets in TARGET_COUNTS: + bound, estimates = _bound(n_targets) + repeats = max(3, 30_000 // n_targets) + start = time.perf_counter() + for _ in range(repeats): + # snapshot() validates the payload it builds, so this is the full + # build + validate cost a real emission pays. + payload = bound.snapshot( + estimates, epoch=1, epochs=1, iterate="current", loss=0.1 + ) + build_seconds = (time.perf_counter() - start) / repeats + + start = time.perf_counter() + for _ in range(repeats): + serialized = json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) + serialize_seconds = (time.perf_counter() - start) / repeats + + with tempfile.TemporaryDirectory() as directory: + writer = TargetSnapshotWriter(Path(directory), history_limit=8) + start = time.perf_counter() + for index in range(repeats): + copy = dict(payload) + copy["sequence"] = index + 1 + writer(copy) + write_seconds = (time.perf_counter() - start) / repeats + + rows.append( + { + "n_targets": n_targets, + "build_and_validate_ms": round(build_seconds * 1e3, 3), + "serialize_ms": round(serialize_seconds * 1e3, 3), + "store_write_ms": round(write_seconds * 1e3, 3), + "serialized_bytes": len(serialized.encode("utf-8")), + "bytes_per_target": round( + len(serialized.encode("utf-8")) / n_targets, 1 + ), + } + ) + return rows + + +def _synthetic_frame_and_targets() -> tuple[Frame, TargetSet]: + rng = np.random.default_rng(1) + household_ids = np.arange(SOLVER_RECORDS, dtype="int64") + columns = { + f"measure_{index:03d}": rng.lognormal(3.0, 0.5, SOLVER_RECORDS) + for index in range(SOLVER_TARGETS) + } + weights = np.full(SOLVER_RECORDS, 100.0) + household = pd.DataFrame({"household_id": household_ids, **columns}) + person = pd.DataFrame( + {"person_id": household_ids, "person_household_id": household_ids} + ) + frame = Frame( + {"person": person, "household": household}, + _SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + targets = TargetSet( + [ + Target( + name=name, + period=2024, + entity="household", + measure=name, + # An invented 8% miss, so the optimizer has somewhere to go. + value=float((values * weights).sum()) * 1.08, + ) + for name, values in columns.items() + ] + ) + return frame, targets + + +def solver_overhead() -> list[dict[str, object]]: + """Added wall-clock from snapshot emission, at three cadences. + + One discarded warm-up run first (the first calibrate of a process pays + matrix-build and torch warm-up that would otherwise be charged to whichever + cadence happened to run first), then the best of ``REPEATS`` timed runs per + cadence. Each run's returned weights are compared bitwise against the + observer-off run, so the table also evidences that emission changed nothing. + """ + frame, targets = _synthetic_frame_and_targets() + cadences = ( + ("observer_off", None), + ("bounded_every_25", TargetSnapshotCadence(every=25)), + ("every_epoch", TargetSnapshotCadence(every=EVERY_EPOCH)), + ) + calibrate(frame, targets, epochs=SOLVER_EPOCHS, seed=0) # warm-up, discarded + + rows: list[dict[str, object]] = [] + reference: np.ndarray | None = None + for label, cadence in cadences: + best_seconds = float("inf") + emitted = retained_chunks = retained_bytes = 0 + for _ in range(REPEATS): + with tempfile.TemporaryDirectory() as directory: + observer = ( + None + if cadence is None + else TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path(directory)), + run_id="synthetic", + cadence=cadence, + ) + ) + start = time.perf_counter() + result = calibrate( + frame, + targets, + epochs=SOLVER_EPOCHS, + seed=0, + target_snapshots=observer, + ) + best_seconds = min(best_seconds, time.perf_counter() - start) + history = Path(directory) / "history" + if observer is not None: + retained_chunks = len(list(history.glob("*.json"))) + retained_bytes = sum( + path.stat().st_size for path in history.glob("*.json") + ) + emitted = int( + json.loads( + (Path(directory) / "history_index.json").read_text( + encoding="utf-8" + ) + )["last_sequence"] + ) + if reference is None: + reference = result.weights.copy() + identical = True + else: + identical = bool(np.array_equal(reference, result.weights)) + rows.append( + { + "cadence": label, + f"best_of_{REPEATS}_seconds": round(best_seconds, 4), + "closing_loss": float(result.closing_loss), + "weights_bitwise_equal_to_observer_off": identical, + "snapshots_emitted": emitted or None, + "chunks_retained_on_disk": retained_chunks, + "retained_bytes_on_disk": retained_bytes, + } + ) + return rows + + +def main() -> int: + report = { + "label": "SYNTHETIC — invented matrices and frames, not US or UK calibration", + "dimension_points": { + "codec_target_counts": list(TARGET_COUNTS), + "solver_records": SOLVER_RECORDS, + "solver_targets": SOLVER_TARGETS, + "solver_epochs": SOLVER_EPOCHS, + }, + "bounds": { + "numeric_threads": torch.get_num_threads(), + "cpu_seconds_limit_applied": _CPU_LIMIT, + "memory_bytes_limit_applied": _MEMORY_LIMIT, + "peak_rss_bytes": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "native_inputs_read": 0, + }, + "codec_costs": codec_costs(), + "solver_overhead": solver_overhead(), + } + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From aaae9a90751cfc39fe589ad068ce97d87c631a58 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:00:54 -0400 Subject: [PATCH 117/255] Record the slice's state and the #908 scope it does not cover Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshots.md | 50 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md index affcd9cb1..68e4ec12e 100644 --- a/PROGRESS-908-target-snapshots.md +++ b/PROGRESS-908-target-snapshots.md @@ -5,7 +5,9 @@ Journals are history, not state (see CLAUDE.md): check git/GitHub for current tr ## State -In progress. Bounded first slice only — see "Out of scope for this slice". +Bounded first slice implemented, tested and committed on this branch. Not +pushed, no PR opened. #908 is NOT complete — see "Remaining #908 acceptance +scope" below. ## Scope of this slice @@ -31,8 +33,50 @@ In progress. Bounded first slice only — see "Out of scope for this slice". ## Done -- (nothing yet) +- `packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py`: + shared leaf codec. Schema name + version, aggregate-only validation + (unknown top-level keys and record-level key names refused at any depth), + compiled-order row check, finite-value check, signed relative error with + the solver's existing zero-target convention, ordered `(name, value)` + sha256 identity digest, `TargetSnapshotCadence` (bounded + `EVERY_EPOCH`), + and `TargetSnapshotWriter` (atomic `latest.json`, write-once history + chunks, bounded retention with recorded drops). +- Solver integration in `solve.py`: emission from the Adam loop and the + proximal loop off the estimate tensor the epoch's loss was already + computed from (no second forward pass, so the hard-concrete gates' RNG + stream is untouched); budget-search probes carry their own search + identity; `l0_selection` / `post_l0_refit` carry their phase; one monotone + sequence counter spans every phase; the closing `selected` snapshot comes + off the float64 estimates the final diagnostics use. +- Public opt-in parameter `target_snapshots=` on `calibrate`, + `refit_l0_selection` and `calibrate_l0_refit`; exports in the shard + `__init__`. Off by default. +- 24 new tests in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (flat path; `tools/ci_test_groups.py --verify` = ok, lands in fast `rest` + and engine `us-am`, never `[defaulted]`). +- Synthetic benchmark + receipts under `experiments/`, explicitly labelled + synthetic. +- changelog.d fragment `908-calibration-target-snapshots.added.md`. + +## Remaining #908 acceptance scope (NOT done here) + +- Version-2 staging persistence/upload of the snapshot and its history. + PR #896 is open, draft, CONFLICTING and absent from `main`; its + `calibration_progress` filters on `kind == "calibration_epoch"` and its + per-event schema is `additionalProperties: false`, so it needs a new + schema name, not a widened one. Nothing here claims remote publication. +- US host wiring (`tools/build_us_fiscal_refresh_release.py` still builds v1 + `StagingTelemetry`) and UK host cadence flags. +- Exact-k ladder phases in `microcosm.build.us_runtime.exact_k_ladder` and + the UK size-search phases in `uk_runtime.dataset_size` are not wired. +- A test asserting intermediate snapshots never substitute for the canonical + `calibration_diagnostics.json`. +- Real UK/US target/epoch counts, wall-clock, and upload time, and therefore + the production default cadence. Forbidden on this host; the default stays + off. +- Calibration Diagnostics UI (different repository, by the issue's own text). ## Next -- Read scouts, write failing tests, implement, bench, commit. +- Independent review of this slice, then the staging/host wiring as a + separate slice once #896 lands. From 35fc76dd1d13e8685ee131bd9f9dfdd0ca5336b3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:05:01 -0400 Subject: [PATCH 118/255] Re-pin only the calibrate parity case's source identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graph shard's `calibrate.adam@1` kernel hashes its defining module's exact source bytes plus its dependency versions, so editing microcosm.calibrate.solve legitimately moves `implementation_hash` and, through it, the node key. Two tests read those pins (test_acceptance_h_parity::test_h1_kernel_parity and test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes); both were green on origin/main and red here purely on identity. This updates that one case. `fit.qrf` and `simulate` are untouched — no blanket re-pin. The regeneration asserted first that graph.json, inputs.csv and direct.csv come back byte-identical to what is checked in, so this is an identity update and demonstrably not a numeric change: the calibrate kernel still produces the same bytes. The pinned dependency versions (numpy 2.4.6, pandas 3.0.3, scipy 1.17.1, torch 2.12.0) are unchanged too, so the move is attributable to the source edit alone. microcosm-graph suite: 456 passed. Co-Authored-By: Claude Opus 5 --- .../tests/fixtures/parity/kernels/calibrate/pins.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index a9610f6b8..437d54659 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"5f5f74d9bce03175a37006d5c3e5e4dcefdec1118982351fd2ca66e3793dc48b","kernel":"calibrate.adam@1","node":"calibrate","node_key":"184ccd0a25e1d00cc4393b2880ff13a601ac04207bc37b0160b7ae60596cb3d7","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"184ccd0a25e1d00cc4393b2880ff13a601ac04207bc37b0160b7ae60596cb3d7"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"bd0206f77cc3e1b54179e43f888e894f2b2b561a57f451fd810fb71be4ae1951","kernel":"calibrate.adam@1","node":"calibrate","node_key":"f29e0537c41345d9cfdd80f4789083b09bcbda7b3205411a2b95459231e37091","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"f29e0537c41345d9cfdd80f4789083b09bcbda7b3205411a2b95459231e37091"}},"seed":0} From a18fd60751397831e6211e7cdab8a2a678e844f0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:19:52 -0400 Subject: [PATCH 119/255] Retain checked PUF55 outputs for downstream graph stages --- changelog.d/puf55-checked-output.added.md | 1 + .../us-puf55-checked-output-20260912.md | 83 +++++++ .../build/us_runtime/graph_survey_puf55.py | 198 ++++++++++++++- .../tests/test_us_graph_survey_puf55.py | 227 ++++++++++++++++++ 4 files changed, 508 insertions(+), 1 deletion(-) create mode 100644 changelog.d/puf55-checked-output.added.md create mode 100644 experiments/us-puf55-checked-output-20260912.md diff --git a/changelog.d/puf55-checked-output.added.md b/changelog.d/puf55-checked-output.added.md new file mode 100644 index 000000000..c5fe0d86c --- /dev/null +++ b/changelog.d/puf55-checked-output.added.md @@ -0,0 +1 @@ +Retain PUF55 output handles for source, producer, store, and complete-population rechecks before downstream consumption, revoking a handle after a failed check without rerunning imputation. diff --git a/experiments/us-puf55-checked-output-20260912.md b/experiments/us-puf55-checked-output-20260912.md new file mode 100644 index 000000000..7ce57a7f2 --- /dev/null +++ b/experiments/us-puf55-checked-output-20260912.md @@ -0,0 +1,83 @@ +# Retained PUF55 output verification + +On September 12, 2026, the PUF55 host gained an in-process checked run handle +for downstream graph stages. The actual `run_survey_puf55` execution retains +the handle only after its existing final source, artifact, numerical, and +complete-population checks. Constructing or copying `SurveyPuf55Run` does not +issue a handle. + +`check_survey_puf55_run(run)` and `run.checked_view()` return a descriptive +`CheckedSurveyPuf55Run(payload, digest, population)`. Downstream owners must +retain the original run, check it immediately before consumption, and recheck +it after their last relevant I/O before returning or exporting a successor. +A failed public check revokes that handle; restoring fields cannot reissue it. +The payload and view cannot authorize a reconstructed or copied run. + +## Retained checks + +The checker reuses the attachment boundary's real financial/source owners, +source and producer keys, donor resource declaration, source codecs, kernel +registry, recipient qualification, and receiving support. It reads declared +artifacts through the existing producer/type/store verifier, borrows the +boundary again, then checks the output without further external reads. + +The output checks preserve original object bindings, the compiled graph, +manifest JSON, exact attached population and ledger roster, heterogeneous +manifest contents, and the complete final and independently reconstructed +populations. Existing physical seals include values and missing backing +storage, axes, schema, memberships, weights and kinds, design anchors, strata, +owners, metadata, mass log, and ledger. The retained state holds final output +and expected populations, rather than all 245 observed node snapshots. + +Rechecking does not fit, execute a graph, decode model pickles, or reconstruct +the canonical donor. It verifies the current ancestry and identities of the +artifact bytes that actual execution already checked numerically. It still +performs source, implementation, and store checks; consumers should use it at +stage boundaries rather than inside calibration iterations. + +## Verification and independent review + +- Three public-constructor/API controls first failed against the absent API, + then passed against the implementation. The original red evidence remains. +- The first complete invented-source run passed 23 controls. Independent + review then reproduced a gap: adding an extra attached Frame or ledger key + did not change portable manifest JSON or the old expected-version seals. + The checker now requires the exact attachment roster, with separate + population and ledger regression cases. +- The corrected source passed all 25 controls through an actual invented + financial19 to PUF245 cold execution and required replay. It used 901.07 + CPU seconds, 910.28 wall seconds, and 740,917,248 bytes peak RSS. All 985 + source/owned-file and 15 resource hashes remained unchanged; the guard + recorded no unexpected refusals or child processes and one numeric thread. +- The 12 new test functions were then moved beside the existing host tests + without changing their decorated ASTs, any existing function/class AST, or + production source. Independent review approved the move. Collection found + 45 cases, including the 25 new cases, sharing one `composed` fixture + definition. The three cheap checks passed again. The combined 45-case suite + was collected, not rerun as one full suite. +- Ruff and the CI test inventory verifier pass. The source repair and test + move each received independent review with no remaining actionable finding. + +The first post-move guard recorded the expected 45-case collection and three +passing tests but returned nonzero because two pytest sessions doubled an +already-denied optional locale metadata probe from 12 to 24. Its evidence +remains. A new owned run adjusted only that denied-probe reporting ceiling; +it closed green without allowing the read or changing source or tests. + +Frozen production SHA256: +`5ba34ce40e19cb2bf22f9c9976576f9dc19e92b19a5a5cc1763569df7a40b796`. +Final colocated test module SHA256: +`bf8b53d229a61565337e2ee9766900158b2bde4edc3042c60c52fe09ecdc531f`. +Corrected 25-case receipt SHA256: +`e47f4408387fcfab6564f66ad9479c9c1cf32615862b2268ee854ba7781a4553`. +Post-move collection/cheap receipt SHA256: +`016a5defba66ce8d36372e890a000295d82f4825d1e516d477185eaccfc70633`. + +## Scope + +This is invented-source software acceptance. It grants no native source or +population admission, calibration acceptance, release eligibility, or +publication authorization. Graph node and artifact contracts are unchanged. +The attachment kernel hashes the host module, so this additive change moves +PUF producer identities on its own source revision. The separately frozen +d35 native pilot packet and its evidence remain unchanged. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py index 5bfd0debb..4de2e1c52 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py @@ -8,6 +8,7 @@ from __future__ import annotations +import weakref from dataclasses import dataclass, replace from pathlib import Path @@ -23,17 +24,20 @@ run_graph, ) from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json from . import graph_puf55_route_attachment as attach require = attach.require financial, recipient, canonical = attach.financial, attach.recipient, attach.canonical physical, codec, population_ops = attach.physical, attach.codec, attach.population_ops +RUN_PROTOCOL = "microcosm.us.survey-puf55-checked-run.v1" +_ISSUED_RUNS = {} @dataclass(frozen=True) class SurveyPuf55Run: - """Checked result values, not financial/weight/release admission evidence.""" + """Actual result handle; constructor/copies do not issue run authority.""" financial_run: financial.AtomicSurveyFinancialRunValues population: population_ops.Population @@ -44,6 +48,176 @@ class SurveyPuf55Run: sources: tuple[tuple[str, Path], ...] receipt: bytes + def checked_view(self): + """Recheck this retained handle without fitting or graph execution.""" + return check_survey_puf55_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyPuf55Run: + """Descriptive values; only the original run retains in-process authority. + + A downstream owner must check that run immediately before consumption and + after its last relevant I/O, before returning or exporting a successor. + Reconstructing this view from JSON or copying it cannot authorize a run. + Neither this view nor the run grants calibration or release admission. + """ + + payload: bytes + digest: str + population: population_ops.Population + + +@dataclass(frozen=True) +class _RunState: + boundary: attach.Boundary + population: population_ops.Population + population_stamp: str + expected: population_ops.Population + expected_stamp: str + manifest: object + manifest_bytes: str + manifest_populations: tuple + receipt: bytes + artifact_hashes: tuple + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is SurveyPuf55Run and entry is not None and entry[0]() is run, + "UNISSUED_PUF55_RUN", + ) + return entry + + +def _forget_run(run, entry): + if _ISSUED_RUNS.get(id(run)) is entry: + _ISSUED_RUNS.pop(id(run)) + + +def _run_document(state): + """Describe checked ancestry without granting authority to portable bytes.""" + boundary = state.boundary + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(boundary.declaration.encode()), + "manifest_key": state.manifest.key, + "financial_run_sha256": codec.sha(boundary.entry[1]), + "receipt_sha256": codec.sha(state.receipt), + "node_count": len(boundary.compiled.order), + "node_keys": dict(boundary.keys), + "source_keys": dict(boundary.source_keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "population_version": state.population.version, + "population_physical_sha256": state.population_stamp, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_run(run, entry): + """Check the retained output after all external reads have completed.""" + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + state, boundary = entry[2], entry[2].boundary + require( + run.financial_run is boundary.run + and run.population is state.population + and run.manifest is state.manifest + and run.compiled is boundary.compiled + and run.store is boundary.store + and run.kernels is boundary.kernels + and run.sources is boundary.paths + and run.receipt == state.receipt, + "PUF55_RUN_BINDINGS_CHANGED", + ) + require( + physical._population_stamp(run.population) == state.population_stamp + and physical._population_stamp(state.expected) == state.expected_stamp, + "PUF55_RUN_POPULATION_CHANGED", + ) + require( + set(run.manifest.populations) + == set(run.manifest.mass_ledgers) + == set(run.compiled.versions.values()), + "PUF55_RUN_MANIFEST_ROSTER_CHANGED", + ) + require( + run.manifest.to_json() == state.manifest_bytes + and _heterogeneous_manifest_seals(run.manifest, run.compiled) + == state.manifest_populations, + "PUF55_RUN_MANIFEST_CHANGED", + ) + boundary.pure() + require( + graph_to_json(run.compiled.graph) == boundary.declaration + and run.compiled == compile_graph(run.compiled.graph), + "PUF55_RUN_DECLARATION_CHANGED", + ) + physical.replay.same_replayed_population(state.expected, run.population) + require(_run_document(state) == entry[1], "PUF55_RUN_DOCUMENT_CHANGED") + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + + +def check_survey_puf55_run(run): + """Requalify actual source/store ancestry, then seal the complete output. + + Does not fit, execute a graph, decode model pickles, or reconstruct the PUF + donor. The actual execution already verified those exact artifact bytes. + A failed check revokes this handle; restoring its fields cannot reissue it. + Downstream owners must recheck before consumption and after their last + relevant I/O before returning/exporting a successor. A successful check is + a point-in-time claim, not a lease permitting future unchecked mutation. + """ + entry = _run_entry(run) + try: + _pure_run(run, entry) + state, boundary = entry[2], entry[2].boundary + boundary.borrow() + loaded = financial._artifacts( + run.manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == state.artifact_hashes, + "PUF55_RUN_ARTIFACT_CHANGED", + ) + result = CheckedSurveyPuf55Run(entry[1], codec.sha(entry[1]), run.population) + _pure_run(run, entry) + return result + except BaseException: + _forget_run(run, entry) + raise + + +def _issue_run(output, state): + """Retain only a fully verified execution, with no new baseline reads.""" + identifier = id(output) + require(identifier not in _ISSUED_RUNS, "PUF55_RUN_REISSUANCE") + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier) + + entry = (weakref.ref(output, forget), _run_document(state), state) + _ISSUED_RUNS[identifier] = entry + try: + _pure_run(output, entry) + except BaseException: + _forget_run(output, entry) + raise + return entry + def _registry(existing, donor): require(type(existing) is codecs.SourceCodecRegistry, "UPSTREAM_CODEC_REGISTRY") @@ -446,4 +620,26 @@ def observe(node_id, population): "FINAL_OUTPUT_CHANGED", ) physical.replay.same_replayed_population(expected, population) + state = _RunState( + boundary, + population, + stamps[attach.ATTACH_NODE], + expected, + expected_stamp, + manifest, + manifest_bytes, + full_seals, + receipt, + artifact_hashes, + ) + # Baselines were captured before the preceding last-I/O fence. The entry + # retains the final output, not all 245 executor-observed snapshots. + try: + entry = _issue_run(output, state) + _pure_run(output, entry) + except BaseException: + candidate = _ISSUED_RUNS.get(id(output)) + if candidate is not None and candidate[0]() is output: + _forget_run(output, candidate) + raise return output diff --git a/packages/microcosm-build/tests/test_us_graph_survey_puf55.py b/packages/microcosm-build/tests/test_us_graph_survey_puf55.py index 6b481a5b2..e085ea772 100644 --- a/packages/microcosm-build/tests/test_us_graph_survey_puf55.py +++ b/packages/microcosm-build/tests/test_us_graph_survey_puf55.py @@ -7,6 +7,7 @@ import sys from _thread import get_ident from contextlib import contextmanager +from copy import copy from dataclasses import replace from types import SimpleNamespace @@ -19,6 +20,7 @@ from microcosm.build.us_runtime import graph_survey_puf55 as graph from microcosm.graph import StructuralDelta from microcosm.graph.keys import opaque_artifact_key +from microcosm.graph.store import StoreCorrupt @pytest.fixture(scope="module") @@ -853,3 +855,228 @@ def trace(frame, event, arg): assert b.computed is None b.pure() composed.boundary.pure() + + +@pytest.mark.parametrize( + "constructor", [lambda: object(), lambda: graph.SurveyPuf55Run(*([None] * 8))] +) +def test_unissued_public_values_cannot_be_checked(constructor): + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + graph.check_survey_puf55_run(constructor()) + + +def test_public_values_method_does_not_issue_authority(): + values = graph.SurveyPuf55Run(*([None] * 8)) + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + values.checked_view() + + +def test_real_cold_and_required_handles_recheck_without_execution(composed): # noqa: F811 + entered = [] + + def trace(frame, event, arg): + if event == "call": + entered.append(frame.f_code.co_name) + + with _observe_exact_code_events( + trace, + starts=( + graph.run_graph, + graph.attach.Boundary.finalize, + graph.attach.Boundary.canonical_donor, + ), + ): + cold = composed.cold.checked_view() + warm = graph.check_survey_puf55_run(composed.warm) + assert entered == [] + assert cold.population is composed.cold.population + assert warm.population is composed.warm.population + assert cold.payload == warm.payload + assert cold.digest == warm.digest == graph.codec.sha(cold.payload) + document = graph.codec.decode_json(cold.payload) + assert document["release_eligible"] is False + assert document["population_admission_issued"] is False + assert document["node_count"] == 245 + assert len(document["artifact_payload_sha256"]) > 220 + + +@pytest.mark.parametrize("clone", [copy, replace]) +def test_copied_real_run_is_unissued(composed, clone): # noqa: F811 + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + graph.check_survey_puf55_run(clone(composed.cold)) + graph._pure_run(composed.cold, graph._run_entry(composed.cold)) + + +@pytest.mark.parametrize( + "field", + [ + "financial_run", + "population", + "manifest", + "compiled", + "store", + "kernels", + "sources", + "receipt", + ], +) +def test_retained_public_field_replacement_is_refused(composed, field): # noqa: F811 + run = composed.cold + original = getattr(run, field) + entry = graph._run_entry(run) + try: + object.__setattr__(run, field, None) + with pytest.raises(ValueError, match="PUF55_RUN_BINDINGS_CHANGED"): + graph._pure_run(run, entry) + finally: + object.__setattr__(run, field, original) + graph._pure_run(run, entry) + + +@pytest.mark.parametrize("surface", ["amount", "design_anchor", "owner", "mass_ledger"]) +def test_retained_complete_population_mutation_is_refused(composed, surface): # noqa: F811 + run, entry = composed.cold, graph._run_entry(composed.cold) + population = run.population + if surface == "amount": + table = population.frame.person + name = "age" + before = table[name].copy(deep=True) + table.loc[table.index[0], name] += 1 + + def restore(): + table[name] = before + + elif surface == "design_anchor": + before = population.design_weights + changed = {name: vector.copy() for name, vector in before.items()} + changed["household"][0] += 1 + object.__setattr__(population, "design_weights", changed) + + def restore(): + object.__setattr__(population, "design_weights", before) + + else: + name = "owners" if surface == "owner" else "mass_ledger" + before = getattr(population, name) + assert before + changed = dict(before) if surface == "owner" else () + if surface == "owner": + changed[next(iter(changed))] = "invented_wrong_writer" + object.__setattr__(population, name, changed) + + def restore(): + object.__setattr__(population, name, before) + + try: + with pytest.raises(ValueError, match="PUF55_RUN_POPULATION_CHANGED"): + graph._pure_run(run, entry) + finally: + restore() + graph._pure_run(run, entry) + + +def test_compiled_execution_order_mutation_is_refused(composed): # noqa: F811 + run, entry = composed.cold, graph._run_entry(composed.cold) + original = run.compiled.order + try: + object.__setattr__(run.compiled, "order", original[::-1]) + with pytest.raises(ValueError, match="PUF55_RUN_DECLARATION_CHANGED"): + graph._pure_run(run, entry) + finally: + object.__setattr__(run.compiled, "order", original) + graph._pure_run(run, entry) + + +def test_attached_manifest_ledger_mutation_is_refused(composed): # noqa: F811 + run, entry = composed.cold, graph._run_entry(composed.cold) + original = run.manifest.mass_ledgers + changed = dict(original) + assert changed[run.population.version] + changed[run.population.version] = () + try: + object.__setattr__(run.manifest, "mass_ledgers", changed) + with pytest.raises(ValueError, match="PUF55_RUN_MANIFEST_CHANGED"): + graph._pure_run(run, entry) + finally: + object.__setattr__(run.manifest, "mass_ledgers", original) + graph._pure_run(run, entry) + + +@pytest.mark.parametrize("field", ["populations", "mass_ledgers"]) +def test_extra_attached_manifest_version_is_refused(composed, field): # noqa: F811 + run, entry = composed.cold, graph._run_entry(composed.cold) + original = getattr(run.manifest, field) + changed = {**original, "invented_extra_version": next(iter(original.values()))} + original_json = run.manifest.to_json() + try: + object.__setattr__(run.manifest, field, changed) + assert run.manifest.to_json() == original_json + with pytest.raises(ValueError, match="PUF55_RUN_MANIFEST_ROSTER_CHANGED"): + graph._pure_run(run, entry) + finally: + object.__setattr__(run.manifest, field, original) + graph._pure_run(run, entry) + + +def test_reused_artifact_verifier_refuses_actual_store_mutation(composed): # noqa: F811 + run, boundary = composed.cold, composed.boundary + key = run.manifest.node(graph.attach.ATTACH_NODE).opaque_artifacts["finalization"] + path = run.store.object_path(key) / "payload.bin" + original = path.read_bytes() + try: + path.write_bytes(original + b"invented corruption") + with pytest.raises(StoreCorrupt): + graph.financial._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + finally: + path.write_bytes(original) + graph._pure_run(run, graph._run_entry(run)) + + +def test_changed_original_donor_revokes_actual_run(composed): # noqa: F811 + run = composed.cold + path = next(iter(composed.paths.values())) + original = path.read_bytes() + try: + path.write_bytes(original + b"\n") + with pytest.raises(ValueError): + graph.check_survey_puf55_run(run) + finally: + path.write_bytes(original) + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + graph.check_survey_puf55_run(run) + + +def test_late_io_output_mutation_revokes_actual_run(composed): # noqa: F811 + run = composed.warm + table = run.population.frame.person + before = table.age.copy(deep=True) + entry = graph._run_entry(run) + mutated = False + + def trace(frame, event, arg): + nonlocal mutated + if ( + event == "return" + and arg is not None + and frame.f_locals.get("manifest") is run.manifest + ): + table.loc[table.index[0], "age"] += 1 + mutated = True + + try: + with _observe_exact_code_events(trace, returns=(graph.financial._artifacts,)): + with pytest.raises(ValueError, match="PUF55_RUN_POPULATION_CHANGED"): + graph.check_survey_puf55_run(run) + finally: + table["age"] = before + assert mutated + assert graph._ISSUED_RUNS.get(id(run)) is not entry + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + graph.check_survey_puf55_run(run) From d5cbe60b2c6402648565f5139dbf7d94be209def Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:25:50 -0400 Subject: [PATCH 120/255] Update release path for retained PUF output checks --- docs/us-uk-release-path.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 18162ad3a..57882d2bd 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -101,13 +101,22 @@ their source: the complete-population host must authenticate the original sampling references and retained parent before and after use. Native fit, full-size memory and calibration acceptance remain unestablished. -Complete-population ancestry remains an explicit requirement. The PUF host -checks its output before return but does not issue a persistent authority token -for its public dataclass. Later calibration and export must check the actual -retained output; they cannot treat its receipt as an admission mechanism. The -minimal invented fixture's 61 present input names out of 161 are diagnostic -only, with unresolved original-channel values. Native coverage will be measured -on the new candidate rather than inferred from this fixture. +Complete-population ancestry remains an explicit requirement. The PUF host now +retains its actual checked result for downstream consumption through +`check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot +issue that authority. Consumers must recheck the original run before use and +after their last relevant I/O. The corrected checker passes 25 controls on +actual invented cold/required execution, including extra manifest attachments, +source/store corruption and late output mutation. Independent review approved +the source and mechanical test colocation; 45 cases share one expensive fixture. +See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). + +The minimal invented fixture's 61 present input names out of 161 remain a +diagnostic inventory, with unresolved original-channel values. Native coverage +will be measured on the new candidate. Parallel successor work is qualifying +unemployment receipt/amount, three health-cost amounts and current interview +health coverage from the actual survey sources. Reporting universes and missing +values remain explicit; no prior-wage or age-only zero fallback is introduced. The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale From fb6d1dc5f2bd575a0cd7c947d1a50ae9ad06a424 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:30:54 -0400 Subject: [PATCH 121/255] Stop the snapshot observer from being able to break the run it watches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three adversarial review lenses raised 21 findings against the first draft of this slice; each went to a verifier told to refute it, and 10 survived. All 10 are fixed here, with a regression test each. The serious ones broke the slice's central promise — that turning the observer on changes nothing. It could abort a calibration that succeeds without it, twice over. Compiled row labels are the lossy f"{name}@{period}", so ("income", 2024) and ("income", "2024") produce two rows named income@2024; the digest refused duplicates and killed the run before epoch 1. And a non-finite float32 in-loop estimate raised, even though the capped loss absorbs it, the float64 closing estimate is finite, and the unobserved run returns weights. The digest now carries the row index rather than refusing duplicates, and a non-finite estimate or target serializes as null with a `non_finite_rows` count, following diagnostics._finite's "JSON has no NaN" convention. Two were honesty defects in the closing snapshot: it stamped the closing epoch on a retained-best iterate whose values came from an earlier one, and reported best_retained.available=false on a run that had retained and returned a best. Both now read the selection receipt, and say "retained, epoch unrecorded" where the closing iterate won rather than inventing an epoch. The store could collide across runs sharing a directory (sequences restart at 1 per observer), prune another run's chunks, or prune the chunk it had just written. Construction now refuses a populated directory unless explicitly adopted, a store refuses a second run's snapshots, and pruning is scoped to the chunks that writer actually wrote. The aggregate-only scan covered a hand-listed subset of keys and so missed candidate_id, phase and created_at; it now walks the whole payload, which also means a key added to the schema later is covered without anyone remembering to extend a list. Numeric fields are type-checked, so a stringified number no longer validates. Identity pins that the solve.py edits move are re-pinned: the graph shard's calibrate parity case, the spec-engine seed protocol and map digests plus the regenerated coverage evidence, and the resolved-spec golden vectors in the loader and pool-tool tests. Each was confirmed to reproduce the committed value on pristine origin/main under the same interpreter first, so the drift is attributable to this source edit. They are computed against this branch's base and will need recomputing on the merge ref, which has since moved. calibrate suite: 291 passed. Benchmark re-measured after the fixes. Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshots.md | 56 ++++ docs/evidence/spec-engine/us-f0-coverage.json | 16 +- .../908-target-snapshot-bench-receipt.json | 44 +-- .../908-target-snapshot-bench-receipts.md | 20 +- .../build/spec_engine/inventory_coverage.py | 4 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- .../src/microcosm/calibrate/solve.py | 26 +- .../microcosm/calibrate/target_snapshots.py | 184 +++++++++--- .../tests/test_target_snapshots.py | 268 +++++++++++++++++- .../parity/kernels/calibrate/pins.json | 2 +- 11 files changed, 526 insertions(+), 98 deletions(-) diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md index 68e4ec12e..d98b99b43 100644 --- a/PROGRESS-908-target-snapshots.md +++ b/PROGRESS-908-target-snapshots.md @@ -76,6 +76,62 @@ scope" below. off. - Calibration Diagnostics UI (different repository, by the issue's own text). +## Adversarial review round (same lane, before hand-off) + +Three independent review lenses (determinism, label honesty, codec/store) +raised 21 findings; each was handed to an adversarial verifier told to refute +it, and 10 survived. All 10 are fixed on this branch, with a regression test +each: + +- Enabling the observer aborted runs that succeed without it, in two ways — + duplicate compiled row labels (`row_name` is the lossy `f"{name}@{period}"`, + so `("income", 2024)` and `("income", "2024")` collide) and a non-finite + float32 in-loop estimate (the capped loss absorbs it and the run returns + weights). The digest now carries the row index instead of refusing + duplicates, and a non-finite estimate or target serializes as null with a + `non_finite_rows` count, following `diagnostics._finite`. +- The `selected` snapshot stamped the closing epoch on a retained-best + iterate, and claimed `best_retained.available: false` on a run that + retained and returned a best. Both now read the selection receipt. +- The store silently collided across runs sharing a directory (sequences + restart at 1 per observer) and could prune another run's chunks, or the + chunk it had just written. Construction now refuses a populated directory + unless explicitly adopted, a store refuses a second run's snapshots, and + pruning is scoped to the chunks that writer wrote. +- The aggregate-only scan covered a hand-listed subset of keys; it now walks + the whole payload. Numeric fields are type-checked, so a stringified number + no longer validates. + +## Source identity re-pins (disclosed) + +`microcosm.calibrate.solve` is an attested module: it is in +`_DIRECT_KERNEL_MODULES` (`spec_engine/seeds.py`) and it defines the graph +shard's `calibrate.adam@1` kernel. Editing it legitimately moves several +pinned identities. Each was re-pinned to a value computed from this branch, +after first confirming with the *same* interpreter that pristine `origin/main` +reproduces the committed pins exactly — so the drift is attributable to the +source edit, not to an environment leak: + +- `packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json` + (`implementation_hash`, `node_key`). `fit.qrf` and `simulate` untouched; the + regeneration asserted `graph.json`, `inputs.csv` and `direct.csv` come back + byte-identical, so the kernel's numbers did not change. +- `EXPECTED_HASHES["seed_protocol"]` and `["seed_map"]` in + `spec_engine/inventory_coverage.py`, plus the regenerated + `docs/evidence/spec-engine/us-f0-coverage.json`. +- The resolved-spec golden vectors in + `test_spec_engine_loader.py` and `test_us_multispine_pool_tool.py`. + +No other pin was touched. + +**These pins are correct for this branch's base (0c3f4f651) only.** +`origin/main` has since advanced to 116d46ee9 (#912, +`amend-keyed-seed-and-uniform-draws`), which itself edited the attested +`microcosm.fit.qrf` and re-pinned the same loader golden vector and pool-tool +spec digest. Every identity pin above must therefore be recomputed on the +merge ref before this branch merges, per CLAUDE.md's "CI tests the merge ref, +so merge main and re-pin". Do not treat the values here as final. + ## Next - Independent review of this slice, then the staging/host wiring as a diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 45e73dcbb..fba5dc97d 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "32dea3048a868f0b283f6ba17f23c5c2335c89a2abd13ca1c8023fe3251f05bf", + "protocol_sha256": "f4dc507c20bcf92e214ed3b8500be40a310a91d448be199115a4de43d50980c8" }, "failures": [], "observed": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "32dea3048a868f0b283f6ba17f23c5c2335c89a2abd13ca1c8023fe3251f05bf", + "protocol_sha256": "f4dc507c20bcf92e214ed3b8500be40a310a91d448be199115a4de43d50980c8" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "f4dc507c20bcf92e214ed3b8500be40a310a91d448be199115a4de43d50980c8", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "f4dc507c20bcf92e214ed3b8500be40a310a91d448be199115a4de43d50980c8", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "35a3623b9b1c17fdf04d8fc3cd0352ebc98297c1274d79ddec2fd520dc61e7f8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "35a3623b9b1c17fdf04d8fc3cd0352ebc98297c1274d79ddec2fd520dc61e7f8" }, "status": "pass" } diff --git a/experiments/908-target-snapshot-bench-receipt.json b/experiments/908-target-snapshot-bench-receipt.json index d6dc03578..f57535753 100644 --- a/experiments/908-target-snapshot-bench-receipt.json +++ b/experiments/908-target-snapshot-bench-receipt.json @@ -15,47 +15,47 @@ "numeric_threads": 1, "cpu_seconds_limit_applied": 120, "memory_bytes_limit_applied": null, - "peak_rss_bytes": 289226752, + "peak_rss_bytes": 289800192, "native_inputs_read": 0 }, "codec_costs": [ { "n_targets": 500, - "build_and_validate_ms": 1.687, - "serialize_ms": 0.507, - "store_write_ms": 3.385, - "serialized_bytes": 88602, + "build_and_validate_ms": 2.099, + "serialize_ms": 0.603, + "store_write_ms": 3.913, + "serialized_bytes": 88625, "bytes_per_target": 177.2 }, { "n_targets": 2000, - "build_and_validate_ms": 7.345, - "serialize_ms": 2.038, - "store_write_ms": 12.424, - "serialized_bytes": 353872, + "build_and_validate_ms": 9.087, + "serialize_ms": 2.321, + "store_write_ms": 13.491, + "serialized_bytes": 353895, "bytes_per_target": 176.9 }, { "n_targets": 5000, - "build_and_validate_ms": 16.641, - "serialize_ms": 4.912, - "store_write_ms": 26.369, - "serialized_bytes": 885576, + "build_and_validate_ms": 18.943, + "serialize_ms": 5.723, + "store_write_ms": 29.589, + "serialized_bytes": 885599, "bytes_per_target": 177.1 }, { "n_targets": 10000, - "build_and_validate_ms": 33.189, - "serialize_ms": 9.833, - "store_write_ms": 53.045, - "serialized_bytes": 1771910, + "build_and_validate_ms": 38.339, + "serialize_ms": 11.357, + "store_write_ms": 60.114, + "serialized_bytes": 1771933, "bytes_per_target": 177.2 } ], "solver_overhead": [ { "cadence": "observer_off", - "best_of_3_seconds": 0.0326, + "best_of_3_seconds": 0.036, "closing_loss": 2.7968459435242565e-07, "weights_bitwise_equal_to_observer_off": true, "snapshots_emitted": null, @@ -64,21 +64,21 @@ }, { "cadence": "bounded_every_25", - "best_of_3_seconds": 0.0555, + "best_of_3_seconds": 0.0597, "closing_loss": 2.7968459435242565e-07, "weights_bitwise_equal_to_observer_off": true, "snapshots_emitted": 14, "chunks_retained_on_disk": 14, - "retained_bytes_on_disk": 230903 + "retained_bytes_on_disk": 231241 }, { "cadence": "every_epoch", - "best_of_3_seconds": 0.603, + "best_of_3_seconds": 0.594, "closing_loss": 2.7968459435242565e-07, "weights_bitwise_equal_to_observer_off": true, "snapshots_emitted": 301, "chunks_retained_on_disk": 256, - "retained_bytes_on_disk": 4214808 + "retained_bytes_on_disk": 4220712 } ] } diff --git a/experiments/908-target-snapshot-bench-receipts.md b/experiments/908-target-snapshot-bench-receipts.md index 211f6691c..faf4d8816 100644 --- a/experiments/908-target-snapshot-bench-receipts.md +++ b/experiments/908-target-snapshot-bench-receipts.md @@ -9,7 +9,7 @@ points were chosen to bracket an order of magnitude; they are not measured from the real US or UK target registries. Run 2026-09-12 on macOS 26.6.2 / arm64, one numeric thread, CPU rlimit 120 s, -peak RSS 0.29 GiB (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the +peak RSS 0.29 GiB (re-measured after the adversarial-review fixes) (well inside the 2 GiB cap; macOS refused `RLIMIT_AS`, so the script records the limit it actually applied and the peak RSS it reached). Machine-readable copy: [`908-target-snapshot-bench-receipt.json`](908-target-snapshot-bench-receipt.json). @@ -21,10 +21,10 @@ replacement plus the index rewrite. | targets | build+validate (ms) | serialize (ms) | store write (ms) | serialized bytes | bytes/target | |--------:|--------------------:|---------------:|-----------------:|-----------------:|-------------:| -| 500 | 1.69 | 0.51 | 3.39 | 88,602 | 177.2 | -| 2,000 | 7.35 | 2.04 | 12.42 | 353,872 | 176.9 | -| 5,000 | 16.64 | 4.91 | 26.37 | 885,576 | 177.1 | -| 10,000 | 33.19 | 9.83 | 53.05 | 1,771,910 | 177.2 | +| 500 | 2.10 | 0.60 | 3.91 | 88,625 | 177.2 | +| 2,000 | 9.09 | 2.32 | 13.49 | 353,895 | 176.9 | +| 5,000 | 18.94 | 5.72 | 29.59 | 885,599 | 177.1 | +| 10,000 | 38.34 | 11.36 | 60.11 | 1,771,933 | 177.2 | Both cost and size are linear in target count at ~177 bytes per target row (indent=1, sorted keys). A single 10,000-target snapshot is ~1.7 MB. @@ -36,9 +36,9 @@ discarded warm-up run, then best of three timed runs per cadence. | cadence | best-of-3 (s) | snapshots emitted | chunks retained | retained bytes | weights bitwise equal to observer-off | |---|---:|---:|---:|---:|---| -| observer off | 0.0326 | — | 0 | 0 | yes (reference) | -| bounded, every 25 | 0.0555 | 14 | 14 | 230,903 | yes | -| every epoch | 0.6030 | 301 | 256 | 4,214,808 | yes | +| observer off | 0.0360 | — | 0 | 0 | yes (reference) | +| bounded, every 25 | 0.0597 | 14 | 14 | 231,241 | yes | +| every epoch | 0.5940 | 301 | 256 | 4,220,712 | yes | The closing loss is identical to all printed digits across all three (`2.7968459435242565e-07`), and the returned weight vectors are bitwise equal. @@ -48,8 +48,8 @@ The closing loss is identical to all printed digits across all three It does establish that emission is linear, that ~177 bytes/target is the wire cost, and that turning the observer on does not change the optimizer's result. -It does **not** license a production cadence default. This solve costs 33 ms -total, so per-snapshot cost dominates it by construction and the 19x +It does **not** license a production cadence default. This solve costs 36 ms +total, so per-snapshot cost dominates it by construction and the 16x every-epoch ratio above is an artifact of a trivially cheap solve — a real calibration's epoch is far more expensive, so the same absolute per-snapshot cost would be a much smaller fraction of it. Choosing the production default diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 89171055c..39df3e7e8 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "seed_protocol": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "seed_map": "32dea3048a868f0b283f6ba17f23c5c2335c89a2abd13ca1c8023fe3251f05bf", + "seed_protocol": "f4dc507c20bcf92e214ed3b8500be40a310a91d448be199115a4de43d50980c8", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 0b583de0a..66d072e06 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "57026e5896dd52a382746fe7641f615dcc4c55965a1f50bea5971a8f92710d71" + "a67bb78c38c13382fb95340d8ab64059f4f94991d3b9ad7dbdae3f5f965ba163" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 019e495d4..013afdbcb 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2504,7 +2504,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6", + "spec_sha256": "35a3623b9b1c17fdf04d8fc3cd0352ebc98297c1274d79ddec2fd520dc61e7f8", }, } diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index 67a258e8b..b9a20cc86 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -2161,13 +2161,37 @@ def calibrate( # read off the same float64 estimates the final diagnostics use — not # the last in-loop iterate, which the optimizer may have discarded in # favour of an earlier better one or changed by a closing projection. + selected_epoch = iterate_selection_receipt.get("selected_epoch") + # A non-empty receipt means the optimizer ran the retain-best rule. It + # records the epoch it selected, which IS the best iterate's epoch when + # an earlier iterate won; when the closing iterate won, no separate best + # epoch was recorded, so the snapshot says "retained, epoch unrecorded" + # instead of inventing one. + retained_best = bool(iterate_selection_receipt) + best_is_earlier = ( + retained_best + and isinstance(selected_epoch, int) + and selected_epoch < epochs + ) snapshots.emit( final_estimates, - epoch=epochs, + # The epoch whose iterate was actually selected, not the last one + # executed: a retained-best run returns an earlier iterate, and + # stamping the closing epoch on it would misattribute the values. + epoch=(int(selected_epoch) if isinstance(selected_epoch, int) else epochs), epochs=epochs, iterate=ITERATE_SELECTED, precision="float64", loss=float(closing_loss), + best_retained={ + "available": retained_best, + "epoch": int(selected_epoch) if best_is_earlier else None, + "loss": ( + iterate_selection_receipt.get("selected_loss_float32") + if best_is_earlier + else None + ), + }, selection=dict(iterate_selection_receipt) or None, ) effective_target_loss_weights = ( diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py index 807a034ee..873760ec0 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -141,6 +141,7 @@ "iterate", "precision", "loss", + "non_finite_rows", "best_retained", "selection", "targets_sha256", @@ -159,6 +160,20 @@ _RELATIVE_ERROR_RTOL = 1e-9 +def _strict_number(value: object, *, what: str) -> float | None: + """A JSON number or null. A bool or a numeric string is neither.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TargetSnapshotError( + f"{what} must be a JSON number or null, got {value!r}." + ) + result = float(value) + if not math.isfinite(result): + raise TargetSnapshotError(f"{what} must be finite when present.") + return result + + class TargetSnapshotError(ValueError): """A snapshot payload violates the versioned contract.""" @@ -167,6 +182,23 @@ def _utc_now() -> datetime: return datetime.now(UTC) +def _finite_or_none(value: object) -> float | None: + """A float, or ``None`` when it is not finite. + + JSON has no NaN or Infinity, and a diverging optimizer can legitimately + hold a non-finite estimate mid-run — the capped loss absorbs it and the + run still returns weights. Following + :func:`microcosm.calibrate.diagnostics._finite`, such a value serializes + as null rather than aborting the calibration that an unobserved run would + have completed. + """ + try: + result = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None + + def _finite_float(value: object, *, what: str) -> float: try: result = float(value) # type: ignore[arg-type] @@ -189,7 +221,7 @@ def _canonical_json_bytes(value: object) -> bytes: ).encode("utf-8") -def signed_relative_error(estimate: float, target: float) -> float: +def signed_relative_error(estimate: float | None, target: float | None) -> float | None: """The signed miss of ``estimate`` against ``target``. ``(estimate - target) / target``, except that a zero target has no @@ -199,39 +231,47 @@ def signed_relative_error(estimate: float, target: float) -> float: ``_target_diagnostics`` implements, so a mid-run snapshot and the run's final diagnostics never disagree about what a target's error means. """ - estimate = _finite_float(estimate, what="estimate") - target = _finite_float(target, what="target") + estimate = _finite_or_none(estimate) + target = _finite_or_none(target) + if estimate is None or target is None: + # No honest signed error exists; the row reports null rather than + # inventing one or aborting the run. + return None if target == 0.0: return estimate - target - return (estimate - target) / target + result = estimate - target if target == 0.0 else (estimate - target) / target + return _finite_or_none(result) def target_identity_digest( names: Sequence[str], targets: Sequence[float] | np.ndarray ) -> str: - """A sha256 over the ordered ``(name, target value)`` list. + """A sha256 over the ordered ``(row index, name, target value)`` list. Order-sensitive by construction: the digest covers the list, not a set, so a consumer comparing two snapshots' digests is comparing the exact ordered - target identity the rows are aligned to. + target identity the rows are aligned to. The row index is part of each + entry, so identity stays unambiguous even where a compiled problem happens + to produce two rows with the same ``name@period`` label — a snapshot must + never be the thing that aborts a calibration that would otherwise return + weights. A non-finite target digests as null, for the same reason. """ names = tuple(str(name) for name in names) - values = [ - _finite_float(value, what="target value") for value in np.asarray(targets) - ] + values = [_finite_or_none(value) for value in np.asarray(targets)] if len(names) != len(values): raise TargetSnapshotError( f"names and targets must align: {len(names)} names, {len(values)} targets." ) if not names: raise TargetSnapshotError("a calibration snapshot needs at least one target.") - if len(set(names)) != len(names): - raise TargetSnapshotError("target row names must be unique.") if any(not name for name in names): raise TargetSnapshotError("target row names must be non-empty.") return hashlib.sha256( _canonical_json_bytes( - [[name, value] for name, value in zip(names, values, strict=True)] + [ + [index, name, value] + for index, (name, value) in enumerate(zip(names, values, strict=True)) + ] ) ).hexdigest() @@ -323,9 +363,7 @@ def bind( ) -> BoundTargetSnapshots: """Bind this observer to a compiled problem's ordered targets.""" names = tuple(str(name) for name in names) - values = tuple( - _finite_float(value, what="target value") for value in np.asarray(targets) - ) + values = tuple(_finite_or_none(value) for value in np.asarray(targets)) return BoundTargetSnapshots( observer=self, names=names, @@ -350,7 +388,7 @@ class BoundTargetSnapshots: observer: TargetSnapshotObserver names: tuple[str, ...] - targets: tuple[float, ...] + targets: tuple[float | None, ...] digest: str counter: list[int] phase: str | None = None @@ -399,12 +437,13 @@ def snapshot( f"{values.shape[0]}, expected {len(self.names)}." ) rows: list[dict[str, object]] = [] + non_finite = 0 for index, (name, target) in enumerate( zip(self.names, self.targets, strict=True) ): - estimate = _finite_float( - values[index], what=f"estimate for target {name!r}" - ) + estimate = _finite_or_none(values[index]) + if estimate is None or target is None: + non_finite += 1 rows.append( { "index": index, @@ -432,7 +471,8 @@ def snapshot( "search": None if self.search is None else dict(self.search), "iterate": iterate, "precision": precision, - "loss": None if loss is None else _finite_float(loss, what="loss"), + "loss": _finite_or_none(loss), + "non_finite_rows": non_finite, "best_retained": ( {"available": False, "epoch": None, "loss": None} if best_retained is None @@ -491,8 +531,14 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: value = payload[key] if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise TargetSnapshotError(f"{key} must be a non-negative integer.") - if payload["loss"] is not None: - _finite_float(payload["loss"], what="loss") + _strict_number(payload["loss"], what="loss") + non_finite_rows = payload["non_finite_rows"] + if ( + isinstance(non_finite_rows, bool) + or not isinstance(non_finite_rows, int) + or non_finite_rows < 0 + ): + raise TargetSnapshotError("non_finite_rows must be a non-negative integer.") best = payload["best_retained"] if not isinstance(best, Mapping) or "available" not in best: raise TargetSnapshotError("best_retained must record whether one is available.") @@ -512,7 +558,8 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: f"n_targets is {payload['n_targets']} but {len(rows)} rows are present." ) names: list[str] = [] - values: list[float] = [] + values: list[float | None] = [] + observed_non_finite = 0 for position, row in enumerate(rows): if not isinstance(row, Mapping): raise TargetSnapshotError(f"target row {position} is not a mapping.") @@ -534,32 +581,49 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: name = row["name"] if not isinstance(name, str) or not name: raise TargetSnapshotError(f"target row {position} needs a non-empty name.") - target = _finite_float(row["target"], what=f"target for {name!r}") - estimate = _finite_float(row["estimate"], what=f"estimate for {name!r}") - stored = _finite_float( + target = _strict_number(row["target"], what=f"target for {name!r}") + estimate = _strict_number(row["estimate"], what=f"estimate for {name!r}") + stored = _strict_number( row["relative_error"], what=f"relative_error for {name!r}" ) - expected = signed_relative_error(estimate, target) - if not math.isclose( - stored, expected, rel_tol=_RELATIVE_ERROR_RTOL, abs_tol=0.0 - ): - raise TargetSnapshotError( - f"relative_error for {name!r} is {stored!r}; the signed relative " - f"error of {estimate!r} against {target!r} is {expected!r}." - ) + if estimate is None or target is None: + observed_non_finite += 1 + if stored is not None: + raise TargetSnapshotError( + f"relative_error for {name!r} must be null when its estimate " + "or target is." + ) + else: + expected = signed_relative_error(estimate, target) + if stored is None or expected is None: + raise TargetSnapshotError( + f"relative_error for {name!r} must be present when both its " + "estimate and target are." + ) + if not math.isclose( + stored, expected, rel_tol=_RELATIVE_ERROR_RTOL, abs_tol=0.0 + ): + raise TargetSnapshotError( + f"relative_error for {name!r} is {stored!r}; the signed relative " + f"error of {estimate!r} against {target!r} is {expected!r}." + ) names.append(name) values.append(target) - if len(set(names)) != len(names): - raise TargetSnapshotError("target row names must be unique within a snapshot.") + if observed_non_finite != non_finite_rows: + raise TargetSnapshotError( + f"non_finite_rows says {non_finite_rows} but {observed_non_finite} rows " + "carry a null estimate or target." + ) digest = target_identity_digest(names, values) if payload["targets_sha256"] != digest: raise TargetSnapshotError( "targets_sha256 does not digest this snapshot's ordered target " f"identity: declared {payload['targets_sha256']!r}, computed {digest!r}." ) - for key in ("context", "search", "selection", "best_retained"): - _reject_record_level_keys(payload[key], where=f"snapshot {key}") - _reject_record_level_keys(payload["targets"], where="snapshot targets") + # Scan the WHOLE payload, not a chosen subset: a key added to the schema + # later must be covered by the aggregate-only rule without anyone + # remembering to extend a list here. + _reject_record_level_keys(payload, where="snapshot") # A last strictness check: the payload must be strict JSON, so a consumer # never receives NaN/Infinity tokens no parser outside Python accepts. _canonical_json_bytes(payload) @@ -609,7 +673,16 @@ class TargetSnapshotWriter: directory: Path history_limit: int | None = 256 + #: Adopt a directory that already holds history chunks. Off by default: + #: chunk names come from the observer's sequence counter, which restarts + #: at 1 for every new observer, so a second run sharing a directory would + #: collide with — and prune — the first run's chunks. Failing at + #: construction names that at the point a caller can fix it, instead of + #: mid-calibration. + allow_existing_history: bool = False _dropped: int = field(default=0, init=False, repr=False) + _written: list[int] = field(default_factory=list, init=False, repr=False) + _run_id: str | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: if self.history_limit is not None and ( @@ -624,6 +697,15 @@ def __post_init__(self) -> None: self.directory = Path(self.directory) self.directory.mkdir(parents=True, exist_ok=True) self.history_directory.mkdir(parents=True, exist_ok=True) + existing = self.retained() + if existing and not self.allow_existing_history: + raise TargetSnapshotError( + f"{self.history_directory} already holds {len(existing)} snapshot " + "chunk(s). Snapshot sequences restart at 1 for every observer, so " + "writing a second run here would collide with and prune the first " + "run's history. Use a per-run directory, or pass " + "allow_existing_history=True if you have reconciled the sequences." + ) @property def history_directory(self) -> Path: @@ -661,8 +743,17 @@ def _write_chunk(self, path: Path, payload: Mapping[str, object]) -> None: def __call__(self, payload: Mapping[str, object]) -> None: validate_target_snapshot(payload) + run_id = str(payload["run_id"]) + if self._run_id is None: + self._run_id = run_id + elif run_id != self._run_id: + raise TargetSnapshotError( + f"this store belongs to run {self._run_id!r}; it will not accept a " + f"snapshot from run {run_id!r}. Give each run its own directory." + ) sequence = int(payload["sequence"]) self._write_chunk(self.history_directory / f"{sequence:08d}.json", payload) + self._written.append(sequence) self._prune() _atomic_write_json(self.latest_path, dict(payload)) retained = self.retained() @@ -683,15 +774,20 @@ def __call__(self, payload: Mapping[str, object]) -> None: ) def _prune(self) -> int: - """Drop the oldest chunks past ``history_limit``; return how many.""" + """Drop the oldest chunks THIS writer wrote, past ``history_limit``. + + Pruning is scoped to this writer's own sequences: a directory it was + told to adopt keeps whatever was already there, rather than this run + silently deleting another one's evidence. + """ if self.history_limit is None: return 0 - chunks = self.retained() - excess = len(chunks) - self.history_limit + excess = len(self._written) - self.history_limit if excess <= 0: return 0 - for name in chunks[:excess]: - (self.history_directory / name).unlink(missing_ok=True) + for sequence in self._written[:excess]: + (self.history_directory / f"{sequence:08d}.json").unlink(missing_ok=True) + del self._written[:excess] self._dropped += excess return excess diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index 15db55a5a..c9d6751e3 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -114,13 +114,32 @@ def test_target_identity_digest_is_order_sensitive_and_value_sensitive(): assert len(base) == 64 -def test_target_identity_digest_refuses_duplicate_or_non_finite(): +def test_target_identity_digest_separates_duplicate_names_by_row(): + """Duplicate row labels must not abort a run the observer is only watching. + + A compiled problem can produce two rows labelled the same (row_name is the + lossy f"{name}@{period}"), so the digest carries the row index and stays + unambiguous instead of refusing. + """ + digest = target_identity_digest(("a", "a"), (1.0, 2.0)) + assert digest != target_identity_digest(("a", "a"), (2.0, 1.0)) + assert len(digest) == 64 + + +def test_target_identity_digest_digests_a_non_finite_target_as_null(): + nan = target_identity_digest(("a", "b"), (1.0, float("nan"))) + inf = target_identity_digest(("a", "b"), (1.0, float("inf"))) + assert nan == inf # both are "no finite target value" + assert nan != target_identity_digest(("a", "b"), (1.0, 2.0)) + + +def test_target_identity_digest_refuses_misaligned_or_empty_input(): with pytest.raises(TargetSnapshotError): - target_identity_digest(("a", "a"), (1.0, 2.0)) + target_identity_digest(("a", "b"), (1.0,)) with pytest.raises(TargetSnapshotError): - target_identity_digest(("a", "b"), (1.0, float("nan"))) + target_identity_digest((), ()) with pytest.raises(TargetSnapshotError): - target_identity_digest(("a", "b"), (1.0,)) + target_identity_digest(("",), (1.0,)) def test_signed_relative_error_matches_the_solver_zero_target_convention(): @@ -161,10 +180,15 @@ def test_validate_rejects_non_finite_and_inconsistent_relative_error(): with pytest.raises(TargetSnapshotError, match="relative_error"): validate_target_snapshot(broken) - with pytest.raises(TargetSnapshotError, match="finite"): - bound.snapshot( - np.array([float("inf")]), epoch=1, epochs=1, iterate=ITERATE_CURRENT - ) + # A non-finite estimate is recorded as null, not raised: an unobserved run + # would have completed, so the observer must not be what kills it. + diverged = bound.snapshot( + np.array([float("inf")]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + assert diverged["targets"][0]["estimate"] is None + assert diverged["targets"][0]["relative_error"] is None + assert diverged["non_finite_rows"] == 1 + validate_target_snapshot(diverged) def test_validate_refuses_record_level_identifiers_in_context(): @@ -472,3 +496,231 @@ def test_snapshots_carry_no_record_level_vectors(tmp_path: Path): and value and isinstance(value[0], float) ) + + +# -------------------------------------------------------------------------- +# Regressions found by adversarial review of the first draft of this slice +# -------------------------------------------------------------------------- + + +def test_a_diverging_run_still_returns_weights_with_the_observer_on(): + """The observer must never be the thing that aborts an otherwise-good run. + + At this learning rate the estimate blows up to inf mid-run; the capped loss + absorbs it and calibrate returns weights. The first draft raised on the + non-finite estimate and destroyed a run that would have succeeded. + """ + baseline = calibrate(_frame(), _targets(), epochs=60, seed=0, learning_rate=14.0) + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + observed = calibrate( + _frame(), + _targets(), + epochs=60, + seed=0, + learning_rate=14.0, + target_snapshots=observer, + ) + np.testing.assert_array_equal(baseline.weights, observed.weights) + np.testing.assert_array_equal(baseline.loss_trajectory, observed.loss_trajectory) + assert any(s["non_finite_rows"] > 0 for s in seen), ( + "this fixture is meant to drive a non-finite estimate" + ) + for snapshot in seen: + validate_target_snapshot(snapshot) + + +def test_duplicate_compiled_row_labels_do_not_abort_the_run(): + """`row_name` is the lossy f"{name}@{period}", so labels can collide.""" + targets = TargetSet( + [ + Target( + name="income", + period=2024, + entity="household", + measure="income", + value=12000.0, + ), + Target( + name="income", + period="2024", + entity="household", + measure="income", + value=13000.0, + ), + ] + ) + baseline = calibrate(_frame(), targets, epochs=10, seed=0) + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + observed = calibrate( + _frame(), targets, epochs=10, seed=0, target_snapshots=observer + ) + np.testing.assert_array_equal(baseline.weights, observed.weights) + assert seen + names = [row["name"] for row in seen[0]["targets"]] + assert names == ["income@2024", "income@2024"] + for snapshot in seen: + validate_target_snapshot(snapshot) + + +def test_the_selected_snapshot_names_the_epoch_it_actually_selected(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), + _targets(), + epochs=30, + seed=0, + learning_rate=0.1, + target_snapshots=observer, + ) + receipt = result.options["iterate_selection_receipt"] + selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED][-1] + assert selected["epoch"] == receipt["selected_epoch"] + + +def test_a_store_refuses_to_share_a_directory_with_another_run(): + """Sequences restart at 1 per observer, so a shared directory collides.""" + import tempfile + + with tempfile.TemporaryDirectory() as directory: + first = TargetSnapshotWriter(Path(directory)) + for payload in _payloads(2): + first(payload) + with pytest.raises(TargetSnapshotError, match="already holds"): + TargetSnapshotWriter(Path(directory)) + # Even the explicit adoption escape hatch keeps chunks write-once: a + # colliding sequence is refused, never silently overwritten. + adopted = TargetSnapshotWriter(Path(directory), allow_existing_history=True) + with pytest.raises(TargetSnapshotError, match="immutable"): + adopted(_payloads(1)[0]) + + +def test_pruning_never_deletes_another_runs_retained_chunks(): + import tempfile + + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + with tempfile.TemporaryDirectory() as directory: + first = TargetSnapshotWriter(Path(directory)) + for index in range(2): + first( + bound.snapshot( + np.array([9.0 + index]), + epoch=index + 1, + epochs=2, + iterate=ITERATE_CURRENT, + ) + ) + # A second writer adopting the directory continues the same observer's + # sequence, so nothing collides; its own bounded history must prune only + # its own chunks. + second = TargetSnapshotWriter( + Path(directory), history_limit=1, allow_existing_history=True + ) + for index in range(3): + second( + bound.snapshot( + np.array([20.0 + index]), + epoch=index + 1, + epochs=3, + iterate=ITERATE_CURRENT, + ) + ) + retained = sorted(p.name for p in (Path(directory) / "history").iterdir()) + assert "00000001.json" in retained + assert "00000002.json" in retained + assert retained[-1] == "00000005.json" + + +def test_a_store_refuses_a_snapshot_from_a_different_run(): + import tempfile + + with tempfile.TemporaryDirectory() as directory: + writer = TargetSnapshotWriter(Path(directory)) + first = TargetSnapshotObserver(sink=lambda _p: None, run_id="run-a").bind( + names=("a@2024",), targets=np.array([10.0]) + ) + second = TargetSnapshotObserver(sink=lambda _p: None, run_id="run-b").bind( + names=("a@2024",), targets=np.array([10.0]) + ) + writer( + first.snapshot(np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT) + ) + with pytest.raises(TargetSnapshotError, match="run 'run-a'"): + writer( + second.snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + ) + + +def test_validation_refuses_string_typed_numbers(): + _, observer = _collect() + payload = observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + for field_name in ("target", "estimate", "relative_error"): + broken = json.loads(json.dumps(payload)) + broken["targets"][0][field_name] = str(broken["targets"][0][field_name]) + with pytest.raises(TargetSnapshotError, match="JSON number"): + validate_target_snapshot(broken) + broken = json.loads(json.dumps(payload)) + broken["loss"] = "0.5" + with pytest.raises(TargetSnapshotError, match="JSON number"): + validate_target_snapshot(broken) + + +def test_the_aggregate_only_scan_covers_the_whole_payload(): + _, observer = _collect() + payload = observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + smuggled = json.loads(json.dumps(payload)) + smuggled["selection"] = {"rule": "x", "row_id": 7} + with pytest.raises(TargetSnapshotError, match="aggregate-only"): + validate_target_snapshot(smuggled) + + smuggled = json.loads(json.dumps(payload)) + smuggled["best_retained"]["source_values"] = [1.0, 2.0] + with pytest.raises(TargetSnapshotError, match="aggregate-only"): + validate_target_snapshot(smuggled) + + smuggled = json.loads(json.dumps(payload)) + smuggled["targets"][0]["name"] = "a@2024" + smuggled["search"] = {"budget_iteration": 1, "benunit_id": 3} + with pytest.raises(TargetSnapshotError, match="aggregate-only"): + validate_target_snapshot(smuggled) + + +def test_the_selected_snapshot_admits_the_run_retained_a_best(): + """A retained-best run must not report best_retained.available=false.""" + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), + _targets(), + epochs=30, + seed=0, + learning_rate=0.1, + target_snapshots=observer, + ) + receipt = result.options["iterate_selection_receipt"] + assert receipt, "this fixture is meant to exercise the retain-best rule" + selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED][-1] + assert selected["best_retained"]["available"] is True + if receipt["selected_epoch"] < 30: + assert selected["best_retained"]["epoch"] == receipt["selected_epoch"] + else: + assert selected["best_retained"]["epoch"] is None + + +def test_a_closing_state_run_still_reports_no_retained_best_on_selected(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + calibrate( + _frame(), + _targets(), + epochs=10, + seed=0, + mass=CONSERVE_MASS, + target_snapshots=observer, + ) + selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED][-1] + assert selected["best_retained"]["available"] is False diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index 437d54659..44e121b7d 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"bd0206f77cc3e1b54179e43f888e894f2b2b561a57f451fd810fb71be4ae1951","kernel":"calibrate.adam@1","node":"calibrate","node_key":"f29e0537c41345d9cfdd80f4789083b09bcbda7b3205411a2b95459231e37091","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"f29e0537c41345d9cfdd80f4789083b09bcbda7b3205411a2b95459231e37091"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32","kernel":"calibrate.adam@1","node":"calibrate","node_key":"970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311"}},"seed":0} From bae1887ffc5f6e32edefa9774ec6428568286247 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 10:42:25 -0400 Subject: [PATCH 122/255] Record how this lane ran the suite without installing anything Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshots.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md index d98b99b43..35b8b6d9e 100644 --- a/PROGRESS-908-target-snapshots.md +++ b/PROGRESS-908-target-snapshots.md @@ -132,6 +132,33 @@ spec digest. Every identity pin above must therefore be recomputed on the merge ref before this branch merges, per CLAUDE.md's "CI tests the merge ref, so merge main and re-pin". Do not treat the values here as final. +## How this lane ran the tests (no installs, no uv sync) + +An isolated interpreter with this worktree's shard sources ahead of a shared +venv's site-packages, plus an import-path assertion so `microcosm.*` can never +resolve outside this worktree: + +```python +# run.py — invoke as: /bin/python -I -B -S run.py +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VENV_SITE = Path("/site-packages") +sys.path[:0] = [ + *sorted(str(p) for p in ROOT.glob("packages/*/src")), + str(VENV_SITE), + str(ROOT), +] +import microcosm.calibrate as _cal +assert str(ROOT) in str(Path(_cal.__file__).resolve()), _cal.__file__ +import pytest +raise SystemExit(pytest.main(sys.argv[1:])) +``` + +Note pytest `addopts` already carries `-q`, so gate on the exit code rather +than adding another `-q` (which hides the summary line). + ## Next - Independent review of this slice, then the staging/host wiring as a From e3bbb7a45c002deeea03f1b62654dfb705fcfac5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:06:55 -0400 Subject: [PATCH 123/255] Open ASEC income reporting/routing source lane journal Record the verified 2025 dictionary literals for the pension/annuity, IRA distribution, net property, farm and other-income families, together with the printed source-level ambiguities that must stay distinguishable from modeling judgments. Co-Authored-By: Claude Opus 5 --- PROGRESS-current-asec-income-routing.md | 101 ++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 PROGRESS-current-asec-income-routing.md diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md new file mode 100644 index 000000000..adde17838 --- /dev/null +++ b/PROGRESS-current-asec-income-routing.md @@ -0,0 +1,101 @@ +# Current ASEC income reporting/routing source qualifier — lane progress + +Branch `current-asec-income-routing-20260912`, worktree +`_worktrees/microcosm-current-asec-income-routing-20260912`, base +`d5cbe60b2c6402648565f5139dbf7d94be209def` (reviewed US integration). + +Journal, not state. Check git/GitHub for current truth. + +## Scope + +Deliver a thin **source qualifier** plus a **pure reporting/routing projection** +for the five original-channel ASEC income families the current graph does not +yet supply: pension/annuity, IRA distributions, net property income, farm, and +other income. Consumed later by the already-owned `graph_us_survey_enrichment` +host, which root's active amount owner owns. This lane creates **no** host, no +issuer, no engine execution, no native cell writes, no release claim. + +## State + +- [x] Read the actual-current-graph trace + (`_recovered/scratch-backup/893/codex-takeover-20260912/original-tax-graph-gap-source-trace.md`) + and confirmed the current host does not call `derive_cps_carried_current_leaves` + or the preclone gap-fill. +- [x] Verified the official 2025 dictionary bytes locally: + SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +- [x] Read the accepted UC pattern + (`current_asec_unemployment_source.py`, reviewed hash + `624c76ae058e7b2a466a017eba348327c94751088d97011d3474cd8314187519`) read-only + in the successor worktree; not mutated. +- [x] Read the in-base siblings `current_social_security_source.py`, + `current_asec_demographics.py`, `current_survey_predictors.py`. +- [x] Extracted every target field's printed literal (length/position/range, + verbatim Universe and Values) from the verified dictionary, and + cross-checked each against `asec_current_money_domains_v1.json`: exact match + on position, length, universe and values for all nine money fields. +- [ ] Module `current_asec_income_routing_source.py` +- [ ] Tests `test_us_current_asec_income_routing.py` +- [ ] Source contract note + changelog fragment +- [ ] Ruff, CI inventory `--verify`, bounded pytest + +## Next + +Write the qualifier and its invented-source tests; run red-then-green. + +## Verified source literals (2025 dictionary, pages 43-49 + allocation pages) + +All quoted verbatim from the pinned PDF. + +| Field | Len | Pos | Range | Universe as printed | +| --- | --- | --- | --- | --- | +| PNSN_VAL | 7 | 571 | (0:9999999) | `PEN_YN = 1` | +| PEN_YN | 1 | 570 | (0:2) | `All Persons aged 15+` | +| ANN_VAL | 6 | 438 | (-1:999999) | `ANN_YN = 1` | +| ANN_YN | 1 | 444 | (0:2) | `All Persons aged 15+` | +| DST_VAL1 | 6 | 495 | (000000:999999) | `DST_SC1 = 1` | +| DST_VAL1_YNG | 6 | 501 | (000000:999999) | `DST_SC1_YNG = 1` | +| DST_VAL2 | 6 | 507 | (000000:999999) | `DST_SC2 = 1` | +| DST_VAL2_YNG | 6 | 513 | (000000:999999) | `DST_SC2_YNG = 1` | +| DST_SC1 | 1 | 491 | (0:7) | `DST_VAL1 > 0 and a_age >= 58` | +| DST_SC1_YNG | 1 | 492 | (0:7) | `DST_YN_YNG = 1 and a_age < 58` | +| DST_SC2 | 1 | 493 | (0:7) | `DST_VAL2 > 0 and a_age >= 58` | +| DST_SC2_YNG | 1 | 494 | (0:7) | `DST_VAL_YNG > 0 and a_age < 58` | +| DST_YN | 1 | 519 | (0:2) | `Persons aged 58 and over (a_age >= 58)` | +| DST_YN_YNG | 1 | 520 | (0:2) | `Persons under age 58 (a_age < 58)` | +| RNT_VAL | 6 | 621 | (-9999:999999) | `RNT_YN = 1` | +| RNT_YN | 1 | 627 | (0:2) | `All Persons aged 15+` | +| FRSE_VAL | 7 | 390 | (-9999999:9999999) | `ERN_YN=1 or FRMOTR=1` | +| FRSE_YN | 1 | 397 | (0:2) | `ERN_YN=1 or FRMOTR=1` | +| ERN_YN | 1 | 381 | (0:2) | `WORKYN=1 OR WTEMP=1` | +| FRMOTR | 1 | 389 | (0:2) | `ERN_OTR = 1` | +| OI_VAL | 6 | 549 | (0:999999) | `OI_YN = 1` | +| OI_OFF | 2 | 547 | (0:20) | `OI_YN = 1` | +| OI_YN | 1 | 555 | (0:2) | `All Persons aged 15+` | + +Source-level questions that stay open (evidence, not modeling judgments): + +- `PNSN_VAL` is printed as "total combined amount of pension income received + from **all** pension sources". It is not an observed private/taxable amount. + The legacy 0.590 split is a modeled assumption and is not applied here. +- `RNT_YN`'s printed question covers rent, royalties, roomers/boarders **and** + estates or trusts; `RNT_VAL`'s printed question asks only about "income from + rent after expenses". The receipt and amount questions do not have the same + printed coverage, so the total is not independently labelled rental. +- `DST_SC*` code 4 is `Regular IRA`. Account identity does not observe a taxable + fraction, so no taxable component is derived. +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` (401k account) although its + label is the source-1 distribution amount; preserved verbatim as an + unresolved printed-universe ambiguity rather than silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry; preserved verbatim. +- `OI_YN`'s printed `0` label is `none or niu`, unlike `PEN_YN`/`ANN_YN`/ + `RNT_YN`/`DST_YN` whose `0` is `niu`. Zero receipt literals are therefore not + interchangeable across the five families. +- Published allocation flags exist for `ANN_VAL` (`I_ANNVAL`), `ANN_YN` + (`I_ANNYN`), `PEN_YN` (`I_PENYN`), `RNT_VAL` (`I_RNTVAL`), `RNT_YN` + (`I_RNTYN`), `OI_VAL` (`I_OIVAL`), `ERN_YN` (`I_ERNYN`), and the DST + composites (`I_DSTSC`, `I_DSTSCCOMP`, `I_DSTVAL1COMP`, `I_DSTVAL2COMP`, + `I_DSTYNCOMP`). The dictionary publishes **no** direct flag for `PNSN_VAL`, + `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `DST_VAL*`, `DST_YN` or `FRMOTR`; their + allocation provenance is therefore recorded as unresolved, not as "raw". +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty in the dictionary. From 80342aa58398185514fe559e619027a77eb6834d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:08:53 -0400 Subject: [PATCH 124/255] Close the snapshot metadata, detachment, publication and codec findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The independent review of the #908 slice reproduced four defects. Each is closed here at the contract level rather than at the example level. A closed, typed, bounded aggregate metadata contract replaces the denylist over arbitrary nested JSON. Every seam the schema has — context, search, selection and best_retained — is now a flat mapping of string keys to JSON scalars, bounded in entry count, key length and string length, and best_retained is a closed {available, epoch, loss} triple. A list is not a scalar, so a record-level vector has no shape to travel in at any depth, and a nested payload is refused for the same reason. The shape is the one the solver already emits; context has no in-tree producer, so it takes the same flat contract rather than a wider one invented for a caller that does not exist. The record-level key-name rule is kept on top, because a scalar household_id is still record-level identity. Identifiers are strings, checked rather than coerced: str() of a mapping is a perfectly good Python string, and that is how record content reached a supposedly aggregate-only candidate_id. Because every supported metadata value is an immutable scalar and every container in a payload is freshly built, a sink that mutates what it was handed cannot reach the caller's mapping, the observer, or the next snapshot. A structural test pins that: a delivered payload shares no mutable object with the caller's arguments, the observer, the bound view, or the next snapshot. History chunks are now published, not created and then filled. The bytes go to a hidden temporary that is written, flushed and fsynced first; only a complete file takes the immutable history/.json name, through os.link, which is atomic and refuses rather than overwrites. A polling reader can no longer open a chunk mid-write, and a write that fails part-way publishes nothing and leaves no temporary behind. Atomic latest.json replacement, run ownership, duplicate refusal and bounded retention are unchanged. The public codec now refuses the impossible metadata it used to admit: created_at that is not a timezone-aware timestamp, a non-string identifier, an epoch past the stretch it declares, a sequence below its first emission, non_finite_rows past the row count, a best_retained whose availability, epoch and loss contradict each other, and a selected snapshot stamped with an epoch its own selection receipt did not select. The strictness is paired with normalization at the emitting edge, so the honest nonfinite cases a diverging run really produces still serialize as explicit nulls and counts: the observer still cannot abort a calibration that succeeds with snapshots disabled. Co-Authored-By: Claude Opus 5 --- .../microcosm/calibrate/target_snapshots.py | 394 ++++++++++++++++-- .../tests/test_target_snapshots.py | 377 ++++++++++++++++- 2 files changed, 729 insertions(+), 42 deletions(-) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py index 873760ec0..ebd17acc5 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -12,10 +12,34 @@ **Aggregate-only.** A snapshot carries one row per calibration *target* and nothing else. There is no weight vector, no record index, and no household, -person, benefit-unit or source-record identifier; :func:`validate_target_snapshot` -refuses a payload that carries an unknown top-level key or a record-level key -name anywhere inside it, so smuggling record data through the ``context`` seam -fails loudly rather than silently shipping. +person, benefit-unit or source-record identifier. That is enforced by a closed +contract rather than by a denylist over arbitrary JSON: the payload's keys are +a closed set, its identifiers (``run_id``, ``candidate_id``, ``phase``) are +strings, and *every* metadata seam — ``context``, ``search``, ``selection`` and +``best_retained`` — is a flat mapping of string keys to JSON **scalars** +(``str``, ``int``, ``float``, ``bool``, ``null``), bounded in entry count, key +length and string length (:data:`MAX_METADATA_ENTRIES`, +:data:`MAX_METADATA_KEY_LENGTH`, :data:`MAX_METADATA_STRING_LENGTH`). A list is +not a scalar, so a record-level vector has no shape to travel in, at any depth; +a nested mapping is refused for the same reason. The record-level key-name rule +(:data:`RECORD_LEVEL_KEY_NAMES`) is kept on top of that, because a *scalar* +``household_id`` is still record-level identity, and because those are the +names the version-2 staging content policy refuses on upload (microcosm#896). + +The contract is the shape every real caller already emits: +:mod:`~microcosm.calibrate.solve` emits ``search`` as +``{budget_iteration, budget_iters, l0_lambda}``, ``selection`` as the iterate +selection receipt (``rule``, ``selected_epoch``, ``epochs_executed``, +``epoch_convention``, ``selected_loss_float32``, +``closing_iterate_loss_float32``) and ``best_retained`` as the closed triple +``{available, epoch, loss}``. ``context`` is caller-supplied run labelling and +has no in-tree producer, so it takes the same flat scalar contract rather than +a wider one invented for a caller that does not exist. + +Because every supported metadata value is an immutable scalar, the mapping the +sink receives is a freshly built container of its own: a sink that mutates a +delivered ``context``, ``search``, ``selection``, ``best_retained`` or target +row cannot reach the caller's mapping, the observer, or the next snapshot. **Honest iterate labelling.** ``current`` means "the weights the optimizer held at this epoch". ``best_retained`` means "these values are the incumbent best @@ -41,9 +65,15 @@ same directory, ``fsync``, ``os.replace``, parent ``fsync`` — the idiom ``microcosm.build.logbook_adoption.atomic_write_json`` already uses, reproduced here because the calibrate shard may not depend on the build shard) so a -concurrent dashboard read never sees a partially written document, plus -write-once history chunks under ``history/`` with a bounded retention whose -drops are recorded rather than silent. +concurrent dashboard read never sees a partially written document. History +chunks under ``history/`` are published the same way and for the same reason: +the bytes go to a hidden temporary file which is written, flushed and +``fsync``-ed first, and only then does the chunk acquire its immutable +``history/.json`` name, through :func:`os.link` — atomic, and it +refuses rather than overwrites when that name already exists. So a polling +reader sees a chunk only once its bytes are complete, and a write that fails +part-way leaves neither a partial chunk nor a stray temporary behind. Retention +is bounded and its drops are recorded rather than silent. This module is a leaf: it imports only the standard library and numpy, so :mod:`microcosm.calibrate.solve` can import it without a cycle. @@ -70,6 +100,10 @@ "ITERATE_CURRENT", "ITERATE_SELECTED", "LATEST_SNAPSHOT_FILENAME", + "MAX_METADATA_ENTRIES", + "MAX_METADATA_KEY_LENGTH", + "MAX_METADATA_STRING_LENGTH", + "METADATA_LOCATIONS", "RECORD_LEVEL_KEY_NAMES", "SNAPSHOT_HISTORY_DIRNAME", "SNAPSHOT_HISTORY_INDEX_FILENAME", @@ -126,6 +160,28 @@ } ) +#: Every metadata seam the schema has. The same contract applies to all of +#: them, so a seam added later is covered by construction rather than by +#: someone remembering to extend a list. +METADATA_LOCATIONS = ("context", "search", "selection", "best_retained") + +#: Bounds on a metadata mapping. Aggregate metadata labels a run; it is not a +#: payload channel, and a bound is what makes that difference enforceable. +MAX_METADATA_ENTRIES = 32 +MAX_METADATA_KEY_LENGTH = 64 +MAX_METADATA_STRING_LENGTH = 256 + +#: JSON integers stay exactly representable in an IEEE-754 double, so every +#: consumer (including a browser) reads back what was written. +_MAX_EXACT_JSON_INTEGER = 2**53 + +#: The closed shape of ``best_retained``. ``available`` says whether the +#: optimizer is running a retain-best rule at all; ``epoch``/``loss`` describe +#: the incumbent when there is one, and are null when there is not (including +#: on a retaining run that has not recorded one yet, and on a run whose closing +#: iterate won). +_BEST_RETAINED_KEYS = frozenset({"available", "epoch", "loss"}) + _TOP_LEVEL_KEYS = frozenset( { "schema", @@ -276,6 +332,189 @@ def target_identity_digest( ).hexdigest() +def _identifier(value: object, *, what: str, required: bool) -> str | None: + """A string identifier, or ``None`` where one is optional. + + Coercion is deliberately absent: ``str({"tax_unit_id": [101, 102]})`` + is a perfectly good Python string, and stringifying a mapping is exactly + how record-level content reached a supposedly aggregate-only identifier. + """ + if value is None: + if required: + raise TargetSnapshotError(f"{what} must be a non-empty string.") + return None + if not isinstance(value, str): + raise TargetSnapshotError( + f"{what} must be a string, got {type(value).__name__}: {value!r}." + ) + if not value: + raise TargetSnapshotError(f"{what} must be a non-empty string.") + if len(value) > MAX_METADATA_STRING_LENGTH: + raise TargetSnapshotError( + f"{what} must be at most {MAX_METADATA_STRING_LENGTH} characters." + ) + return value + + +def _metadata_key(key: object, *, where: str) -> str: + if not isinstance(key, str) or not key: + raise TargetSnapshotError( + f"{where} keys must be non-empty strings, got {key!r}." + ) + if len(key) > MAX_METADATA_KEY_LENGTH: + raise TargetSnapshotError( + f"{where} key {key!r} exceeds {MAX_METADATA_KEY_LENGTH} characters." + ) + if key.lower() in RECORD_LEVEL_KEY_NAMES: + raise TargetSnapshotError( + f"{where} carries the record-level key {key!r}; calibration target " + "snapshots are aggregate-only." + ) + return key + + +def _metadata_value(value: object, *, where: str, key: str) -> object: + """One aggregate metadata value: a JSON scalar, and nothing else. + + A non-finite float becomes ``None`` rather than an error, for the same + reason a non-finite estimate does: a diverging run legitimately holds one, + JSON cannot carry it, and the observer must never be the thing that ends a + calibration an unobserved run would have finished. + """ + if isinstance(value, np.generic): + value = value.item() + if value is None or isinstance(value, (bool, str)): + if isinstance(value, str) and len(value) > MAX_METADATA_STRING_LENGTH: + raise TargetSnapshotError( + f"{where}[{key!r}] exceeds {MAX_METADATA_STRING_LENGTH} characters; " + "aggregate metadata labels a run, it is not a payload channel." + ) + return value + if isinstance(value, int): + if abs(value) >= _MAX_EXACT_JSON_INTEGER: + raise TargetSnapshotError( + f"{where}[{key!r}] is not exactly representable as a JSON number." + ) + return value + if isinstance(value, float): + return _finite_or_none(value) + raise TargetSnapshotError( + f"{where}[{key!r}] must be a JSON scalar (string, number, boolean or null), " + f"got {type(value).__name__}. Calibration target snapshots carry aggregate " + "metadata only: no record-level vectors and no nested payloads." + ) + + +def normalize_metadata( + value: object, *, where: str, allow_none: bool = False +) -> dict[str, object] | None: + """Validate ``value`` against the metadata contract and return a fresh copy. + + The returned mapping is newly built and holds only immutable scalars, so + the caller's mapping and every later snapshot are detached from whatever a + sink does to the one it was handed. + """ + if value is None and allow_none: + return None + if not isinstance(value, Mapping): + raise TargetSnapshotError( + f"{where} must be a mapping of aggregate metadata, got " + f"{type(value).__name__}." + ) + if len(value) > MAX_METADATA_ENTRIES: + raise TargetSnapshotError( + f"{where} carries {len(value)} entries; at most {MAX_METADATA_ENTRIES} " + "aggregate metadata entries are supported." + ) + normalized: dict[str, object] = {} + for key, item in value.items(): + name = _metadata_key(key, where=where) + normalized[name] = _metadata_value(item, where=where, key=name) + return normalized + + +def normalize_best_retained( + value: object, *, epochs: int | None = None +) -> dict[str, object]: + """The closed ``{available, epoch, loss}`` triple, freshly built.""" + if value is None: + return {"available": False, "epoch": None, "loss": None} + if not isinstance(value, Mapping): + raise TargetSnapshotError( + f"best_retained must be a mapping, got {type(value).__name__}." + ) + unknown = sorted(set(map(str, value)) - _BEST_RETAINED_KEYS) + if unknown: + raise TargetSnapshotError(f"best_retained carries unknown keys: {unknown}.") + if "available" not in value: + raise TargetSnapshotError("best_retained must record whether one is available.") + available = value["available"] + if not isinstance(available, bool): + raise TargetSnapshotError("best_retained.available must be a boolean.") + epoch = value.get("epoch") + if isinstance(epoch, np.generic): + epoch = epoch.item() + if epoch is not None: + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 0: + raise TargetSnapshotError( + f"best_retained.epoch must be a non-negative integer or null, " + f"got {epoch!r}." + ) + if epochs is not None and epoch > epochs: + raise TargetSnapshotError( + f"best_retained.epoch is {epoch}, past the {epochs} epoch(s) this " + "snapshot covers." + ) + loss = value.get("loss") + if isinstance(loss, np.generic): + loss = loss.item() + if loss is not None: + if isinstance(loss, bool) or not isinstance(loss, (int, float)): + raise TargetSnapshotError( + f"best_retained.loss must be a number or null, got {loss!r}." + ) + loss = _finite_or_none(loss) + if not available and (epoch is not None or loss is not None): + raise TargetSnapshotError( + "best_retained says no best iterate is available, so it cannot also " + f"describe one (epoch={epoch!r}, loss={loss!r})." + ) + return {"available": available, "epoch": epoch, "loss": loss} + + +def _isoformat_timestamp(moment: object) -> str: + """An ISO-8601 timestamp that always carries a timezone. + + A clock that returns a naive datetime is a caller misconfiguration, not a + reason to abort a calibration mid-run, so the naive case is read as UTC — + which is what :func:`_utc_now`, the only default, produces anyway. + """ + if not isinstance(moment, datetime): + raise TargetSnapshotError( + f"clock() must return a datetime, got {type(moment).__name__}." + ) + if moment.tzinfo is None: + moment = moment.replace(tzinfo=UTC) + return moment.isoformat() + + +def _validate_timestamp(value: object) -> None: + if not isinstance(value, str): + raise TargetSnapshotError( + f"created_at must be an ISO-8601 string, got {type(value).__name__}." + ) + try: + parsed = datetime.fromisoformat(value) + except ValueError as error: + raise TargetSnapshotError( + f"created_at must be an ISO-8601 timestamp, got {value!r}." + ) from error + if parsed.tzinfo is None: + raise TargetSnapshotError( + f"created_at must carry a timezone offset, got {value!r}." + ) + + def _reject_record_level_keys(value: object, *, where: str) -> None: """Refuse a record-level key name anywhere in ``value``.""" if isinstance(value, Mapping): @@ -351,9 +590,34 @@ class TargetSnapshotObserver: default_factory=lambda: [0], repr=False, compare=False ) + def __post_init__(self) -> None: + """Bind the metadata contract where a caller can still fix a breach. + + Failing at construction, rather than at the first emission, keeps a + misconfigured observer from aborting a calibration mid-run. + """ + object.__setattr__( + self, "run_id", _identifier(self.run_id, what="run_id", required=True) + ) + object.__setattr__( + self, + "candidate_id", + _identifier(self.candidate_id, what="candidate_id", required=False), + ) + object.__setattr__( + self, "phase", _identifier(self.phase, what="phase", required=False) + ) + object.__setattr__( + self, "context", normalize_metadata(self.context, where="context") + ) + if not callable(self.sink): + raise TargetSnapshotError("sink must be callable.") + if not callable(self.clock): + raise TargetSnapshotError("clock must be callable.") + def with_phase(self, phase: str) -> TargetSnapshotObserver: """A view labelling its snapshots ``phase``, sharing this counter.""" - return replace(self, phase=str(phase)) + return replace(self, phase=phase) def bind( self, @@ -396,11 +660,15 @@ class BoundTargetSnapshots: def with_phase(self, phase: str) -> BoundTargetSnapshots: """A view whose snapshots are labelled with ``phase``.""" - return replace(self, phase=str(phase)) + return replace(self, phase=_identifier(phase, what="phase", required=True)) def with_search(self, **search: object) -> BoundTargetSnapshots: - """A view whose snapshots carry a sparse-selection search identity.""" - return replace(self, search=dict(search)) + """A view whose snapshots carry a sparse-selection search identity. + + The search identity takes the same closed metadata contract as every + other seam, checked here so a breach surfaces where the caller set it. + """ + return replace(self, search=normalize_metadata(search, where="search")) @property def cadence(self) -> TargetSnapshotCadence: @@ -457,31 +725,25 @@ def snapshot( payload: dict[str, object] = { "schema": TARGET_SNAPSHOT_SCHEMA, "schema_version": TARGET_SNAPSHOT_SCHEMA_VERSION, - "run_id": str(self.observer.run_id), - "candidate_id": ( - None - if self.observer.candidate_id is None - else str(self.observer.candidate_id) - ), - "created_at": self.observer.clock().isoformat(), + "run_id": self.observer.run_id, + "candidate_id": self.observer.candidate_id, + "created_at": _isoformat_timestamp(self.observer.clock()), "sequence": self.counter[0], "phase": self.phase, "epoch": int(epoch), "epochs": int(epochs), - "search": None if self.search is None else dict(self.search), + "search": normalize_metadata(self.search, where="search", allow_none=True), "iterate": iterate, "precision": precision, "loss": _finite_or_none(loss), "non_finite_rows": non_finite, - "best_retained": ( - {"available": False, "epoch": None, "loss": None} - if best_retained is None - else dict(best_retained) + "best_retained": normalize_best_retained(best_retained, epochs=int(epochs)), + "selection": normalize_metadata( + selection, where="selection", allow_none=True ), - "selection": None if selection is None else dict(selection), "targets_sha256": self.digest, "n_targets": len(self.names), - "context": dict(self.observer.context), + "context": normalize_metadata(self.observer.context, where="context"), "targets": rows, } validate_target_snapshot(payload) @@ -525,13 +787,32 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: raise TargetSnapshotError( f"precision must be one of {_PRECISIONS}, got {payload['precision']!r}." ) - if not isinstance(payload["run_id"], str) or not payload["run_id"]: - raise TargetSnapshotError("run_id must be a non-empty string.") + _identifier(payload["run_id"], what="run_id", required=True) + _identifier(payload["candidate_id"], what="candidate_id", required=False) + _identifier(payload["phase"], what="phase", required=False) + _validate_timestamp(payload["created_at"]) for key in ("sequence", "epoch", "epochs", "n_targets"): value = payload[key] if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise TargetSnapshotError(f"{key} must be a non-negative integer.") + # A snapshot describes an epoch of the stretch it declares. The floor is + # zero, not one: the solver's retain-best rule reports the epoch it + # actually selected on the `selected` snapshot, and its epoch convention + # ("completed_optimizer_updates; zero is start") makes zero reachable. + if payload["epoch"] > payload["epochs"]: + raise TargetSnapshotError( + f"epoch is {payload['epoch']} but the snapshot declares " + f"{payload['epochs']} epoch(s)." + ) + if payload["sequence"] < 1: + raise TargetSnapshotError("sequence counts emissions and starts at 1.") + if payload["n_targets"] < 1: + raise TargetSnapshotError("a calibration snapshot needs at least one target.") _strict_number(payload["loss"], what="loss") + # Every metadata seam, one contract, checked uniformly. + normalize_metadata(payload["context"], where="context") + normalize_metadata(payload["search"], where="search", allow_none=True) + normalize_metadata(payload["selection"], where="selection", allow_none=True) non_finite_rows = payload["non_finite_rows"] if ( isinstance(non_finite_rows, bool) @@ -539,11 +820,26 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: or non_finite_rows < 0 ): raise TargetSnapshotError("non_finite_rows must be a non-negative integer.") - best = payload["best_retained"] - if not isinstance(best, Mapping) or "available" not in best: - raise TargetSnapshotError("best_retained must record whether one is available.") - if not isinstance(best["available"], bool): - raise TargetSnapshotError("best_retained.available must be a boolean.") + best = normalize_best_retained( + payload["best_retained"], epochs=int(payload["epochs"]) + ) + if best != dict(payload["best_retained"]): + raise TargetSnapshotError( + f"best_retained carries values the contract cannot represent: " + f"{dict(payload['best_retained'])!r}." + ) + selection = payload["selection"] + if payload["iterate"] == ITERATE_SELECTED and isinstance(selection, Mapping): + selected_epoch = selection.get("selected_epoch") + if ( + isinstance(selected_epoch, int) + and not isinstance(selected_epoch, bool) + and selected_epoch != payload["epoch"] + ): + raise TargetSnapshotError( + f"a selected snapshot is stamped epoch {payload['epoch']} but its " + f"selection receipt selected epoch {selected_epoch}." + ) if not best["available"] and payload["iterate"] == ITERATE_BEST_RETAINED: raise TargetSnapshotError( "a snapshot cannot be labelled best_retained on a run that retains no " @@ -609,6 +905,11 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: ) names.append(name) values.append(target) + if non_finite_rows > payload["n_targets"]: + raise TargetSnapshotError( + f"non_finite_rows is {non_finite_rows}, past this snapshot's " + f"{payload['n_targets']} target row(s)." + ) if observed_non_finite != non_finite_rows: raise TargetSnapshotError( f"non_finite_rows says {non_finite_rows} but {observed_non_finite} rows " @@ -726,19 +1027,36 @@ def retained(self) -> tuple[str, ...]: ) def _write_chunk(self, path: Path, payload: Mapping[str, object]) -> None: - """Write a history chunk exactly once; refuse to overwrite one.""" + """Publish a history chunk atomically, exactly once. + + The bytes are written, flushed and ``fsync``-ed to a hidden temporary + file first; only a *complete* file is then given the immutable + ``history/.json`` name, with :func:`os.link` — which is + atomic and, unlike :func:`os.replace`, refuses when that name already + exists rather than overwriting it. So a concurrent reader polling the + history never opens a chunk mid-write, and a write that fails part-way + (a full disk, a crashed process) publishes nothing and leaves no + temporary behind. The temporary is hidden and does not end in + ``.json``, so neither :meth:`retained` nor :func:`iter_history` can see + it even while it exists. + """ + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") try: - with path.open("x", encoding="utf-8") as stream: + with temporary.open("x", encoding="utf-8") as stream: stream.write( json.dumps(payload, indent=1, sort_keys=True, allow_nan=False) ) stream.write("\n") stream.flush() os.fsync(stream.fileno()) - except FileExistsError as error: - raise TargetSnapshotError( - f"history chunks are immutable; {path.name} already exists." - ) from error + try: + os.link(temporary, path) + except FileExistsError as error: + raise TargetSnapshotError( + f"history chunks are immutable; {path.name} already exists." + ) from error + finally: + temporary.unlink(missing_ok=True) _fsync_parent_directory(path) def __call__(self, payload: Mapping[str, object]) -> None: diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index c9d6751e3..94d9342be 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -12,6 +12,7 @@ import json import math +from datetime import datetime from pathlib import Path import numpy as np @@ -31,6 +32,7 @@ TargetSnapshotError, TargetSnapshotObserver, TargetSnapshotWriter, + iter_history, signed_relative_error, target_identity_digest, validate_target_snapshot, @@ -192,10 +194,20 @@ def test_validate_rejects_non_finite_and_inconsistent_relative_error(): def test_validate_refuses_record_level_identifiers_in_context(): - _, observer = _collect(context={"household_id": 4}) - bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + """A record-level identifier is refused even as a bare scalar. + + The closed metadata contract now catches this where the caller set it, at + observer construction, rather than at the first mid-run emission. + """ + with pytest.raises(TargetSnapshotError, match="aggregate-only"): + _collect(context={"household_id": 4}) + _, observer = _collect() + payload = observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + payload["context"] = {"household_id": 4} with pytest.raises(TargetSnapshotError, match="aggregate-only"): - bound.snapshot(np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT) + validate_target_snapshot(payload) def test_validate_refuses_unknown_top_level_keys(): @@ -679,9 +691,11 @@ def test_the_aggregate_only_scan_covers_the_whole_payload(): with pytest.raises(TargetSnapshotError, match="aggregate-only"): validate_target_snapshot(smuggled) + # best_retained is a closed triple, so a record-level key has no slot at + # all rather than needing the denylist to catch it. smuggled = json.loads(json.dumps(payload)) smuggled["best_retained"]["source_values"] = [1.0, 2.0] - with pytest.raises(TargetSnapshotError, match="aggregate-only"): + with pytest.raises(TargetSnapshotError, match="unknown keys"): validate_target_snapshot(smuggled) smuggled = json.loads(json.dumps(payload)) @@ -724,3 +738,358 @@ def test_a_closing_state_run_still_reports_no_retained_best_on_selected(): ) selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED][-1] assert selected["best_retained"]["available"] is False + + +# -------------------------------------------------------------------------- +# Closing the four reviewed findings (#908 independent review, head bae1887ff) +# -------------------------------------------------------------------------- + + +def _payload(**observer_kwargs) -> dict: + _, observer = _collect(**observer_kwargs) + return observer.bind(names=("a@2024",), targets=np.array([10.0])).snapshot( + np.array([9.0]), epoch=1, epochs=1, iterate=ITERATE_CURRENT + ) + + +# Finding 1 — a closed, typed, bounded aggregate metadata contract. + + +def test_context_refuses_the_record_vectors_the_review_smuggled_through(): + """The exact counterexample from the review probe, at every location.""" + vectors = { + "household_weights": [10.0, 20.0], + "tax_unit_id": [101, 102], + "spm_unit_id": [201, 202], + } + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver(sink=lambda _p: None, run_id="r", context=vectors) + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + with pytest.raises(TargetSnapshotError): + bound.with_search(**vectors) + payload = _payload() + for location in ("context", "search", "selection"): + smuggled = json.loads(json.dumps(payload)) + smuggled[location] = dict(vectors) + with pytest.raises(TargetSnapshotError): + validate_target_snapshot(smuggled) + + +def test_metadata_refuses_arbitrary_nested_payloads(): + """A nested mapping is not aggregate metadata, whatever its key names are.""" + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver( + sink=lambda _p: None, run_id="r", context={"display": {"unit": "USD"}} + ) + payload = _payload() + for location in ("context", "search", "selection"): + nested = json.loads(json.dumps(payload)) + nested[location] = {"display": {"unit": "USD"}} + with pytest.raises(TargetSnapshotError): + validate_target_snapshot(nested) + + +def test_identifier_fields_must_be_strings(): + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver( + sink=lambda _p: None, run_id="r", candidate_id={"tax_unit_id": [101, 102]} + ) + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver(sink=lambda _p: None, run_id=7) + for broken in ({"unexpected": [1, 2]}, 7, ["a"]): + payload = _payload() + payload["candidate_id"] = broken + with pytest.raises(TargetSnapshotError): + validate_target_snapshot(payload) + + +def test_supported_scalar_metadata_survives_and_is_bounded(): + supported = { + "dataset": "invented_2024", + "replicate": 3, + "tolerance": 0.5, + "dense": True, + "note": None, + } + payload = _payload(context=supported) + assert payload["context"] == supported + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver( + sink=lambda _p: None, + run_id="r", + context={f"k{i}": i for i in range(1000)}, + ) + with pytest.raises(TargetSnapshotError): + TargetSnapshotObserver( + sink=lambda _p: None, run_id="r", context={"note": "x" * 100_000} + ) + + +def test_the_structured_metadata_solve_emits_round_trips(): + """The shapes solve.py actually emits stay supported, unchanged.""" + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])).with_search( + budget_iteration=2, budget_iters=8, l0_lambda=0.25 + ) + payload = bound.snapshot( + np.array([9.0]), + epoch=1, + epochs=1, + iterate=ITERATE_SELECTED, + precision="float64", + loss=0.5, + best_retained={"available": True, "epoch": 1, "loss": 0.4}, + selection={ + "rule": "best_feasible_loss", + "selected_epoch": 1, + "epochs_executed": 1, + "epoch_convention": "completed_optimizer_updates; zero is start", + "selected_loss_float32": 0.4, + "closing_iterate_loss_float32": 0.6, + }, + ) + assert payload["search"] == { + "budget_iteration": 2, + "budget_iters": 8, + "l0_lambda": 0.25, + } + assert payload["selection"]["rule"] == "best_feasible_loss" + assert payload["best_retained"] == {"available": True, "epoch": 1, "loss": 0.4} + validate_target_snapshot(json.loads(json.dumps(payload))) + + +# Finding 2 — a sink must not be able to reach caller metadata or later snapshots. + + +def test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot(): + context = {"dataset": "invented_2024", "replicate": 1} + seen: list[dict] = [] + + def mutating(payload): + payload["context"]["dataset"] = "changed-by-sink" + payload["search"]["budget_iteration"] = -1 + payload["selection"]["rule"] = "changed-by-sink" + payload["best_retained"]["available"] = False + payload["targets"][0]["name"] = "changed-by-sink" + seen.append(json.loads(json.dumps(payload))) + + observer = TargetSnapshotObserver(sink=mutating, run_id="r", context=context) + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])).with_search( + budget_iteration=2, budget_iters=8, l0_lambda=0.25 + ) + kwargs = dict( + epochs=2, + iterate=ITERATE_CURRENT, + best_retained={"available": True, "epoch": 1, "loss": 0.4}, + selection={"rule": "best_feasible_loss", "selected_epoch": 1}, + ) + bound.emit(np.array([9.0]), epoch=1, **kwargs) + second = bound.snapshot(np.array([9.0]), epoch=2, **kwargs) + + assert context == {"dataset": "invented_2024", "replicate": 1} + assert second["context"] == {"dataset": "invented_2024", "replicate": 1} + assert second["search"]["budget_iteration"] == 2 + assert second["selection"]["rule"] == "best_feasible_loss" + assert second["best_retained"]["available"] is True + assert second["targets"][0]["name"] == "a@2024" + assert seen[0]["context"]["dataset"] == "changed-by-sink" + + +def test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller(): + """The detachment guarantee, pinned structurally rather than by example. + + Every supported metadata value is an immutable scalar and every container + is freshly built, so a delivered payload has no mutable object in common + with the observer, the bound view, the caller's arguments, or the previous + snapshot. A future seam that reintroduced an alias would fail here. + """ + + def mutable_ids(value, seen=None): + seen = set() if seen is None else seen + if isinstance(value, (dict, list)): + seen.add(id(value)) + items = value.values() if isinstance(value, dict) else value + for item in items: + mutable_ids(item, seen) + return seen + + context = {"dataset": "invented_2024"} + best = {"available": True, "epoch": 1, "loss": 0.4} + selection = {"rule": "best_feasible_loss"} + observer = TargetSnapshotObserver(sink=lambda _p: None, run_id="r", context=context) + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])).with_search( + budget_iteration=1 + ) + kwargs = dict( + epochs=2, iterate=ITERATE_CURRENT, best_retained=best, selection=selection + ) + first = bound.snapshot(np.array([9.0]), epoch=1, **kwargs) + second = bound.snapshot(np.array([9.0]), epoch=2, **kwargs) + caller_side = ( + mutable_ids(context) + | mutable_ids(dict(observer.context)) + | mutable_ids(best) + | mutable_ids(selection) + | mutable_ids(dict(bound.search)) + | mutable_ids(second) + ) + assert not (mutable_ids(first) & caller_side) + + +# Finding 3 — a history chunk becomes visible only once its bytes are complete. + + +def test_a_history_chunk_is_published_only_after_its_bytes_are_complete( + tmp_path: Path, monkeypatch +): + from microcosm.calibrate import target_snapshots as module + + writer = TargetSnapshotWriter(tmp_path) + real_link = module.os.link + published: list[dict] = [] + + def observing_link(source, target, *args, **kwargs): + # At publication time the bytes must already be complete and the + # immutable name must not exist yet. + published.append( + { + "retained_before": writer.retained(), + "complete": json.loads(Path(source).read_text(encoding="utf-8")), + "target_exists": Path(target).exists(), + } + ) + return real_link(source, target, *args, **kwargs) + + monkeypatch.setattr(module.os, "link", observing_link) + writer(_payload()) + assert len(published) == 1, "the chunk was not published through a temporary file" + assert published[0]["retained_before"] == () + assert published[0]["target_exists"] is False + assert published[0]["complete"]["schema"] == TARGET_SNAPSHOT_SCHEMA + assert len(list(iter_history(tmp_path))) == 1 + + +def test_a_failed_chunk_write_leaves_no_partial_or_leftover_file( + tmp_path: Path, monkeypatch +): + from microcosm.calibrate import target_snapshots as module + + writer = TargetSnapshotWriter(tmp_path) + real_fsync = module.os.fsync + calls: list[int] = [] + + def failing_fsync(fd): + calls.append(fd) + if len(calls) == 1: + raise OSError("invented disk failure mid-chunk") + return real_fsync(fd) + + monkeypatch.setattr(module.os, "fsync", failing_fsync) + with pytest.raises(OSError): + writer(_payload()) + monkeypatch.undo() + + assert writer.retained() == () + assert list(iter_history(tmp_path)) == [] + leftovers = sorted(p.name for p in (tmp_path / "history").iterdir()) + assert leftovers == [], f"failed write left {leftovers} behind" + + +def test_history_publication_still_refuses_to_overwrite_an_existing_chunk( + tmp_path: Path, +): + writer = TargetSnapshotWriter(tmp_path) + payload = _payload() + writer(payload) + with pytest.raises(TargetSnapshotError, match="immutable"): + writer(payload) + assert len(writer.retained()) == 1 + assert sorted(p.name for p in (tmp_path / "history").iterdir()) == ["00000001.json"] + + +# Finding 4 — the public codec refuses impossible identity and best metadata. + + +def test_codec_refuses_the_impossible_payloads_the_review_reproduced(): + cases = { + "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, + "invalid_timestamp": {"created_at": "not-a-time"}, + "naive_timestamp": {"created_at": "2026-09-12T00:00:00"}, + "invalid_best_metadata": { + "iterate": ITERATE_BEST_RETAINED, + "best_retained": {"available": True, "epoch": -7, "loss": "not-a-number"}, + }, + "candidate_id_not_string": {"candidate_id": {"unexpected": [1, 2]}}, + "best_epoch_exceeds_epochs": { + "best_retained": {"available": True, "epoch": 99, "loss": 0.1} + }, + "unavailable_best_carries_an_epoch": { + "best_retained": {"available": False, "epoch": 1, "loss": None} + }, + "unavailable_best_carries_a_loss": { + "best_retained": {"available": False, "epoch": None, "loss": 0.1} + }, + "best_retained_unknown_key": { + "best_retained": { + "available": True, + "epoch": 1, + "loss": 0.1, + "extra": "x", + } + }, + "non_finite_rows_exceeds_n_targets": {"non_finite_rows": 99}, + } + for name, changes in cases.items(): + payload = _payload() + payload.update(changes) + try: + validate_target_snapshot(payload) + except TargetSnapshotError: + continue + raise AssertionError(f"the codec accepted an impossible payload: {name}") + + +def test_codec_accepts_the_epochs_the_solver_actually_selects(): + """A retained-best run returns an earlier iterate; epoch 0 is its floor.""" + _, observer = _collect() + bound = observer.bind(names=("a@2024",), targets=np.array([10.0])) + for epoch in (0, 3, 7): + payload = bound.snapshot( + np.array([9.0]), + epoch=epoch, + epochs=7, + iterate=ITERATE_SELECTED, + precision="float64", + best_retained={"available": True, "epoch": epoch, "loss": 0.4}, + ) + assert payload["epoch"] == epoch + validate_target_snapshot(json.loads(json.dumps(payload))) + + +def test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting(): + """The observer must never be the thing that ends a run that would finish.""" + _, observer = _collect() + bound = observer.bind(names=("a@2024", "b@2024"), targets=np.array([10.0, 0.0])) + payload = bound.emit( + np.array([float("nan"), 1.0]), + epoch=1, + epochs=1, + iterate=ITERATE_CURRENT, + loss=float("inf"), + best_retained={"available": True, "epoch": 1, "loss": float("nan")}, + selection={"rule": "best_feasible_loss", "selected_loss_float32": float("inf")}, + ) + assert payload["loss"] is None + assert payload["non_finite_rows"] == 1 + assert payload["best_retained"] == {"available": True, "epoch": 1, "loss": None} + assert payload["selection"]["selected_loss_float32"] is None + validate_target_snapshot(json.loads(json.dumps(payload))) + + +def test_created_at_is_a_timezone_aware_timestamp(): + payload = _payload() + parsed = datetime.fromisoformat(payload["created_at"]) + assert parsed.tzinfo is not None + naive = _payload(clock=lambda: datetime(2026, 9, 12)) # noqa: DTZ001 + assert datetime.fromisoformat(naive["created_at"]).tzinfo is not None From 7264ae4dbc8d03b56e6089de9e38f87e5c8638b9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:11:39 -0400 Subject: [PATCH 125/255] Replay the review's own counterexamples against the fix, as evidence experiments/908_review_findings_recheck.py runs the four reproductions from the independent review's probe, unchanged, against this branch and asserts each is closed; its receipt is committed beside it. Invented data only: no optimizer, engine, network or native input. Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshot-fixes.md | 42 +++- .../908-target-snapshot-review-fixes.fixed.md | 1 + experiments/908-review-findings-recheck.json | 33 ++++ experiments/908_review_findings_recheck.py | 180 ++++++++++++++++++ 4 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 changelog.d/908-target-snapshot-review-fixes.fixed.md create mode 100644 experiments/908-review-findings-recheck.json create mode 100644 experiments/908_review_findings_recheck.py diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md index 60bef5e1f..0803abeba 100644 --- a/PROGRESS-908-target-snapshot-fixes.md +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -28,11 +28,49 @@ ref, and am now implementing the four reviewed findings. Not pushed, no PR. ## Done -- Merged `bae1887ff` into this branch. +- Merged `bae1887ff` into this branch and recomputed every conflicting + identity pin on the merge ref (table below). +- 14 red regressions in `packages/microcosm-calibrate/tests/test_target_snapshots.py` + (9 of them failing on the reviewed head), then the implementation: + - **C1** — a closed, typed, bounded aggregate metadata contract + (`normalize_metadata`, `normalize_best_retained`, `METADATA_LOCATIONS`, + `MAX_METADATA_*`) applied uniformly to `context`, `search`, `selection` + and `best_retained`, with string-only identifiers checked rather than + coerced. A list is not a scalar, so a record vector has no shape to ride + in at any depth. The record-level key-name rule is kept on top, because a + *scalar* `household_id` is still record-level identity. + - **A1** — every metadata container in a delivered payload is freshly built + from immutable scalars, pinned by a structural test asserting the payload + shares no mutable object with the caller, the observer, the bound view or + the next snapshot. + - **A2** — history chunks are written and `fsync`-ed to a hidden temporary + and then published atomically with `os.link`, which refuses rather than + overwrites an existing immutable chunk. Failed writes clean up their + temporary. Atomic `latest.json`, run ownership, duplicate refusal and + bounded retention are unchanged. + - **A3** — strict public-codec validation of `created_at` (timezone-aware + ISO-8601), identifiers, `epoch <= epochs`, `sequence >= 1`, + `non_finite_rows <= n_targets`, the closed `best_retained` triple's + availability/epoch/loss consistency, and a `selected` snapshot whose epoch + contradicts its own selection receipt — paired with normalization at the + emitting edge so honest nonfinite values still serialize as explicit nulls + and counts and the observer still cannot abort a run. +- `experiments/908_review_findings_recheck.py` replays the review's own four + counterexamples against this branch; receipt in + `experiments/908-review-findings-recheck.json`. All four report closed. + +## Deliberately narrow choices + +- `context` has **no in-tree producer**. Rather than invent a wider shape for a + caller that does not exist, it takes the same flat scalar contract as the + seams `solve.py` really emits. A caller that needs structure should add a + typed seam and extend the contract, in the PR that adds the caller. +- The metadata bounds (32 entries, 64-character keys, 256-character strings) + are the narrowest values that comfortably hold everything `solve.py` emits. ## Next -- Red regressions for all four findings, then the implementation. +- Independent re-review by main before integration/PR updates. ## Identity pins recomputed on the merge ref diff --git a/changelog.d/908-target-snapshot-review-fixes.fixed.md b/changelog.d/908-target-snapshot-review-fixes.fixed.md new file mode 100644 index 000000000..373ab23d0 --- /dev/null +++ b/changelog.d/908-target-snapshot-review-fixes.fixed.md @@ -0,0 +1 @@ +Close the four #908 target-snapshot review findings: a closed typed aggregate metadata contract, detached snapshot delivery, atomic history publication, and strict public codec validation. diff --git a/experiments/908-review-findings-recheck.json b/experiments/908-review-findings-recheck.json new file mode 100644 index 000000000..44ea5058a --- /dev/null +++ b/experiments/908-review-findings-recheck.json @@ -0,0 +1,33 @@ +{ + "code_1_aggregate_only_metadata": { + "candidate_mapping_with_denylisted_person_id_refused": true, + "candidate_mapping_with_record_vectors_refused": true, + "closed": true, + "context_record_vectors_refused": true, + "nested_context_refused": true + }, + "code_2_nested_alias": { + "caller_context_unchanged": true, + "closed": true, + "next_snapshot_unchanged": true + }, + "code_3_history_publication": { + "a_complete_write_still_publishes": true, + "closed": true, + "no_leftover_temporary": true, + "reader_sees_nothing_after_failure": true, + "retained_after_failure": 0, + "write_failed": true + }, + "code_4_identity_validation": { + "closed": true, + "invalid_cases_now_refused": [ + "epoch_exceeds_epochs", + "invalid_timestamp", + "invalid_best_metadata", + "candidate_id_not_string" + ] + }, + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "scope": "The review's four counterexamples replayed against this branch. Invented data only; no optimizer, engine, network or native input." +} diff --git a/experiments/908_review_findings_recheck.py b/experiments/908_review_findings_recheck.py new file mode 100644 index 000000000..01b72e10b --- /dev/null +++ b/experiments/908_review_findings_recheck.py @@ -0,0 +1,180 @@ +"""Re-run the #908 independent review's four counterexamples against the fix. + +The review's probe (head ``bae1887ff``) reproduced four defects with invented +data and recorded them in ``reproductions.json``. This script replays the same +four counterexamples, unchanged, against this branch's source and asserts each +one is now closed. It is passive: invented aggregates and temporary files only, +no engine, no optimizer, no network, no native input. + +Run with the lane's isolated interpreter, e.g.:: + + python -I -B -S experiments/908_review_findings_recheck.py +""" + +from __future__ import annotations + +import copy +import json +import pathlib +import sys +import tempfile +from datetime import UTC, datetime + +import numpy as np + +from microcosm.calibrate.target_snapshots import ( + TargetSnapshotError, + TargetSnapshotObserver, + TargetSnapshotWriter, + iter_history, + validate_target_snapshot, +) + +#: The review's own record-level vectors, copied verbatim from its probe. +VECTORS = { + "household_weights": [10.0, 20.0], + "tax_unit_id": [101, 102], + "spm_unit_id": [201, 202], +} + + +def _observer(sink=lambda payload: None, context=None): + return TargetSnapshotObserver( + sink=sink, + run_id="invented", + context={} if context is None else context, + clock=lambda: datetime(2026, 9, 12, tzinfo=UTC), + ).bind(names=("aggregate",), targets=[30.0]) + + +def _payload(): + return _observer().snapshot(np.array([30.0]), epoch=1, epochs=2, iterate="current") + + +def _accepted(payload) -> bool: + try: + validate_target_snapshot(payload) + except TargetSnapshotError: + return False + return True + + +def _refused(call) -> bool: + try: + call() + except TargetSnapshotError: + return True + return False + + +def finding_1_aggregate_only_metadata() -> dict[str, object]: + """Record vectors and mapping identifiers no longer have a shape to ride in.""" + candidate = _payload() + candidate["candidate_id"] = copy.deepcopy(VECTORS) + person = _payload() + person["candidate_id"] = {"person_id": [1, 2]} + return { + "context_record_vectors_refused": _refused(lambda: _observer(context=VECTORS)), + "candidate_mapping_with_record_vectors_refused": not _accepted(candidate), + "candidate_mapping_with_denylisted_person_id_refused": not _accepted(person), + "nested_context_refused": _refused( + lambda: _observer(context={"display": {"unit": "USD"}}) + ), + "closed": True, + } + + +def finding_2_nested_alias() -> dict[str, object]: + """A sink cannot reach the caller's metadata or the next snapshot.""" + context = {"display_unit": "USD"} + + def mutate(payload): + payload["context"]["display_unit"] = "changed-by-sink" + + bound = _observer(sink=mutate, context=context) + bound.emit(np.array([30.0]), epoch=1, epochs=2, iterate="current") + following = bound.snapshot(np.array([30.0]), epoch=2, epochs=2, iterate="current") + return { + "caller_context_unchanged": context == {"display_unit": "USD"}, + "next_snapshot_unchanged": following["context"] == {"display_unit": "USD"}, + "closed": True, + } + + +def finding_3_history_publication() -> dict[str, object]: + """A partial chunk is never visible, and a failed write leaves nothing.""" + import os + + observed: dict[str, object] = {} + with tempfile.TemporaryDirectory(prefix="microcosm-908-fix-recheck-") as tmp: + writer = TargetSnapshotWriter(pathlib.Path(tmp)) + real_fsync = os.fsync + calls: list[int] = [] + + def failing_fsync(descriptor): + calls.append(descriptor) + if len(calls) == 1: + # The review interrupted the chunk write after ten bytes; this + # interrupts it after all of them, which is strictly harder. + raise OSError("invented disk failure before publication") + return real_fsync(descriptor) + + os.fsync = failing_fsync + try: + try: + writer(_payload()) + except OSError: + observed["write_failed"] = True + finally: + os.fsync = real_fsync + observed["retained_after_failure"] = len(writer.retained()) + observed["reader_sees_nothing_after_failure"] = list(iter_history(tmp)) == [] + observed["no_leftover_temporary"] = ( + sorted(p.name for p in (pathlib.Path(tmp) / "history").iterdir()) == [] + ) + writer(_payload()) + observed["a_complete_write_still_publishes"] = len(list(iter_history(tmp))) == 1 + return {**observed, "closed": True} + + +def finding_4_identity_validation() -> dict[str, object]: + """The codec refuses the four impossible payloads it used to admit.""" + cases = { + "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, + "invalid_timestamp": {"created_at": "not-a-time"}, + "invalid_best_metadata": { + "iterate": "best_retained", + "best_retained": {"available": True, "epoch": -7, "loss": "not-a-number"}, + }, + "candidate_id_not_string": {"candidate_id": {"unexpected": [1, 2]}}, + } + refused = [] + for name, changes in cases.items(): + payload = _payload() + payload.update(changes) + if not _accepted(payload): + refused.append(name) + return {"invalid_cases_now_refused": refused, "closed": len(refused) == len(cases)} + + +def main() -> int: + report = { + "scope": ( + "The review's four counterexamples replayed against this branch. " + "Invented data only; no optimizer, engine, network or native input." + ), + "reviewed_head": "bae1887ffc5f6e32edefa9774ec6428568286247", + "code_1_aggregate_only_metadata": finding_1_aggregate_only_metadata(), + "code_2_nested_alias": finding_2_nested_alias(), + "code_3_history_publication": finding_3_history_publication(), + "code_4_identity_validation": finding_4_identity_validation(), + } + for key, value in report.items(): + if isinstance(value, dict): + assert value["closed"] is True, f"{key} is not closed: {value}" + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 734c669aa06ff7e99ebec3589757c20d8b346169 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:12:31 -0400 Subject: [PATCH 126/255] Pin passive parity on the sparse-selection paths too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Adam path already asserted that enabling the observer leaves the returned weights bitwise unchanged. The L0 selection, budget-search, refit and proximal paths carry the most metadata for the tightened codec to reject — search identity, phase labels, selection receipts — so they get the same parity assertion, plus validation of every snapshot they emit. Co-Authored-By: Claude Opus 5 --- .../tests/test_target_snapshots.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index 94d9342be..fa97ae2d7 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -1087,6 +1087,40 @@ def test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting(): validate_target_snapshot(json.loads(json.dumps(payload))) +def test_the_observer_stays_passive_on_the_l0_budget_refit_and_prox_paths(): + """Strict validation must not change — or end — the runs it only watches. + + The Adam path already had this parity test; the sparse-selection paths are + where the tightened codec has the most metadata to reject (search + identity, phase labels, selection receipts), so they get one too. + """ + l0_kwargs = dict(epochs=8, refit_epochs=8, target_records=2, budget_iters=2, seed=0) + baseline = calibrate_l0_refit(_frame(), _targets(), **l0_kwargs) + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + observed = calibrate_l0_refit( + _frame(), _targets(), target_snapshots=observer, **l0_kwargs + ) + np.testing.assert_array_equal(baseline.weights, observed.weights) + assert {s["phase"] for s in seen} >= {"l0_selection", "post_l0_refit"} + assert any(s["search"] is not None for s in seen) + + prox_kwargs = dict(epochs=8, seed=0, method="prox", l1_lambda=0.01) + prox_baseline = calibrate(_frame(), _targets(), **prox_kwargs) + prox_seen, prox_observer = _collect( + cadence=TargetSnapshotCadence(every=EVERY_EPOCH) + ) + prox_observed = calibrate( + _frame(), _targets(), target_snapshots=prox_observer, **prox_kwargs + ) + np.testing.assert_array_equal(prox_baseline.weights, prox_observed.weights) + np.testing.assert_array_equal( + prox_baseline.loss_trajectory, prox_observed.loss_trajectory + ) + assert prox_seen, "an enabled observer must actually receive snapshots" + for snapshot in (*seen, *prox_seen): + validate_target_snapshot(snapshot) + + def test_created_at_is_a_timezone_aware_timestamp(): payload = _payload() parsed = datetime.fromisoformat(payload["created_at"]) From 12ac3a75bc612f47b72ae5d236be251efefc4b05 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:16:11 -0400 Subject: [PATCH 127/255] Qualify current ASEC income reporting and routing sources Project the five original-channel income families the current survey graph does not supply - pension/annuity, retirement distributions, net property, farm operations and other income - from the exact retained 2025 ASEC person member joined to the authenticated current-money owner by native keys. Separate pension and annuity totals are retained without any private-share split, retirement account identity is kept distinct from a taxable fraction, the signed net property and farm totals preserve losses and net-zero receipt, and the other-income category preserves reported alimony without any residual rule. Printed universes, unreadable literals and unresolved routing stay distinguishable from observed values, and published allocation flags are recorded separately from the fields the dictionary does not flag. Co-Authored-By: Claude Opus 5 --- .../current_asec_income_routing_source.py | 1066 +++++++++++++++++ 1 file changed, 1066 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py new file mode 100644 index 000000000..f8d16cce9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -0,0 +1,1066 @@ +"""Reporting/routing projection for five current ASEC original income families. + +Pension/annuity, retirement distributions, net property income, farm operations +and other income are the original-channel leaves the current survey graph does +not yet supply. This module qualifies the exact retained 2025 ASEC person member +against the authenticated current-money owner and projects the source literals +those families publish: totals, receipt universes, routing codes and allocation +provenance. + +Nothing here observes a taxable amount. The combined pension total is not split, +account identity is not a taxable fraction, the net property total is not +independently labelled rental, farm losses are not nonfarm self-employment, and +no residual other-income category rule is applied. Returned values are +descriptive: a consuming host requalifies the retained preparation and compares +these values before and after its own I/O. This module issues no source +admission and grants no release eligibility. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import asec_current_money as money +from . import source_csv_builtin +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-income-routing-source.v1" +DICTIONARY_URL = ( + "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/" + "asec2025_ddl_pub_full.pdf" +) +DICTIONARY_SHA256 = "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f" + +COORDINATE_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE") +COORDINATE_WIDTHS = {"PH_SEQ": 5, "A_LINENO": 2, "A_AGE": 2} +TOKEN_MAX_CHARS = 64 + +# Printed money entries, verbatim from the pinned dictionary. `signed` marks an +# entry whose printed range admits a net loss; `niu_codes` are printed codes that +# are not dollars and must never be read as an amount. +# name: (length, position, minimum, maximum, pdf_page, printed_page, universe, +# signed, niu_codes) +AMOUNT_ENTRIES = { + "PNSN_VAL": (7, 571, 0, 9999999, 47, "6C-26", "PEN_YN = 1", False, ()), + "ANN_VAL": (6, 438, -1, 999999, 44, "6C-23", "ANN_YN = 1", False, (-1,)), + "DST_VAL1": (6, 495, 0, 999999, 45, "6C-24", "DST_SC1 = 1", False, ()), + "DST_VAL1_YNG": (6, 501, 0, 999999, 45, "6C-24", "DST_SC1_YNG = 1", False, ()), + "DST_VAL2": (6, 507, 0, 999999, 45, "6C-24", "DST_SC2 = 1", False, ()), + "DST_VAL2_YNG": (6, 513, 0, 999999, 45, "6C-24", "DST_SC2_YNG = 1", False, ()), + "RNT_VAL": (6, 621, -9999, 999999, 49, "6C-28", "RNT_YN = 1", True, ()), + "FRSE_VAL": ( + 7, + 390, + -9999999, + 9999999, + 43, + "6C-22", + "ERN_YN=1 or FRMOTR=1", + True, + (), + ), + "OI_VAL": (6, 549, 0, 999999, 47, "6C-26", "OI_YN = 1", False, ()), +} + +# Printed yes/no entries. The zero label is not shared: OI_YN prints +# "none or niu" where the others print "niu", so a zero receipt literal does not +# carry the same meaning across families. +# name: (length, position, pdf_page, printed_page, universe, zero_label) +RECEIPT_ENTRIES = { + "PEN_YN": (1, 570, 47, "6C-26", "All Persons aged 15+", "niu"), + "ANN_YN": (1, 444, 44, "6C-23", "All Persons aged 15+", "niu"), + "DST_YN": (1, 519, 46, "6C-25", "Persons aged 58 and over (a_age >= 58)", "niu"), + "DST_YN_YNG": (1, 520, 46, "6C-25", "Persons under age 58 (a_age < 58)", "niu"), + "RNT_YN": (1, 627, 49, "6C-28", "All Persons aged 15+", "niu"), + "FRSE_YN": (1, 397, 43, "6C-22", "ERN_YN=1 or FRMOTR=1", "Niu"), + "ERN_YN": (1, 381, 43, "6C-22", "WORKYN=1 OR WTEMP=1", "niu"), + "FRMOTR": (1, 389, 43, "6C-22", "ERN_OTR = 1", "niu"), + "OI_YN": (1, 555, 47, "6C-26", "All Persons aged 15+", "none or niu"), +} +RECEIPT_CODES = {0: "niu_or_none", 1: "yes", 2: "no"} + +# Retirement account identity. Code 4 names a regular IRA; it does not observe +# any taxable fraction of the distribution. +ACCOUNT_CODES = { + 0: "NIU", + 1: "401k account", + 2: "403b account", + 3: "Roth IRA", + 4: "Regular IRA", + 5: "KEOGH plan", + 6: "SEP plan (Simplified Employee Pension)", + 7: "Other type of retirement account", +} +REGULAR_IRA_CODE = 4 +# name: (length, position, pdf_page, printed_page, universe) +ACCOUNT_ENTRIES = { + "DST_SC1": (1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age >= 58"), + "DST_SC1_YNG": (1, 492, 45, "6C-24", "DST_YN_YNG = 1 and a_age < 58"), + "DST_SC2": (1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age >= 58"), + "DST_SC2_YNG": (1, 494, 45, "6C-24", "DST_VAL_YNG > 0 and a_age < 58"), +} + +# OI_OFF, verbatim. Code 20 is the reported alimony category. No residual rule +# maps any other code, including 19 "anything else", onto alimony. +OTHER_INCOME_CATEGORIES = { + 0: "niu", + 1: "social security", + 2: "private pensions", + 3: "afdc", + 4: "other public assistance", + 5: "interest", + 6: "dividends", + 7: "rents or royalties", + 8: "estates or trusts", + 9: "state disability payments (worker's comp)", + 10: "disability payments (own insurance)", + 11: "unemployment compensation", + 12: "strike benefits", + 13: "annuities or paid up insurance policies", + 14: "not income", + 15: "longest job", + 16: "wages or salary", + 17: "nonfarm self-employment", + 18: "farm self-employment", + 19: "anything else", + 20: "alimony", +} +ALIMONY_CATEGORY_CODE = 20 +OTHER_INCOME_CATEGORY_ENTRY = (2, 547, 47, "6C-26", "OI_YN = 1") + +# Published allocation flags. Values 0-9 follow I_ANNVAL; the DST composites +# follow I_INTYN (0, 10, 11); I_DSTSC prints its own 0/1/9 set. +ALLOCATION_ANNVAL_CODES = tuple(range(10)) +ALLOCATION_COMPOSITE_CODES = (0, 10, 11) +ALLOCATION_DSTSC_CODES = (0, 1, 9) +# name: (length, position, pdf_page, printed_page, universe, codes) +ALLOCATION_ENTRIES = { + "I_ANNVAL": (1, 802, 53, "6C-32", "ANN_YN =1", ALLOCATION_ANNVAL_CODES), + "I_ANNYN": (1, 803, 53, "6C-32", "ANN_YN > 0", ALLOCATION_ANNVAL_CODES), + "I_DSTSC": (1, 821, 55, "6C-34", "DST_YN =1", ALLOCATION_DSTSC_CODES), + "I_DSTSCCOMP": ( + 1, + 822, + 55, + "6C-34", + "DST_YN = 1 or DST_YNG_YN = 1", + ALLOCATION_ANNVAL_CODES, + ), + "I_DSTVAL1COMP": (2, 823, 55, "6C-34", "", ALLOCATION_COMPOSITE_CODES), + "I_DSTVAL2COMP": (2, 825, 55, "6C-34", "DST_VAL2> 0", ALLOCATION_COMPOSITE_CODES), + "I_DSTYNCOMP": (2, 827, 55, "6C-34", "DST_YN > 0", ALLOCATION_COMPOSITE_CODES), + "I_ERNYN": (1, 833, 55, "6C-34", "ERN_YN > 0", ALLOCATION_ANNVAL_CODES), + "I_FRMYN": (1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_ANNVAL_CODES), + "I_OIVAL": (1, 843, 56, "6C-35", "OI_VAL > 0", ALLOCATION_ANNVAL_CODES), + "I_PENYN": (1, 854, 57, "6C-36", "PEN_YN > 0", ALLOCATION_ANNVAL_CODES), + "I_RNTVAL": (1, 861, 57, "6C-36", "RNT_VAL > 0", ALLOCATION_ANNVAL_CODES), + "I_RNTYN": (1, 862, 57, "6C-36", "RNT_YN > 0", ALLOCATION_ANNVAL_CODES), +} + +# Fields these families publish with no direct allocation flag in the 2025 +# dictionary. Their allocation provenance is unresolved, never "raw respondent". +UNFLAGGED_FIELDS = ( + "PNSN_VAL", + "FRSE_VAL", + "FRSE_YN", + "FRMOTR", + "OI_OFF", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "DST_YN", + "DST_YN_YNG", + "DST_SC1", + "DST_SC1_YNG", + "DST_SC2", + "DST_SC2_YNG", +) + +READ_COLUMNS = ( + *COORDINATE_COLUMNS, + *AMOUNT_ENTRIES, + *RECEIPT_ENTRIES, + *ACCOUNT_ENTRIES, + "OI_OFF", + *ALLOCATION_ENTRIES, +) + +# Printed labels that stay separable from any modelled component. +PENSION_TOTAL_SCOPE = ( + "total combined amount of pension income received from all pension sources" +) +NET_PROPERTY_AMOUNT_SCOPE = "income from rent after expenses" +NET_PROPERTY_RECEIPT_SCOPE = ( + "own any land, property, rented to others, or receive income from royalties, " + "roomers or boarders, or from estates or trusts" +) +FARM_AMOUNT_SCOPE = "total amount of farm self-employment earnings" + +FAMILIES = ( + "pension_annuity", + "retirement_distribution", + "net_property", + "farm", + "other_income", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_INCOME_ROUTING_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _amount_pattern(field): + length, _, minimum, *_ = AMOUNT_ENTRIES[field] + sign = "-?" if minimum < 0 else "" + return re.compile(sign + r"[0-9]{1," + str(length) + r"}", re.ASCII) + + +_AMOUNT_PATTERNS = {name: _amount_pattern(name) for name in AMOUNT_ENTRIES} + + +def literal_code(token, allowed, *, width): + """Keep an unreadable literal unknown; never recode it to a printed value. + + Missing, malformed and out-of-range literals stay distinguishable from each + other and from a printed code, so a consumer cannot mistake an unresolved + routing answer for an observed one. A literal wider than the printed entry + is malformed, not a zero-padded reading of a shorter code. + """ + require(type(token) is str and len(token) <= TOKEN_MAX_CHARS, "TOKEN_BOUND") + if token == "": + return None, "missing" + if re.fullmatch(r"[0-9]{1," + str(width) + r"}", token, re.ASCII) is None: + return None, "malformed" + value = int(token) + if value not in allowed: + return None, "outside_printed_range" + return value, "in_printed_range" + + +def amount_state(value, *, niu_codes): + """Split a printed non-dollar code away from the dollar reading of a slot.""" + if not np.isfinite(value): + require(np.isnan(value), "AMOUNT_NOT_FINITE") + return "missing", np.nan + if int(value) in niu_codes and value == int(value): + return "declared_niu", np.nan + return ("zero" if value == 0 else "nonzero"), float(value) + + +# Statuses whose dollar reading is established by the source itself. Every other +# status leaves the canonical amount unknown rather than completing it with zero. +KNOWN_AMOUNT_STATUSES = ( + "known_receipt", + "receipt_with_net_zero", + "known_nonreceipt", +) + + +def receipt_status(universe, receipt, amount_kind, *, net_measure): + """Classify one family row without inventing receipt, absence or dollars. + + `universe` is True, False, or None when the printed universe itself is not + resolved by the retained literals. A zero amount under a yes answer is a + genuine net zero only for a signed net measure; for a gross entry the + printed zero reads "none or niu" and stays ambiguous. + """ + code, status = receipt + if universe is None: + return "unresolved_reporting_universe" + if universe is False: + outside = code == 0 and amount_kind in ("zero", "declared_niu", "missing") + return ( + "outside_reporting_universe" + if outside + else "contradictory_outside_reporting_universe" + ) + if status == "missing": + return "missing_receipt_literal" + if status != "in_printed_range": + return "unrecognized_receipt_literal" + if amount_kind == "declared_niu": + return "niu" if code == 0 else "contradictory_declared_niu_amount" + if amount_kind == "missing": + return "missing_amount" + if code == 0: + return "niu" if amount_kind == "zero" else "contradictory_niu_nonzero" + if code == 1: + if amount_kind == "nonzero": + return "known_receipt" + return "receipt_with_net_zero" if net_measure else "ambiguous_recipient_zero" + return "known_nonreceipt" if amount_kind == "zero" else "contradictory_no_nonzero" + + +def _classify(amounts, universes, receipts, *, niu_codes, net_measure): + """Row-wise family classification returning labels and canonical amounts.""" + labels, canonical = [], np.full(len(amounts), np.nan, dtype=np.float64) + kinds, sources = [], np.full(len(amounts), np.nan, dtype=np.float64) + for i, value in enumerate(amounts): + kind, dollars = amount_state(value, niu_codes=niu_codes) + kinds.append(kind) + sources[i] = dollars + label = receipt_status(universes[i], receipts[i], kind, net_measure=net_measure) + labels.append(label) + if label in KNOWN_AMOUNT_STATUSES: + canonical[i] = dollars if label == "known_receipt" else 0.0 + return labels, canonical, kinds, sources + + +def _codes_frame(prefix, tokens, allowed, labels=None, *, width=1): + """Typed code/label/status columns for one printed routing literal.""" + pairs = [literal_code(t, allowed, width=width) for t in tokens] + frame = pd.DataFrame(index=range(len(tokens))) + frame[prefix + "_literal"] = pd.array(list(tokens), dtype="string") + frame[prefix + "_code"] = pd.array([c for c, _ in pairs], dtype="Int16") + frame[prefix + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + if labels is not None: + frame[prefix + "_label"] = pd.array( + [labels[c] if c is not None else pd.NA for c, _ in pairs], dtype="string" + ) + return frame, [c for c, _ in pairs], [s for _, s in pairs] + + +def _age_universe(ages): + return [bool(a >= 15) for a in ages] + + +def _allocation_origin(flags, unflagged): + """Family allocation provenance from published flags only. + + Every published flag here prints a conditional universe, so an unpopulated + literal is not a defect and is not "no allocation" either. A family that + also publishes unflagged fields can never report a clean "no allocation": + the dictionary does not publish that evidence for those fields. + """ + origins = [] + count = len(next(iter(flags.values()))[0]) + for i in range(count): + codes = [flags[f][0][i] for f in flags] + statuses = [flags[f][1][i] for f in flags] + if any(s not in ("missing", "in_printed_range") for s in statuses): + origins.append("unresolved_allocation_provenance") + elif any(c is not None and c != 0 for c in codes): + origins.append("publisher_allocated") + elif any(s == "missing" for s in statuses): + origins.append("allocation_flag_not_populated") + elif unflagged: + origins.append("publisher_no_allocation_on_published_flags_only") + else: + origins.append("publisher_no_allocation") + return pd.array(origins, dtype="string") + + +def _pension_annuity(raw, ages): + """Separate pension and annuity totals; the combined total is never split.""" + out = pd.DataFrame(index=range(len(ages))) + universe = _age_universe(ages) + for prefix, amount_field, receipt_field in ( + ("pension", "PNSN_VAL", "PEN_YN"), + ("annuity", "ANN_VAL", "ANN_YN"), + ): + frame, codes, statuses = _codes_frame( + "pension_annuity_" + prefix + "_receipt", + raw[receipt_field], + RECEIPT_CODES, + RECEIPT_CODES, + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"][amount_field], + universe, + list(zip(codes, statuses, strict=True)), + niu_codes=AMOUNT_ENTRIES[amount_field][8], + net_measure=False, + ) + out = pd.concat([out, frame], axis=1) + out["pension_annuity_" + prefix + "_source_total"] = sources + out["pension_annuity_" + prefix + "_amount_kind"] = pd.array( + kinds, dtype="string" + ) + out["pension_annuity_" + prefix + "_reporting_status"] = pd.array( + labels, dtype="string" + ) + out["pension_annuity_" + prefix + "_known_amount"] = canonical + out["pension_annuity_source_reporting_universe"] = pd.array( + universe, dtype="boolean" + ) + out["pension_annuity_combined_total_scope"] = pd.array( + [PENSION_TOTAL_SCOPE] * len(ages), dtype="string" + ) + out["pension_annuity_private_share_applied"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + out["pension_annuity_taxable_amount_known"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + out["pension_annuity_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_PENYN", "I_ANNVAL", "I_ANNYN")}, + unflagged=True, + ) + out["pension_annuity_pension_total_has_published_flag"] = pd.array( + [False] * len(ages), dtype="boolean" + ) + return out + + +def _retirement_distribution(raw, ages): + """Preserve every printed slot, the age route and the unresolved tax share.""" + rows = len(ages) + out = pd.DataFrame(index=range(rows)) + route = ["age58_and_over" if a >= 58 else "under_age58" for a in ages] + out["retirement_distribution_route"] = pd.array(route, dtype="string") + slots = ( + ("slot1", "DST_SC1", "DST_VAL1", "age58_and_over"), + ("slot2", "DST_SC2", "DST_VAL2", "age58_and_over"), + ("slot1_young", "DST_SC1_YNG", "DST_VAL1_YNG", "under_age58"), + ("slot2_young", "DST_SC2_YNG", "DST_VAL2_YNG", "under_age58"), + ) + slot_codes, slot_amounts, slot_kinds = {}, {}, {} + for name, code_field, amount_field, slot_route in slots: + prefix = "retirement_distribution_" + name + "_account" + frame, codes, statuses = _codes_frame( + prefix, raw[code_field], ACCOUNT_CODES, ACCOUNT_CODES + ) + out = pd.concat([out, frame], axis=1) + values = raw["amounts"][amount_field] + kinds = [amount_state(v, niu_codes=())[0] for v in values] + out["retirement_distribution_" + name + "_amount"] = values + out["retirement_distribution_" + name + "_applicable"] = pd.array( + [r == slot_route for r in route], dtype="boolean" + ) + statuses_out = [] + for i in range(rows): + if route[i] != slot_route: + statuses_out.append( + "off_route" + if kinds[i] in ("zero", "missing") + else "off_route_nonzero" + ) + elif codes[i] is None: + statuses_out.append("unresolved_slot_account") + elif kinds[i] == "missing": + statuses_out.append("missing_slot_amount") + elif codes[i] == 0: + statuses_out.append( + "niu_slot" + if kinds[i] == "zero" + else "contradictory_niu_slot_amount" + ) + else: + statuses_out.append( + "known_slot" if kinds[i] == "nonzero" else "ambiguous_slot_zero" + ) + out["retirement_distribution_" + name + "_slot_status"] = pd.array( + statuses_out, dtype="string" + ) + slot_codes[name] = codes + slot_amounts[name] = values + slot_kinds[name] = kinds + applicable = { + "age58_and_over": ("slot1", "slot2"), + "under_age58": ("slot1_young", "slot2_young"), + } + totals = np.full(rows, np.nan, dtype=np.float64) + ira = np.full(rows, np.nan, dtype=np.float64) + ira_slots, ambiguity, offroute = [], [], [] + for i in range(rows): + names = applicable[route[i]] + others = [n for n, *_ in slots if n not in names] + offroute.append(any(slot_kinds[n][i] == "nonzero" for n in others)) + if all(slot_kinds[n][i] != "missing" for n in names): + totals[i] = float(sum(slot_amounts[n][i] for n in names)) + ambiguity.append( + any( + slot_codes[n][i] not in (None, 0) and slot_kinds[n][i] == "zero" + for n in names + ) + ) + if all( + slot_codes[n][i] is not None and slot_kinds[n][i] != "missing" + for n in names + ): + matched = [n for n in names if slot_codes[n][i] == REGULAR_IRA_CODE] + ira[i] = float(sum(slot_amounts[n][i] for n in matched)) + ira_slots.append(len(matched)) + else: + ira_slots.append(None) + receipt_tokens = [ + raw["DST_YN"][i] if route[i] == "age58_and_over" else raw["DST_YN_YNG"][i] + for i in range(rows) + ] + frame, codes, statuses = _codes_frame( + "retirement_distribution_receipt", receipt_tokens, RECEIPT_CODES, RECEIPT_CODES + ) + out = pd.concat([out, frame], axis=1) + # The two printed DST universes name only the age-58 split; unlike every + # other family in this slice they do not print the 15+ floor. Whether a + # person under 15 is inside them is a source question, left unresolved. + universes = [True if a >= 15 else None for a in ages] + labels, canonical, _, _ = _classify( + totals, + universes, + list(zip(codes, statuses, strict=True)), + niu_codes=(), + net_measure=False, + ) + out["retirement_distribution_source_total"] = totals + out["retirement_distribution_reporting_status"] = pd.array(labels, dtype="string") + out["retirement_distribution_known_amount"] = canonical + out["retirement_distribution_regular_ira_amount"] = ira + out["retirement_distribution_regular_ira_slots"] = pd.array(ira_slots, dtype="Int8") + out["retirement_distribution_slot_zero_ambiguity"] = pd.array( + ambiguity, dtype="boolean" + ) + out["retirement_distribution_offroute_nonzero"] = pd.array( + offroute, dtype="boolean" + ) + out["retirement_distribution_source_reporting_universe"] = pd.array( + universes, dtype="boolean" + ) + out["retirement_distribution_taxable_amount_known"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["retirement_distribution_allocation_origin"] = _allocation_origin( + { + f: raw["allocations"][f] + for f in ( + "I_DSTSC", + "I_DSTSCCOMP", + "I_DSTVAL1COMP", + "I_DSTVAL2COMP", + "I_DSTYNCOMP", + ) + }, + unflagged=True, + ) + out["retirement_distribution_slot_amounts_have_published_flag"] = pd.array( + [False] * rows, dtype="boolean" + ) + return out + + +def _net_property(raw, ages): + """Signed net property total; receipt scope is wider than the amount scope.""" + rows = len(ages) + universe = _age_universe(ages) + frame, codes, statuses = _codes_frame( + "net_property_receipt", raw["RNT_YN"], RECEIPT_CODES, RECEIPT_CODES + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["RNT_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + niu_codes=(), + net_measure=True, + ) + out = frame + out["net_property_source_total"] = sources + out["net_property_amount_kind"] = pd.array(kinds, dtype="string") + out["net_property_reporting_status"] = pd.array(labels, dtype="string") + out["net_property_known_amount"] = canonical + out["net_property_is_net_loss"] = pd.array( + [bool(np.isfinite(v) and v < 0) for v in sources], dtype="boolean" + ) + out["net_property_source_reporting_universe"] = pd.array(universe, dtype="boolean") + out["net_property_amount_scope"] = pd.array( + [NET_PROPERTY_AMOUNT_SCOPE] * rows, dtype="string" + ) + out["net_property_receipt_scope"] = pd.array( + [NET_PROPERTY_RECEIPT_SCOPE] * rows, dtype="string" + ) + out["net_property_component_split_known"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["net_property_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_RNTVAL", "I_RNTYN")}, unflagged=False + ) + return out + + +def _farm(raw, ages): + """Farm universe comes from ERN_YN/FRMOTR evidence, never from age alone.""" + rows = len(ages) + out = pd.DataFrame(index=range(rows)) + universe_codes = {} + for field in ("ERN_YN", "FRMOTR"): + frame, codes, _ = _codes_frame( + "farm_" + field.lower(), raw[field], RECEIPT_CODES, RECEIPT_CODES + ) + out = pd.concat([out, frame], axis=1) + universe_codes[field] = codes + universe = [] + for i in range(rows): + pair = (universe_codes["ERN_YN"][i], universe_codes["FRMOTR"][i]) + if 1 in pair: + universe.append(True) + elif all(c is not None for c in pair): + universe.append(False) + else: + universe.append(None) + frame, codes, statuses = _codes_frame( + "farm_receipt", raw["FRSE_YN"], RECEIPT_CODES, RECEIPT_CODES + ) + out = pd.concat([out, frame], axis=1) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["FRSE_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + niu_codes=(), + net_measure=True, + ) + out["farm_source_total"] = sources + out["farm_amount_kind"] = pd.array(kinds, dtype="string") + out["farm_reporting_status"] = pd.array(labels, dtype="string") + out["farm_known_amount"] = canonical + out["farm_is_net_loss"] = pd.array( + [bool(np.isfinite(v) and v < 0) for v in sources], dtype="boolean" + ) + out["farm_source_reporting_universe"] = pd.array(universe, dtype="boolean") + out["farm_amount_scope"] = pd.array([FARM_AMOUNT_SCOPE] * rows, dtype="string") + out["farm_is_nonfarm_self_employment"] = pd.array([False] * rows, dtype="boolean") + out["farm_allocation_origin"] = _allocation_origin( + {f: raw["allocations"][f] for f in ("I_ERNYN", "I_FRMYN")}, unflagged=True + ) + out["farm_total_has_published_flag"] = pd.array([False] * rows, dtype="boolean") + return out + + +def _other_income(raw, ages): + """Reported category 20 is observed alimony; no residual rule creates one.""" + rows = len(ages) + universe = _age_universe(ages) + frame, codes, statuses = _codes_frame( + "other_income_receipt", raw["OI_YN"], RECEIPT_CODES, RECEIPT_CODES + ) + category, category_codes, category_statuses = _codes_frame( + "other_income_category", + raw["OI_OFF"], + OTHER_INCOME_CATEGORIES, + OTHER_INCOME_CATEGORIES, + width=OTHER_INCOME_CATEGORY_ENTRY[0], + ) + labels, canonical, kinds, sources = _classify( + raw["amounts"]["OI_VAL"], + universe, + list(zip(codes, statuses, strict=True)), + niu_codes=(), + net_measure=False, + ) + out = pd.concat([frame, category], axis=1) + out["other_income_source_total"] = sources + out["other_income_amount_kind"] = pd.array(kinds, dtype="string") + out["other_income_reporting_status"] = pd.array(labels, dtype="string") + out["other_income_known_amount"] = canonical + out["other_income_source_reporting_universe"] = pd.array(universe, dtype="boolean") + routing = [] + for i in range(rows): + code, status = category_codes[i], category_statuses[i] + if status == "missing": + routing.append("missing_category_literal") + elif code is None: + routing.append("unrecognized_category_literal") + elif codes[i] == 1 and code == 0: + routing.append("receipt_without_category") + elif codes[i] in (0, 2) and code != 0: + routing.append("category_without_receipt") + elif code == 0: + routing.append("niu_category") + else: + routing.append("reported_category") + out["other_income_routing_status"] = pd.array(routing, dtype="string") + out["other_income_is_reported_alimony"] = pd.array( + [ + bool(c == ALIMONY_CATEGORY_CODE and r == "reported_category") + for c, r in zip(category_codes, routing, strict=True) + ], + dtype="boolean", + ) + out["other_income_residual_rule_applied"] = pd.array( + [False] * rows, dtype="boolean" + ) + out["other_income_allocation_origin"] = _allocation_origin( + {"I_OIVAL": raw["allocations"]["I_OIVAL"]}, unflagged=True + ) + out["other_income_category_has_published_flag"] = pd.array( + [False] * rows, dtype="boolean" + ) + return out + + +def _read_capture(path, *, rows): + """Bounded literal reader; its path and row count establish no authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, keys, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + positions = [header.index(c) for c in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + record = dict(zip(READ_COLUMNS, (row[i] for i in positions), strict=True)) + key = record["PERIDNUM"] + require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") + for name, width in COORDINATE_WIDTHS.items(): + require( + re.fullmatch( + r"[0-9]{1," + str(width) + r"}", record[name], re.ASCII + ) + is not None, + "COORDINATE:" + name, + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and key not in keys, + "DUPLICATE_OR_INVALID_COORDINATE", + ) + for name, pattern in _AMOUNT_PATTERNS.items(): + require( + record[name] == "" or pattern.fullmatch(record[name]) is not None, + "AMOUNT_TOKEN:" + name, + ) + for name in ( + *RECEIPT_ENTRIES, + *ACCOUNT_ENTRIES, + "OI_OFF", + *ALLOCATION_ENTRIES, + ): + require(len(record[name]) <= TOKEN_MAX_CHARS, "TOKEN_BOUND:" + name) + records.append(record) + keys.add(key) + coordinates.add(pair) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records, columns=READ_COLUMNS).set_index("PERIDNUM", drop=False) + + +def amount_observations(field, positions, literals): + """Join literal validity; missing backing storage never becomes a zero. + + The retained parent requires complete ready money before issuing a + preparation. This keeps that prerequisite separate from the literal join so + a missing field would still be represented correctly if that scope grows. + """ + require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") + require(field.name in AMOUNT_ENTRIES, "MONEY_FIELD_NAME") + positions = np.asarray(positions) + require( + positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" + ) + tokens = tuple(literals) + pattern = _AMOUNT_PATTERNS[field.name] + require( + len(tokens) == len(positions) + and all(type(t) is str and (t == "" or pattern.fullmatch(t)) for t in tokens), + "AMOUNT_TOKEN", + ) + valid = field.validity[positions] == 1 + require( + np.array_equal(valid, np.array([t != "" for t in tokens])), + "CURRENT_AMOUNT_SOURCE_VALIDITY", + ) + amounts = field.amounts[positions].copy() + literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + require( + np.array_equal(amounts[valid], literal_values[valid]), + "CURRENT_AMOUNT_SOURCE_IDENTITY", + ) + amounts[~valid] = np.nan + _, _, minimum, maximum, *_ = AMOUNT_ENTRIES[field.name] + inside = amounts[valid] + require( + ((inside >= minimum) & (inside <= maximum)).all() + and (inside == np.floor(inside)).all(), + "AMOUNT_PRINTED_DOMAIN:" + field.name, + ) + return amounts + + +def _domain_agreement(ready): + """Bind printed entries to the live current-money domain contract.""" + domains = {d.name: d for d in ready.bindings.spec.fields} + for name, entry in AMOUNT_ENTRIES.items(): + domain = domains.get(name) + require(domain is not None, "MONEY_DOMAIN_MISSING:" + name) + require( + domain.entity == "person" + and domain.minimum == entry[2] + and domain.maximum == entry[3], + "MONEY_DOMAIN_DISAGREEMENT:" + name, + ) + + +@dataclass(frozen=True) +class CurrentAsecIncomeRoutingValues: + """Descriptive transport. Constructing this grants no source authority.""" + + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def project_income_routing(raw, ages): + """Pure per-family projection over already-validated literal arrays.""" + ages = np.asarray(ages, dtype=np.float64) + require( + np.isfinite(ages).all() + and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), + "AGE_DOMAIN", + ) + require(set(raw["amounts"]) == set(AMOUNT_ENTRIES), "AMOUNT_ARRAYS") + require( + set(raw["allocations"]) == set(ALLOCATION_ENTRIES) + and all( + len(v) == 2 and len(v[0]) == len(v[1]) == len(ages) + for v in raw["allocations"].values() + ), + "ALLOCATION_ARRAYS", + ) + for name, values in raw["amounts"].items(): + values = np.asarray(values) + require( + values.dtype == np.dtype("float64") + and values.ndim == 1 + and len(values) == len(ages) + and not np.isinf(values).any(), + "AMOUNT_ARRAY_CONTRACT:" + name, + ) + parts = [ + _pension_annuity(raw, ages), + _retirement_distribution(raw, ages), + _net_property(raw, ages), + _farm(raw, ages), + _other_income(raw, ages), + ] + out = pd.concat([p.reset_index(drop=True) for p in parts], axis=1) + out.insert(0, "source_age", ages) + require(len(set(out.columns)) == len(out.columns), "PROJECTION_COLUMN_COLLISION") + return out + + +def _raw_arrays(ordered, ready, positions): + allocations = {} + for name, entry in ALLOCATION_ENTRIES.items(): + pairs = [ + literal_code(t, entry[5], width=entry[0]) for t in ordered[name].tolist() + ] + allocations[name] = ( + [c for c, _ in pairs], + [s for _, s in pairs], + ) + amounts = { + name: amount_observations(ready.field(name), positions, ordered[name]) + for name in AMOUNT_ENTRIES + } + raw = {"amounts": amounts, "allocations": allocations} + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ordered[name].tolist() + return raw + + +def qualify_current_asec_income_routing(preparation): + """Capture the exact current source member retained by the original owner.""" + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + _domain_agreement(ready) + pins = [p for p in coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-income-routing-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[coverage._BODY_MAX], + ) + raw_member = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity + and _file_sha(captured) == digest, + "CAPTURE_CHANGED", + ) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows + and len(set(keys)) == rows + and set(keys) == set(raw_member.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw_member.loc[keys] + parent_people = parent.frame.person.iloc[positions] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent_people[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + ages = ordered.A_AGE.to_numpy(dtype="float64") + basis = project_income_routing(_raw_arrays(ordered, ready, positions), ages) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in AMOUNT_ENTRIES: + field = ready.field(name) + basis["amount_status_" + name] = field.statuses[positions] + basis["amount_validity_" + name] = field.validity[positions] + basis["zero_origin_" + name] = field.zero_origin[positions] + people = state.frame.person + channel = people[support_channel_column("person")] + selected = people.loc[channel.eq("asec")] + native_ids = selected[spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + evidence = { + "protocol": PROTOCOL, + "dictionary": { + "url": DICTIONARY_URL, + "sha256": DICTIONARY_SHA256, + "amount_entries": {k: list(v) for k, v in AMOUNT_ENTRIES.items()}, + "receipt_entries": {k: list(v) for k, v in RECEIPT_ENTRIES.items()}, + "account_entries": {k: list(v) for k, v in ACCOUNT_ENTRIES.items()}, + "other_income_category_entry": list(OTHER_INCOME_CATEGORY_ENTRY), + "allocation_entries": { + k: [*v[:5], list(v[5])] for k, v in ALLOCATION_ENTRIES.items() + }, + "fields_without_published_allocation_flag": list(UNFLAGGED_FIELDS), + "printed_universe_questions": { + "DST_VAL1": "printed universe is DST_SC1 = 1 although the label names " + "the source-1 distribution amount; retained verbatim", + "DST_SC2_YNG": "printed universe names DST_VAL_YNG, which has no " + "dictionary entry; retained verbatim", + "I_DSTVAL1COMP": "printed universe line is empty", + "DST_YN": "printed universes name only the a_age 58 split and do " + "not print the 15+ floor the other families print; coverage below " + "age 15 is unresolved", + "RNT_YN": NET_PROPERTY_RECEIPT_SCOPE, + "RNT_VAL": NET_PROPERTY_AMOUNT_SCOPE, + }, + }, + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(issued[1]), + "money_header_sha256": _sha(ready.header), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "selected_rows": len(out), + "acs_channel_rows": int(channel.eq("acs").sum()), + "families": list(FAMILIES), + "asec_income_year": 2024, + "asec_interview_year": 2025, + "policy": ( + "separate_source_totals_retained; ambiguous_NIU_missing_contradictions_" + "retained_unknown; no_default_zero_completion" + ), + "pension_private_share_applied": False, + "pension_taxable_amount_known": False, + "retirement_distribution_taxable_amount_known": False, + "net_property_component_split_known": False, + "farm_mapped_to_nonfarm_self_employment": False, + "other_income_residual_rule_applied": False, + "acs_components_modeled": False, + "observed_taxable_amount_claim": False, + "unallocated_observation_claim": False, + "source_admission_issued": False, + "release_eligible": False, + "projection_sha256": _sha(out.to_json(orient="table").encode()), + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require( + _sha(out.to_json(orient="table").encode()) == evidence["projection_sha256"], + "FINAL_PROJECTION_CHANGED", + ) + return CurrentAsecIncomeRoutingValues(out, literals, evidence) From bc9f7cf1b3b0d30d37fafac59f203d0a5c702a52 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:15:41 -0400 Subject: [PATCH 128/255] Preserve the survey interest remainder before PUF enrichment --- .../us-survey-interest-conservation.fixed.md | 1 + ...s-survey-interest-conservation-20260912.md | 32 +++++++ .../us_runtime/current_survey_predictors.py | 17 +++- .../test_us_current_survey_predictors.py | 93 ++++++++++++++++++- .../test_us_graph_atomic_survey_financial.py | 2 +- .../tests/test_us_graph_survey_puf55.py | 10 ++ 6 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 changelog.d/us-survey-interest-conservation.fixed.md create mode 100644 experiments/us-survey-interest-conservation-20260912.md diff --git a/changelog.d/us-survey-interest-conservation.fixed.md b/changelog.d/us-survey-interest-conservation.fixed.md new file mode 100644 index 000000000..b58d850b5 --- /dev/null +++ b/changelog.d/us-survey-interest-conservation.fixed.md @@ -0,0 +1 @@ +Retain the tax-exempt remainder when the survey financial graph splits observed ASEC or modeled ACS interest totals, preserving both parts on the original survey support before PUF enrichment. diff --git a/experiments/us-survey-interest-conservation-20260912.md b/experiments/us-survey-interest-conservation-20260912.md new file mode 100644 index 000000000..97bd273a0 --- /dev/null +++ b/experiments/us-survey-interest-conservation-20260912.md @@ -0,0 +1,32 @@ +# Conserving survey interest before PUF enrichment + +The financial graph now retains the tax-exempt remainder of the same interest +total used for its taxable component. Previously it attached only the taxable +part, leaving the original survey support's exempt component unresolved after +the clone1-only PUF attachment. + +The new eighth financial output is `INT_VAL - taxable_interest_income`, computed +before source-key alignment to both clones. ASEC uses its qualified current +amount; ACS uses the existing modeled interest total. The original taxable +calculation and legacy seven-leaf helper are unchanged. The graph's judgment +metadata identifies the split as an assumption, not separately observed +taxable and exempt amounts. It does not establish that the modeled ACS interest +total reconciles to the broader observed ACS property-income aggregate. + +Two new numerical controls first reproduced the missing column. After repair, +all 25 predictor and atomic-financial tests passed in 294.225 seconds, including +nonconstant and zero-regime invented source runs, source-key permutation, +conservation and cold/required graph replay. + +Three full PUF controls then passed in 617.283 seconds: the actual two-route +cold/required extension, independent whole-cohort finalizer comparison and +non-owned storage preservation, and retained checked-parent validation without +re-execution. The original survey complement remains identical through the +PUF attachment; clone1 retains its existing PUF ownership. All tests use +invented sources. No failure, error or skip occurred in either green run. + +Independent source review found no actionable defect. Ruff and CI test inventory +verification pass. The executable source SHA256 is +`0df64e5f7768b326f43d7b0ac927ccb24c89de8e3d21556a9139628a369b5358`. +These checks do not certify a native candidate, fiscal calibration or release. +The running native PUF pilot remains on its earlier frozen source. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py index 7598dba1a..9d4768a0b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -37,7 +37,10 @@ ) TARGETS = ("survey_current_INT_VAL", "survey_current_DIV_VAL", "survey_current_CAP_VAL") MONEY_FIELDS = leaves.CPS_CURRENT_PREDICTOR_MONEY_FIELDS -OUTPUTS = leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES +OUTPUTS = ( + *leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES, + "tax_exempt_interest_income", +) PROTOCOL = "microcosm.us.current-survey-predictor-completion.v1" PHASE = "survey_multispine_current_financial_completion" SEED = 578 @@ -474,6 +477,12 @@ def qualify_current_survey_predictors( "earnings_universe": dict(universe_receipt), "split_contract": leaves.cps_carried_current_leaf_contract(), "model_judgments": { + "interest_partition": { + "total": "source-qualified_ASEC_INT_VAL_or_ACS_modeled_INT_VAL", + "taxable": "INT_VAL*maintained_taxable_interest_fraction", + "tax_exempt": "INT_VAL-taxable_interest_income", + "split_is_observed": False, + }, "conditioning": list(predictors), **( {"demographic_conditioning": demographics[1]} @@ -585,6 +594,12 @@ def complete_predictor_columns(qualified, clone_frame, drawn_money): completed = leaves.derive_cps_current_predictor_leaves( {name: _numeric(values[name]) for name in MONEY_FIELDS} ) + # Retain the complementary part of the same observed or modeled total. + # The split remains a modeling judgment; neither part is separately observed. + # Derive here, before clone alignment, so both support arms conserve INT_VAL. + completed["tax_exempt_interest_income"] = ( + _numeric(values["INT_VAL"]) - completed["taxable_interest_income"] + ) native = pd.DataFrame(completed, index=values.index) person = clone_frame.person stack_ids = person[provenance.support_source_id_column("person")].to_numpy( diff --git a/packages/microcosm-build/tests/test_us_current_survey_predictors.py b/packages/microcosm-build/tests/test_us_current_survey_predictors.py index d342aaaec..2ef68ed17 100644 --- a/packages/microcosm-build/tests/test_us_current_survey_predictors.py +++ b/packages/microcosm-build/tests/test_us_current_survey_predictors.py @@ -9,7 +9,7 @@ import json import pickle from dataclasses import replace -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace import numpy as np import pandas as pd @@ -101,7 +101,7 @@ def _money(): def test_shared_current_money_split_preserves_signs_and_partition_totals(): source = _money() result = leaves.derive_cps_current_predictor_leaves(source) - assert set(result) == set(values.OUTPUTS) + assert set(result) == set(leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES) # The legacy full leaf owner unpacks this mapping. Preserve its original # insertion order so consumers constructing a DataFrame see the same schema. assert tuple(result) == ( @@ -137,6 +137,76 @@ def test_shared_current_money_split_preserves_signs_and_partition_totals(): assert source["WSAL_VAL"][0] == 0.0 +@pytest.mark.parametrize("reverse", (False, True)) +def test_financial_attachment_conserves_native_and_drawn_interest(reverse): + # Descriptive, invented inputs exercise the numerical attachment only; + # constructing this dataclass does not authenticate a survey source. + ids = pd.Index([11, 12, 13], name="person_id") + money = pd.DataFrame(0.0, index=ids, columns=values.MONEY_FIELDS) + money["INT_VAL"] = [120.031, 0.0, np.nan] + origins = pd.DataFrame( + {"native_person_id": [101, 102, 201], "source": ["asec", "asec", "acs"]}, + index=ids, + ) + features = pd.DataFrame(0.0, index=ids[-1:], columns=values.FEATURES) + qualified = values.QualifiedSurveyPredictors( + projection=b"", + matrix=graph.model_input.encode_recipient_matrix( + features, entity="person", entity_ids=ids[-1:].to_numpy(dtype=" Date: Sat, 12 Sep 2026 11:16:29 -0400 Subject: [PATCH 129/255] Reuse bounded source string encodings without changing frame digests --- ...s-source-string-seal-efficiency.changed.md | 1 + .../us-source-string-seal-20260912.json | 96 +++++++++++++++++++ experiments/us-source-string-seal-20260912.md | 31 ++++++ .../us_runtime/asec_current_money_source.py | 16 ++++ .../test_us_asec_current_money_source.py | 53 ++++++++++ 5 files changed, 197 insertions(+) create mode 100644 changelog.d/us-source-string-seal-efficiency.changed.md create mode 100644 experiments/us-source-string-seal-20260912.json create mode 100644 experiments/us-source-string-seal-20260912.md diff --git a/changelog.d/us-source-string-seal-efficiency.changed.md b/changelog.d/us-source-string-seal-efficiency.changed.md new file mode 100644 index 000000000..a0c33268e --- /dev/null +++ b/changelog.d/us-source-string-seal-efficiency.changed.md @@ -0,0 +1 @@ +Reuse bounded, column-local encodings for repeated source strings during ASEC frame hashing while preserving the exact digest byte stream and checking all values on every validation. diff --git a/experiments/us-source-string-seal-20260912.json b/experiments/us-source-string-seal-20260912.json new file mode 100644 index 000000000..c2ea1e348 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.json @@ -0,0 +1,96 @@ +{ + "scope": "invented columns, exact old byte-stream parity; no native build timing claim", + "baseline_commit": "d5cbe60b2c6402648565f5139dbf7d94be209def", + "before_source_sha256": "b1c0ba9856714ce2131f73af6daa7731265a4fcc50c696d095a28f80a9d7cd03", + "after_source_sha256": "9e6ecca118942f7a76a2ef66de22abc082d3e19ca87bca67ca01477958c5b7e2", + "cases": [ + { + "case": "low_cardinality_python_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "44146e967c4e15ce95279827bcd34fed25b12731c223a3b2ffb16b50026df371", + "before_cpu_seconds": 0.15536000000000016, + "after_cpu_seconds": 0.0252460000000001, + "speedup": 6.1538461538461355, + "all_cpu_samples": { + "before": [ + 0.158949, + 0.1550100000000001, + 0.15536000000000016 + ], + "after": [ + 0.025262000000000118, + 0.024386999999999937, + 0.0252460000000001 + ] + } + }, + { + "case": "low_cardinality_arrow_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "03a54e7514b1419afbc6e092f3cd308e3c760be4f30c318e08ab85badc5f0241", + "before_cpu_seconds": 0.15287399999999995, + "after_cpu_seconds": 0.02783200000000008, + "speedup": 5.492742167289434, + "all_cpu_samples": { + "before": [ + 0.1512340000000001, + 0.1538870000000001, + 0.15287399999999995 + ], + "after": [ + 0.028616999999999893, + 0.02783200000000008, + 0.027589000000000086 + ] + } + }, + { + "case": "unique_source_keys", + "rows": 100000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "3fa0e7051459014e6069c01acde5e7afc8de133e769a52e0a520fe75821a6d0b", + "before_cpu_seconds": 0.031774999999999665, + "after_cpu_seconds": 0.035517999999999716, + "speedup": 0.8946168140097955, + "all_cpu_samples": { + "before": [ + 0.03200500000000028, + 0.030060000000000198, + 0.031774999999999665 + ], + "after": [ + 0.035517999999999716, + 0.035597999999999796, + 0.03467199999999959 + ] + } + }, + { + "case": "numeric_control", + "rows": 500000, + "dtype": "dtype('int64')", + "exact_digest_equal": true, + "sha256": "d8577194aeb21cddc40a14a2e0a1f2f13dee18b0b9c26e83842214f3b98bb2ba", + "before_cpu_seconds": 0.0013450000000001516, + "after_cpu_seconds": 0.0013260000000001604, + "speedup": 1.0143288084464472, + "all_cpu_samples": { + "before": [ + 0.0014279999999997628, + 0.0013450000000001516, + 0.0013090000000000046 + ], + "after": [ + 0.0013320000000001109, + 0.0013260000000001604, + 0.0012980000000002434 + ] + } + } + ] +} diff --git a/experiments/us-source-string-seal-20260912.md b/experiments/us-source-string-seal-20260912.md new file mode 100644 index 000000000..2309362a3 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.md @@ -0,0 +1,31 @@ +# Repeated source-string hashing + +The ASEC source validator now reuses the exact length-prefixed encoding of a +repeated string within one column's digest calculation. It still visits every +value and computes a new digest on every validation. The cache holds at most +1,024 entries and 256 KiB of encoded payload, then stops admitting new entries. +Only exact Python strings use it; other scalar families retain the original +codec. Dtype metadata, null sentinels, row order and all digest bytes retain +their prior meaning. + +Thirty invented-source tests pass, including eleven added byte-stream parity +and later-mutation controls. They cover Python and Arrow strings, distinct +Unicode encodings, nulls, more than 1,024 distinct values, strings exceeding the +payload budget, and the unchanged bytes, boolean, integer and floating object +paths. The suite took 1.860 seconds, with no failures, errors or skips. + +The [benchmark receipt](us-source-string-seal-20260912.json) compares the exact +pre-change function from `d5cbe60b2` with the new implementation. Each row reports +the median of three process-CPU measurements on invented columns: + +| Column | Rows | Before | After | Ratio | +| --- | ---: | ---: | ---: | ---: | +| Repeated Python strings | 500,000 | 0.15536 s | 0.02525 s | 6.15× faster | +| Repeated Arrow strings | 500,000 | 0.15287 s | 0.02783 s | 5.49× faster | +| Unique source keys | 100,000 | 0.03178 s | 0.03552 s | 12% slower | +| Integer control | 500,000 | 0.00135 s | 0.00133 s | Approximately unchanged | + +All four complete digests match exactly. These measurements do not establish +native build speed or quality. Source-file identity changes because executable +code changed; an old graph receipt is not acceptance of the new implementation. +The active native PUF pilot retains its original frozen source. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py index afe4fa1ae..93e846b82 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py @@ -111,7 +111,23 @@ def _series_digest(digest, series): digest.update(len(value).to_bytes(8, "little")) digest.update(value) else: + # Repeated literal strings are common in full survey source frames. + # Cache their exact length-prefixed bytes within this column only; every + # value is still visited and every later seal rereads the current data. + # Both entry and payload bounds keep distinct or large strings bounded. + string_tokens = {} + token_bytes = 0 for value in series.to_numpy(dtype=object, copy=False): + if type(value) is str: + token = string_tokens.get(value) + if token is None: + encoded = checkpoint._encode_object_scalar(value) + token = len(encoded).to_bytes(8, "little") + encoded + if len(string_tokens) < 1024 and token_bytes + len(token) <= 262144: + string_tokens[value] = token + token_bytes += len(token) + digest.update(token) + continue encoded = checkpoint._encode_object_scalar(value) digest.update(len(encoded).to_bytes(8, "little")) digest.update(encoded) diff --git a/packages/microcosm-build/tests/test_us_asec_current_money_source.py b/packages/microcosm-build/tests/test_us_asec_current_money_source.py index 15f62d30e..f37b6c088 100644 --- a/packages/microcosm-build/tests/test_us_asec_current_money_source.py +++ b/packages/microcosm-build/tests/test_us_asec_current_money_source.py @@ -1,5 +1,6 @@ """Invented checkpoint-byte tests; private test pins are NOT Census evidence.""" +import hashlib import importlib.util import json from pathlib import Path @@ -16,6 +17,58 @@ from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +@pytest.mark.parametrize( + "series", + [ + pd.Series( + ["HEAD", "DEPENDENT", "HEAD", "", "é", "e\u0301", "\x00", "😀"] * 100, + dtype=object, + ), + pd.Series(["2024", None, "2024", ""] * 100, dtype="string[python]"), + pd.Series(["2024", None, "2024", ""] * 100, dtype="string[pyarrow]"), + pd.Series([str(i) for i in range(1500)] * 2, dtype=object), + pd.Series(["a" * 300000, "b" * 300000, "a" * 300000], dtype=object), + pd.Series([None, pd.NA, pd.NaT, np.nan], dtype=object), + pd.Series([b"a", b"", b"a", None], dtype=object), + pd.Series([True, False, None, np.bool_(True)], dtype=object), + pd.Series([0, 2**80, -(2**80), None], dtype=object), + pd.Series([0.0, -0.0, np.inf, -np.inf, float("nan"), None], dtype=object), + ], +) +def test_source_scalar_digest_preserves_original_exact_byte_stream(series): + # Independent pre-optimization scalar stream, including dtype metadata. + expected = hashlib.sha256() + expected.update( + source._json( + source.checkpoint._series_spec(series, label="authenticated source") + ) + ) + for value in series.to_numpy(dtype=object, copy=False): + encoded = source.checkpoint._encode_object_scalar(value) + expected.update(len(encoded).to_bytes(8, "little")) + expected.update(encoded) + actual = hashlib.sha256() + source._series_digest(actual, series) + assert actual.digest() == expected.digest() + + +def test_repeated_source_strings_do_not_hide_later_value_or_order_mutations(): + series = pd.Series(["2024", "2024", "2025"], dtype="string[python]") + + def digest(): + value = hashlib.sha256() + source._series_digest(value, series) + return value.digest() + + before = digest() + series.iloc[0] = "2025" + assert digest() != before + series.iloc[0] = "2024" + assert digest() == before + series.iloc[:] = ["2025", "2024", "2024"] + assert digest() != before + + def _original_fixture_module(): spec = importlib.util.spec_from_file_location( "current_money_v4_fixture", From 5b6810b222c8bfb4082cbe45e309d24333ca5d21 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:18:01 -0400 Subject: [PATCH 130/255] Make a validated snapshot a snapshot that serializes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metadata contract had one edge doing two jobs. At the emitting edge a numpy scalar and a non-finite float must be absorbed, or the observer becomes the thing that ends a diverging run; in the public codec they must be refused, or json.dumps fails on a payload the codec has already blessed and the caller gets a bare ValueError instead of a TargetSnapshotError. normalize_metadata and normalize_best_retained now take that split explicitly, and the validator uses the strict edge — which also makes the closed best_retained triple complete rather than defaulted when it is being validated rather than built. Co-Authored-By: Claude Opus 5 --- .../microcosm/calibrate/target_snapshots.py | 77 ++++++++++++++----- .../tests/test_target_snapshots.py | 32 ++++++++ 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py index ebd17acc5..e6a0811dd 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -373,15 +373,25 @@ def _metadata_key(key: object, *, where: str) -> str: return key -def _metadata_value(value: object, *, where: str, key: str) -> object: +def _metadata_value(value: object, *, where: str, key: str, coerce: bool) -> object: """One aggregate metadata value: a JSON scalar, and nothing else. - A non-finite float becomes ``None`` rather than an error, for the same - reason a non-finite estimate does: a diverging run legitimately holds one, - JSON cannot carry it, and the observer must never be the thing that ends a - calibration an unobserved run would have finished. + ``coerce`` separates the two edges this contract has. At the *emitting* + edge a numpy scalar becomes its Python equivalent and a non-finite float + becomes ``None``, for the same reason a non-finite estimate does: a + diverging run legitimately holds one, JSON cannot carry it, and the + observer must never be the thing that ends a calibration an unobserved run + would have finished. In the *public codec* neither is accepted, so a + payload that validates is a payload that serializes — the strictness is + what makes the finite-or-null claim true, and the emitter's normalization + is what keeps it from costing a run. """ if isinstance(value, np.generic): + if not coerce: + raise TargetSnapshotError( + f"{where}[{key!r}] is a {type(value).__name__}; a validated " + "snapshot carries plain JSON scalars." + ) value = value.item() if value is None or isinstance(value, (bool, str)): if isinstance(value, str) and len(value) > MAX_METADATA_STRING_LENGTH: @@ -397,7 +407,13 @@ def _metadata_value(value: object, *, where: str, key: str) -> object: ) return value if isinstance(value, float): - return _finite_or_none(value) + if coerce: + return _finite_or_none(value) + if not math.isfinite(value): + raise TargetSnapshotError( + f"{where}[{key!r}] must be finite or null, got {value!r}." + ) + return value raise TargetSnapshotError( f"{where}[{key!r}] must be a JSON scalar (string, number, boolean or null), " f"got {type(value).__name__}. Calibration target snapshots carry aggregate " @@ -406,7 +422,7 @@ def _metadata_value(value: object, *, where: str, key: str) -> object: def normalize_metadata( - value: object, *, where: str, allow_none: bool = False + value: object, *, where: str, allow_none: bool = False, coerce: bool = True ) -> dict[str, object] | None: """Validate ``value`` against the metadata contract and return a fresh copy. @@ -429,14 +445,19 @@ def normalize_metadata( normalized: dict[str, object] = {} for key, item in value.items(): name = _metadata_key(key, where=where) - normalized[name] = _metadata_value(item, where=where, key=name) + normalized[name] = _metadata_value(item, where=where, key=name, coerce=coerce) return normalized def normalize_best_retained( - value: object, *, epochs: int | None = None + value: object, *, epochs: int | None = None, coerce: bool = True ) -> dict[str, object]: - """The closed ``{available, epoch, loss}`` triple, freshly built.""" + """The closed ``{available, epoch, loss}`` triple, freshly built. + + ``coerce`` splits the emitting edge from the public codec exactly as + :func:`normalize_metadata` does, and ``coerce=False`` additionally + requires the triple to be complete rather than defaulted. + """ if value is None: return {"available": False, "epoch": None, "loss": None} if not isinstance(value, Mapping): @@ -448,11 +469,20 @@ def normalize_best_retained( raise TargetSnapshotError(f"best_retained carries unknown keys: {unknown}.") if "available" not in value: raise TargetSnapshotError("best_retained must record whether one is available.") + if not coerce: + missing = sorted(_BEST_RETAINED_KEYS - set(map(str, value))) + if missing: + raise TargetSnapshotError(f"best_retained is missing keys: {missing}.") available = value["available"] if not isinstance(available, bool): raise TargetSnapshotError("best_retained.available must be a boolean.") epoch = value.get("epoch") if isinstance(epoch, np.generic): + if not coerce: + raise TargetSnapshotError( + f"best_retained.epoch is a {type(epoch).__name__}; a validated " + "snapshot carries plain JSON scalars." + ) epoch = epoch.item() if epoch is not None: if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 0: @@ -467,13 +497,23 @@ def normalize_best_retained( ) loss = value.get("loss") if isinstance(loss, np.generic): + if not coerce: + raise TargetSnapshotError( + f"best_retained.loss is a {type(loss).__name__}; a validated " + "snapshot carries plain JSON scalars." + ) loss = loss.item() if loss is not None: if isinstance(loss, bool) or not isinstance(loss, (int, float)): raise TargetSnapshotError( f"best_retained.loss must be a number or null, got {loss!r}." ) - loss = _finite_or_none(loss) + if coerce: + loss = _finite_or_none(loss) + elif not math.isfinite(loss): + raise TargetSnapshotError( + f"best_retained.loss must be finite or null, got {loss!r}." + ) if not available and (epoch is not None or loss is not None): raise TargetSnapshotError( "best_retained says no best iterate is available, so it cannot also " @@ -810,9 +850,11 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: raise TargetSnapshotError("a calibration snapshot needs at least one target.") _strict_number(payload["loss"], what="loss") # Every metadata seam, one contract, checked uniformly. - normalize_metadata(payload["context"], where="context") - normalize_metadata(payload["search"], where="search", allow_none=True) - normalize_metadata(payload["selection"], where="selection", allow_none=True) + normalize_metadata(payload["context"], where="context", coerce=False) + normalize_metadata(payload["search"], where="search", allow_none=True, coerce=False) + normalize_metadata( + payload["selection"], where="selection", allow_none=True, coerce=False + ) non_finite_rows = payload["non_finite_rows"] if ( isinstance(non_finite_rows, bool) @@ -821,13 +863,8 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: ): raise TargetSnapshotError("non_finite_rows must be a non-negative integer.") best = normalize_best_retained( - payload["best_retained"], epochs=int(payload["epochs"]) + payload["best_retained"], epochs=int(payload["epochs"]), coerce=False ) - if best != dict(payload["best_retained"]): - raise TargetSnapshotError( - f"best_retained carries values the contract cannot represent: " - f"{dict(payload['best_retained'])!r}." - ) selection = payload["selection"] if payload["iterate"] == ITERATE_SELECTED and isinstance(selection, Mapping): selected_epoch = selection.get("selected_epoch") diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index fa97ae2d7..869ec2955 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -1121,6 +1121,38 @@ def test_the_observer_stays_passive_on_the_l0_budget_refit_and_prox_paths(): validate_target_snapshot(snapshot) +def test_the_codec_is_stricter_than_the_emitting_edge_and_says_so(): + """A validated payload is a payload that serializes. + + The emitter coerces what a running solver honestly produces (numpy + scalars, a non-finite loss) so the observer can never end a run; the codec + refuses those same values, so `json.dumps` can never be the thing that + fails on a payload the codec has already blessed. + """ + emitted = _payload(context={"tolerance": np.float64(0.5)}) + assert emitted["context"] == {"tolerance": 0.5} + assert isinstance(emitted["context"]["tolerance"], float) + json.dumps(emitted) + + for location in ("context", "search", "selection"): + for bad in (float("nan"), float("inf"), np.float64(0.5)): + payload = _payload() + payload[location] = {"tolerance": bad} + with pytest.raises(TargetSnapshotError): + validate_target_snapshot(payload) + + for bad_best in ( + {"available": True}, + {"available": True, "epoch": 1}, + {"available": True, "epoch": 1, "loss": float("nan")}, + {"available": True, "epoch": np.int64(1), "loss": 0.1}, + ): + payload = _payload() + payload["best_retained"] = bad_best + with pytest.raises(TargetSnapshotError): + validate_target_snapshot(payload) + + def test_created_at_is_a_timezone_aware_timestamp(): payload = _payload() parsed = datetime.fromisoformat(payload["created_at"]) From 536f1ceefcdafda3cc619c14b4da18e012a7be57 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:19:49 -0400 Subject: [PATCH 131/255] Record survey interest and remaining income source work --- docs/us-uk-release-path.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 57882d2bd..163ea01f8 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -82,6 +82,18 @@ runtime or quality of the full native donor. A separate current-source native packet uses a fresh writable store and the actual retained financial run. See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). +The maintained financial graph now retains an eighth field: the tax-exempt +interest remainder of its existing observed or modeled total. The split remains +a modeling assumption. Twenty-five financial/source controls and three full +invented PUF replay, finalizer and retained-output controls pass; the PUF stage +preserves the original-channel remainder. The historical native financial +receipt above still covers seven fields. See +[the interest correction](../experiments/us-survey-interest-conservation-20260912.md). +A separate reviewed source-hashing optimization preserves exact digest bytes +and passes thirty source tests. Its repeated-string benchmark improvement is +not a measured native build improvement; see +[the scoped benchmark](../experiments/us-source-string-seal-20260912.md). + The new fiscal measurement stage passes all 66 interpreter/stage checks on the combined integration, including an actual US-engine dividend calculation. It uses declared target bindings, household-aligned CSR measurements and explicit @@ -111,13 +123,23 @@ source/store corruption and late output mutation. Independent review approved the source and mechanical test colocation; 45 cases share one expensive fixture. See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). -The minimal invented fixture's 61 present input names out of 161 remain a -diagnostic inventory, with unresolved original-channel values. Native coverage +An earlier minimal invented fixture reported 61 present input names out of 161, +a diagnostic inventory with unresolved original-channel values. Native coverage will be measured on the new candidate. Parallel successor work is qualifying unemployment receipt/amount, three health-cost amounts and current interview health coverage from the actual survey sources. Reporting universes and missing values remain explicit; no prior-wage or age-only zero fallback is introduced. +Actual graph tracing also confirms that the legacy full ASEC carry and ACS +gap-fill are not executed by this host. Remaining pension, retirement-account, +property, farm and other-income inputs therefore need source-qualified producers +and appropriate ACS completion. The ACS property-income aggregate should +condition a joint decomposition before the dependent PUF fits; separately adding +rent afterward would not conserve that observed anchor. Source measurement +bridges and overlapping income categories must be resolved explicitly before +claiming reconciliation. The broader ACS retirement total is not merely pension +plus regular-IRA income. + The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale and release verification. The existing age-development runner is a separate From 4c4cc22d64ab5c5c0e1218313677ad32357c73a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:22:30 -0400 Subject: [PATCH 132/255] Record this lane's verification in its journal Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshot-fixes.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md index 0803abeba..62a827267 100644 --- a/PROGRESS-908-target-snapshot-fixes.md +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -68,6 +68,26 @@ ref, and am now implementing the four reviewed findings. Not pushed, no PR. - The metadata bounds (32 entries, 64-character keys, 256-character strings) are the narrowest values that comfortably hold everything `solve.py` emits. +## Verification (isolated interpreter, no installs, threads=1) + +`/Users/maxghenis/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910/.venv/bin/python -I -B -S` +with this worktree's shard `src` roots ahead of that venv's site-packages and +an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`. + +| battery | result | +| --- | --- | +| `packages/microcosm-calibrate/tests` | 307 passed (12 s) | +| `packages/microcosm-fit/tests` | 124 passed (13 s) — main's QRF pins intact | +| `test_spec_engine_loader.py` + `test_us_multispine_pool_tool.py` | 197 passed (309 s) | +| `packages/microcosm-build/tests -k "inventory or coverage or seed"` | 448 passed, 2 skipped (343 s) | +| `packages/microcosm-graph/tests -k parity` | 25 passed (88 s) | +| `tools/spec_engine_coverage.py --check` | 42156/42156 fields, 41/41 inventory | +| `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` | +| `ruff check .` | clean; `ruff format --check` clean on both touched files | + +18 of the 51 tests in `test_target_snapshots.py` are new; 9 of them fail on +the reviewed head `bae1887ff` and pass here. + ## Next - Independent re-review by main before integration/PR updates. From 9a325c8b0e2c24b745958a60256594bf73b4eec9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:23:14 -0400 Subject: [PATCH 133/255] Record the measured red-regression split in the lane journal Co-Authored-By: Claude Opus 5 --- PROGRESS-908-target-snapshot-fixes.md | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md index 62a827267..a2d54ad1c 100644 --- a/PROGRESS-908-target-snapshot-fixes.md +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -85,8 +85,32 @@ an import-path assertion, `OMP/MKL/OPENBLAS/NUMEXPR/VECLIB_NUM_THREADS=1`. | `tools/ci_test_groups.py --verify` | `verification=ok`; the snapshot test file lands in fast `rest` / engine `us-am`, never `[defaulted]` | | `ruff check .` | clean; `ruff format --check` clean on both touched files | -18 of the 51 tests in `test_target_snapshots.py` are new; 9 of them fail on -the reviewed head `bae1887ff` and pass here. +`test_target_snapshots.py` holds 52 tests, 16 of them new here and 2 existing +ones updated to the closed contract's refusal points. Replaying the whole file +against the reviewed head's `target_snapshots.py` (the only source that +differs) gives **12 failed, 40 passed** — 10 new red regressions plus the 2 +updated tests. Every finding has at least one red regression: + +- **C1** — `test_context_refuses_the_record_vectors_the_review_smuggled_through`, + `test_metadata_refuses_arbitrary_nested_payloads`, + `test_identifier_fields_must_be_strings`, + `test_supported_scalar_metadata_survives_and_is_bounded`. +- **A1** — `test_metadata_refuses_arbitrary_nested_payloads` is the review's + own aliasing counterexample; the detachment invariant itself is pinned by + `test_a_delivered_snapshot_shares_no_mutable_object_with_its_caller` and + `test_sink_mutation_cannot_reach_caller_metadata_or_the_next_snapshot`, + which pass on both heads because the flat case was never the defect. +- **A2** — `test_a_history_chunk_is_published_only_after_its_bytes_are_complete`, + `test_a_failed_chunk_write_leaves_no_partial_or_leftover_file`. +- **A3** — `test_codec_refuses_the_impossible_payloads_the_review_reproduced`, + `test_non_finite_diagnostics_stay_null_statuses_rather_than_aborting`, + `test_the_codec_is_stricter_than_the_emitting_edge_and_says_so`, + `test_created_at_is_a_timezone_aware_timestamp`. + +The remaining 6 new tests are preservation tests (the structured metadata +`solve.py` really emits, the epochs the solver really selects, duplicate chunk +refusal, and observer-off/observer-on weight parity on the L0, budget-search, +refit and proximal paths); they pass on both heads by design. ## Next From aa7bde7d41ef64f2c21282e13ab0ff9042ad522a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:25:00 -0400 Subject: [PATCH 134/255] Read printed money entries from the pinned domains artifact Three source-fidelity corrections found by independent review of the first draft: The authenticated money owner normalizes ANN_VAL's printed -1 to a stored zero and records DECLARED_NIU on its status axis, so re-deriving NIU from the stored number would have read that cell as a zero dollar annuity. The amount reading now comes from the parent's own status axis. The nine printed money entries are already attested per vintage in the packaged current-money domains artifact, so they are read from it under its pinned digest instead of being retyped, and the printed universes now carry the dictionary's own U+2265 rather than an ASCII rewrite. The published-allocation roster claimed six fields were unflagged that the dictionary does flag. Only PNSN_VAL, FRSE_VAL, FRSE_YN, OI_OFF and the four under-58 distribution variants lack one; the composite source-code flag's coverage of the under-58 route is recorded as unresolved. Adds the first invented-source integration tests joining a privately pinned member to the real preparation. Co-Authored-By: Claude Opus 5 --- .../current_asec_income_routing_source.py | 304 ++++++++++++----- .../test_us_current_asec_income_routing.py | 308 ++++++++++++++++++ 2 files changed, 535 insertions(+), 77 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_current_asec_income_routing.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index f8d16cce9..15b14e8cb 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -24,6 +24,8 @@ import re import tempfile from dataclasses import dataclass +from functools import lru_cache +from importlib import resources from pathlib import Path import numpy as np @@ -46,32 +48,81 @@ COORDINATE_WIDTHS = {"PH_SEQ": 5, "A_LINENO": 2, "A_AGE": 2} TOKEN_MAX_CHARS = 64 -# Printed money entries, verbatim from the pinned dictionary. `signed` marks an -# entry whose printed range admits a net loss; `niu_codes` are printed codes that -# are not dollars and must never be read as an amount. -# name: (length, position, minimum, maximum, pdf_page, printed_page, universe, -# signed, niu_codes) -AMOUNT_ENTRIES = { - "PNSN_VAL": (7, 571, 0, 9999999, 47, "6C-26", "PEN_YN = 1", False, ()), - "ANN_VAL": (6, 438, -1, 999999, 44, "6C-23", "ANN_YN = 1", False, (-1,)), - "DST_VAL1": (6, 495, 0, 999999, 45, "6C-24", "DST_SC1 = 1", False, ()), - "DST_VAL1_YNG": (6, 501, 0, 999999, 45, "6C-24", "DST_SC1_YNG = 1", False, ()), - "DST_VAL2": (6, 507, 0, 999999, 45, "6C-24", "DST_SC2 = 1", False, ()), - "DST_VAL2_YNG": (6, 513, 0, 999999, 45, "6C-24", "DST_SC2_YNG = 1", False, ()), - "RNT_VAL": (6, 621, -9999, 999999, 49, "6C-28", "RNT_YN = 1", True, ()), - "FRSE_VAL": ( - 7, - 390, - -9999999, - 9999999, - 43, - "6C-22", - "ERN_YN=1 or FRMOTR=1", - True, - (), - ), - "OI_VAL": (6, 549, 0, 999999, 47, "6C-26", "OI_YN = 1", False, ()), -} +# The nine printed money entries are already attested in the packaged current +# money domains artifact, so they are read from it rather than re-typed here. +DOMAINS_RESOURCE = "asec_current_money_domains_v1.json" +DOMAINS_SHA256 = money.RESOURCE_PINS[0] +CURRENT_INCOME_YEAR = 2024 +AMOUNT_FIELDS = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "RNT_VAL", + "FRSE_VAL", + "OI_VAL", +) + + +@dataclass(frozen=True) +class PrintedAmountEntry: + """One money entry exactly as the pinned domains artifact records it.""" + + name: str + printed_length: int + printed_position: int + printed_page: str + pdf_page_1based: int + encoded_minimum: int + encoded_maximum: int + universe_as_printed: str + values_as_printed: str + negative_dollars_permitted: bool + nonmoney_codes: tuple[int, ...] + zero_semantics: str + + +@lru_cache(maxsize=1) +def printed_amount_entries(): + """Read the pinned domains artifact once, on demand, never at import.""" + payload = resources.files(__package__).joinpath(DOMAINS_RESOURCE).read_bytes() + require(_sha(payload) == DOMAINS_SHA256, "DOMAINS_RESOURCE_SHA256") + data = json.loads(payload) + entries = {} + for field in data["fields"]: + if field["name"] not in AMOUNT_FIELDS: + continue + domain = field["domain"] + vintages = [ + v for v in field["vintages"] if v["income_year"] == CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:" + field["name"]) + vintage = vintages[0] + require( + vintage["dictionary_spelling"] == field["name"] + and vintage["pdf_sha256"] == DICTIONARY_SHA256 + and vintage["source_url"] == DICTIONARY_URL, + "DOMAIN_DICTIONARY_PIN:" + field["name"], + ) + entries[field["name"]] = PrintedAmountEntry( + name=field["name"], + printed_length=vintage["ascii_length_as_printed"], + printed_position=vintage["ascii_position_as_printed"], + printed_page=vintage["printed_page"], + pdf_page_1based=vintage["pdf_page_1based"], + encoded_minimum=domain["encoded_range_inclusive"]["minimum"], + encoded_maximum=domain["encoded_range_inclusive"]["maximum"], + universe_as_printed=vintage["universe_as_printed"], + values_as_printed=vintage["values_description"], + negative_dollars_permitted=domain["negative_dollars_permitted"], + nonmoney_codes=tuple(domain["declared_negative_nonmoney_codes"]), + zero_semantics=domain["zero_semantics"], + ) + require(set(entries) == set(AMOUNT_FIELDS), "DOMAIN_FIELD_ROSTER") + return entries + # Printed yes/no entries. The zero label is not shared: OI_YN prints # "none or niu" where the others print "niu", so a zero receipt literal does not @@ -80,7 +131,14 @@ RECEIPT_ENTRIES = { "PEN_YN": (1, 570, 47, "6C-26", "All Persons aged 15+", "niu"), "ANN_YN": (1, 444, 44, "6C-23", "All Persons aged 15+", "niu"), - "DST_YN": (1, 519, 46, "6C-25", "Persons aged 58 and over (a_age >= 58)", "niu"), + "DST_YN": ( + 1, + 519, + 46, + "6C-25", + "Persons aged 58 and over (a_age \u2265 58)", + "niu", + ), "DST_YN_YNG": (1, 520, 46, "6C-25", "Persons under age 58 (a_age < 58)", "niu"), "RNT_YN": (1, 627, 49, "6C-28", "All Persons aged 15+", "niu"), "FRSE_YN": (1, 397, 43, "6C-22", "ERN_YN=1 or FRMOTR=1", "Niu"), @@ -105,9 +163,9 @@ REGULAR_IRA_CODE = 4 # name: (length, position, pdf_page, printed_page, universe) ACCOUNT_ENTRIES = { - "DST_SC1": (1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age >= 58"), + "DST_SC1": (1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age \u2265 58"), "DST_SC1_YNG": (1, 492, 45, "6C-24", "DST_YN_YNG = 1 and a_age < 58"), - "DST_SC2": (1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age >= 58"), + "DST_SC2": (1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age \u2265 58"), "DST_SC2_YNG": (1, 494, 45, "6C-24", "DST_VAL_YNG > 0 and a_age < 58"), } @@ -168,29 +226,44 @@ "I_RNTYN": (1, 862, 57, "6C-36", "RNT_YN > 0", ALLOCATION_ANNVAL_CODES), } -# Fields these families publish with no direct allocation flag in the 2025 -# dictionary. Their allocation provenance is unresolved, never "raw respondent". +# Which printed field each published flag names, and the fields for which the +# 2025 dictionary publishes no flag at all. A family holding an unflagged field +# can never report a clean "no allocation": that evidence is simply not printed. +PUBLISHED_ALLOCATION_FLAG_BY_FIELD = { + "ANN_VAL": "I_ANNVAL", + "ANN_YN": "I_ANNYN", + "PEN_YN": "I_PENYN", + "RNT_VAL": "I_RNTVAL", + "RNT_YN": "I_RNTYN", + "OI_VAL": "I_OIVAL", + "ERN_YN": "I_ERNYN", + "FRMOTR": "I_FRMYN", + "DST_SC1": "I_DSTSC", + "DST_SC2": "I_DSTSC", + "DST_VAL1": "I_DSTVAL1COMP", + "DST_VAL2": "I_DSTVAL2COMP", + "DST_YN": "I_DSTYNCOMP", +} UNFLAGGED_FIELDS = ( "PNSN_VAL", "FRSE_VAL", "FRSE_YN", - "FRMOTR", "OI_OFF", - "DST_VAL1", "DST_VAL1_YNG", - "DST_VAL2", "DST_VAL2_YNG", - "DST_YN", "DST_YN_YNG", - "DST_SC1", "DST_SC1_YNG", - "DST_SC2", "DST_SC2_YNG", ) +AMBIGUOUS_FLAG_COVERAGE = { + "I_DSTSCCOMP": "printed label names DST_SC(2) while its printed universe " + "names DST_YN = 1 or DST_YNG_YN = 1; whether it covers the under-58 source " + "codes is unresolved" +} READ_COLUMNS = ( *COORDINATE_COLUMNS, - *AMOUNT_ENTRIES, + *AMOUNT_FIELDS, *RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF", @@ -234,13 +307,20 @@ def _file_sha(path): return digest.hexdigest() -def _amount_pattern(field): - length, _, minimum, *_ = AMOUNT_ENTRIES[field] - sign = "-?" if minimum < 0 else "" - return re.compile(sign + r"[0-9]{1," + str(length) + r"}", re.ASCII) - - -_AMOUNT_PATTERNS = {name: _amount_pattern(name) for name in AMOUNT_ENTRIES} +@lru_cache(maxsize=1) +def amount_patterns(): + """Token shapes from the printed width and the printed encoded minimum.""" + entries = printed_amount_entries() + return { + name: re.compile( + ("-?" if entry.encoded_minimum < 0 else "") + + r"[0-9]{1," + + str(entry.printed_length) + + r"}", + re.ASCII, + ) + for name, entry in entries.items() + } def literal_code(token, allowed, *, width): @@ -262,12 +342,19 @@ def literal_code(token, allowed, *, width): return value, "in_printed_range" -def amount_state(value, *, niu_codes): - """Split a printed non-dollar code away from the dollar reading of a slot.""" - if not np.isfinite(value): - require(np.isnan(value), "AMOUNT_NOT_FINITE") +def amount_state(value, status): + """Read the dollar meaning of one cell from the parent's own status axis. + + The authenticated money owner normalizes ANN_VAL's printed -1 to a stored + zero and records DECLARED_NIU. Re-deriving NIU from the stored number would + read that cell as a zero dollar annuity, so the status axis decides. + """ + status = int(status) + if status == money.CodebookStatus.MISSING_NULL: + require(np.isnan(value), "MISSING_STATUS_WITH_AMOUNT") return "missing", np.nan - if int(value) in niu_codes and value == int(value): + require(np.isfinite(value), "AMOUNT_NOT_FINITE") + if status == money.CodebookStatus.DECLARED_NIU: return "declared_niu", np.nan return ("zero" if value == 0 else "nonzero"), float(value) @@ -316,12 +403,12 @@ def receipt_status(universe, receipt, amount_kind, *, net_measure): return "known_nonreceipt" if amount_kind == "zero" else "contradictory_no_nonzero" -def _classify(amounts, universes, receipts, *, niu_codes, net_measure): +def _classify(amounts, statuses, universes, receipts, *, net_measure): """Row-wise family classification returning labels and canonical amounts.""" labels, canonical = [], np.full(len(amounts), np.nan, dtype=np.float64) kinds, sources = [], np.full(len(amounts), np.nan, dtype=np.float64) for i, value in enumerate(amounts): - kind, dollars = amount_state(value, niu_codes=niu_codes) + kind, dollars = amount_state(value, statuses[i]) kinds.append(kind) sources[i] = dollars label = receipt_status(universes[i], receipts[i], kind, net_measure=net_measure) @@ -391,9 +478,9 @@ def _pension_annuity(raw, ages): ) labels, canonical, kinds, sources = _classify( raw["amounts"][amount_field], + raw["statuses"][amount_field], universe, list(zip(codes, statuses, strict=True)), - niu_codes=AMOUNT_ENTRIES[amount_field][8], net_measure=False, ) out = pd.concat([out, frame], axis=1) @@ -447,7 +534,8 @@ def _retirement_distribution(raw, ages): ) out = pd.concat([out, frame], axis=1) values = raw["amounts"][amount_field] - kinds = [amount_state(v, niu_codes=())[0] for v in values] + slot_statuses = raw["statuses"][amount_field] + kinds = [amount_state(v, slot_statuses[i])[0] for i, v in enumerate(values)] out["retirement_distribution_" + name + "_amount"] = values out["retirement_distribution_" + name + "_applicable"] = pd.array( [r == slot_route for r in route], dtype="boolean" @@ -485,6 +573,7 @@ def _retirement_distribution(raw, ages): "under_age58": ("slot1_young", "slot2_young"), } totals = np.full(rows, np.nan, dtype=np.float64) + total_status = np.full(rows, int(money.CodebookStatus.AMOUNT_NONZERO), dtype="u1") ira = np.full(rows, np.nan, dtype=np.float64) ira_slots, ambiguity, offroute = [], [], [] for i in range(rows): @@ -493,6 +582,8 @@ def _retirement_distribution(raw, ages): offroute.append(any(slot_kinds[n][i] == "nonzero" for n in others)) if all(slot_kinds[n][i] != "missing" for n in names): totals[i] = float(sum(slot_amounts[n][i] for n in names)) + if totals[i] == 0: + total_status[i] = int(money.CodebookStatus.ZERO_NONE_OR_NIU) ambiguity.append( any( slot_codes[n][i] not in (None, 0) and slot_kinds[n][i] == "zero" @@ -522,9 +613,9 @@ def _retirement_distribution(raw, ages): universes = [True if a >= 15 else None for a in ages] labels, canonical, _, _ = _classify( totals, + np.where(np.isnan(totals), money.CodebookStatus.MISSING_NULL, total_status), universes, list(zip(codes, statuses, strict=True)), - niu_codes=(), net_measure=False, ) out["retirement_distribution_source_total"] = totals @@ -557,8 +648,8 @@ def _retirement_distribution(raw, ages): }, unflagged=True, ) - out["retirement_distribution_slot_amounts_have_published_flag"] = pd.array( - [False] * rows, dtype="boolean" + out["retirement_distribution_route_has_published_flag"] = pd.array( + [r == "age58_and_over" for r in route], dtype="boolean" ) return out @@ -572,9 +663,9 @@ def _net_property(raw, ages): ) labels, canonical, kinds, sources = _classify( raw["amounts"]["RNT_VAL"], + raw["statuses"]["RNT_VAL"], universe, list(zip(codes, statuses, strict=True)), - niu_codes=(), net_measure=True, ) out = frame @@ -627,9 +718,9 @@ def _farm(raw, ages): out = pd.concat([out, frame], axis=1) labels, canonical, kinds, sources = _classify( raw["amounts"]["FRSE_VAL"], + raw["statuses"]["FRSE_VAL"], universe, list(zip(codes, statuses, strict=True)), - niu_codes=(), net_measure=True, ) out["farm_source_total"] = sources @@ -665,9 +756,9 @@ def _other_income(raw, ages): ) labels, canonical, kinds, sources = _classify( raw["amounts"]["OI_VAL"], + raw["statuses"]["OI_VAL"], universe, list(zip(codes, statuses, strict=True)), - niu_codes=(), net_measure=False, ) out = pd.concat([frame, category], axis=1) @@ -745,7 +836,7 @@ def _read_capture(path, *, rows): min(pair) > 0 and pair not in coordinates and key not in keys, "DUPLICATE_OR_INVALID_COORDINATE", ) - for name, pattern in _AMOUNT_PATTERNS.items(): + for name, pattern in amount_patterns().items(): require( record[name] == "" or pattern.fullmatch(record[name]) is not None, "AMOUNT_TOKEN:" + name, @@ -772,13 +863,14 @@ def amount_observations(field, positions, literals): a missing field would still be represented correctly if that scope grows. """ require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") - require(field.name in AMOUNT_ENTRIES, "MONEY_FIELD_NAME") + require(field.name in AMOUNT_FIELDS, "MONEY_FIELD_NAME") positions = np.asarray(positions) require( positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" ) tokens = tuple(literals) - pattern = _AMOUNT_PATTERNS[field.name] + entry = printed_amount_entries()[field.name] + pattern = amount_patterns()[field.name] require( len(tokens) == len(positions) and all(type(t) is str and (t == "" or pattern.fullmatch(t)) for t in tokens), @@ -790,32 +882,45 @@ def amount_observations(field, positions, literals): "CURRENT_AMOUNT_SOURCE_VALIDITY", ) amounts = field.amounts[positions].copy() + statuses = field.statuses[positions].copy() literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + minimum, maximum = entry.encoded_minimum, entry.encoded_maximum + niu_codes = entry.nonmoney_codes + niu = statuses == money.CodebookStatus.DECLARED_NIU + require(not (niu & ~valid).any(), "NIU_WITHOUT_VALIDITY") + # A printed NIU code is stored as a normalized zero under DECLARED_NIU. Its + # literal must still be one of that entry's printed non-dollar codes. require( - np.array_equal(amounts[valid], literal_values[valid]), + np.array_equal(amounts[valid & niu], np.zeros(int((valid & niu).sum()))) + and all(v in niu_codes for v in literal_values[valid & niu]), + "CURRENT_AMOUNT_NIU_IDENTITY", + ) + direct = valid & ~niu + require( + np.array_equal(amounts[direct], literal_values[direct]), "CURRENT_AMOUNT_SOURCE_IDENTITY", ) amounts[~valid] = np.nan - _, _, minimum, maximum, *_ = AMOUNT_ENTRIES[field.name] - inside = amounts[valid] + inside = amounts[direct] require( ((inside >= minimum) & (inside <= maximum)).all() and (inside == np.floor(inside)).all(), "AMOUNT_PRINTED_DOMAIN:" + field.name, ) - return amounts + return amounts, statuses def _domain_agreement(ready): """Bind printed entries to the live current-money domain contract.""" domains = {d.name: d for d in ready.bindings.spec.fields} - for name, entry in AMOUNT_ENTRIES.items(): + for name, entry in printed_amount_entries().items(): domain = domains.get(name) require(domain is not None, "MONEY_DOMAIN_MISSING:" + name) require( domain.entity == "person" - and domain.minimum == entry[2] - and domain.maximum == entry[3], + and domain.minimum == entry.encoded_minimum + and domain.maximum == entry.encoded_maximum + and domain.zero_semantics == entry.zero_semantics, "MONEY_DOMAIN_DISAGREEMENT:" + name, ) @@ -837,7 +942,11 @@ def project_income_routing(raw, ages): and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), "AGE_DOMAIN", ) - require(set(raw["amounts"]) == set(AMOUNT_ENTRIES), "AMOUNT_ARRAYS") + require( + set(raw["amounts"]) == set(AMOUNT_FIELDS) + and set(raw["statuses"]) == set(AMOUNT_FIELDS), + "AMOUNT_ARRAYS", + ) require( set(raw["allocations"]) == set(ALLOCATION_ENTRIES) and all( @@ -848,11 +957,15 @@ def project_income_routing(raw, ages): ) for name, values in raw["amounts"].items(): values = np.asarray(values) + statuses = np.asarray(raw["statuses"][name]) require( values.dtype == np.dtype("float64") and values.ndim == 1 and len(values) == len(ages) - and not np.isinf(values).any(), + and not np.isinf(values).any() + and statuses.dtype == np.dtype("u1") + and len(statuses) == len(ages) + and np.isin(statuses, [int(c) for c in money.CodebookStatus]).all(), "AMOUNT_ARRAY_CONTRACT:" + name, ) parts = [ @@ -878,11 +991,15 @@ def _raw_arrays(ordered, ready, positions): [c for c, _ in pairs], [s for _, s in pairs], ) - amounts = { + observed = { name: amount_observations(ready.field(name), positions, ordered[name]) - for name in AMOUNT_ENTRIES + for name in AMOUNT_FIELDS + } + raw = { + "amounts": {k: v[0] for k, v in observed.items()}, + "statuses": {k: v[1] for k, v in observed.items()}, + "allocations": allocations, } - raw = {"amounts": amounts, "allocations": allocations} for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF"): raw[name] = ordered[name].tolist() return raw @@ -973,7 +1090,7 @@ def qualify_current_asec_income_routing(preparation): basis.index = pd.Index( np.asarray(parent.scope.person_ids)[positions], name="native_person_id" ) - for name in AMOUNT_ENTRIES: + for name in AMOUNT_FIELDS: field = ready.field(name) basis["amount_status_" + name] = field.statuses[positions] basis["amount_validity_" + name] = field.validity[positions] @@ -998,7 +1115,26 @@ def qualify_current_asec_income_routing(preparation): "dictionary": { "url": DICTIONARY_URL, "sha256": DICTIONARY_SHA256, - "amount_entries": {k: list(v) for k, v in AMOUNT_ENTRIES.items()}, + "amount_entries_source": { + "resource": DOMAINS_RESOURCE, + "sha256": DOMAINS_SHA256, + "income_year": CURRENT_INCOME_YEAR, + }, + "amount_entries": { + name: { + "printed_length": e.printed_length, + "printed_position": e.printed_position, + "printed_page": e.printed_page, + "pdf_page_1based": e.pdf_page_1based, + "encoded_range_inclusive": [e.encoded_minimum, e.encoded_maximum], + "universe_as_printed": e.universe_as_printed, + "values_as_printed": e.values_as_printed, + "negative_dollars_permitted": e.negative_dollars_permitted, + "declared_negative_nonmoney_codes": list(e.nonmoney_codes), + "zero_semantics": e.zero_semantics, + } + for name, e in printed_amount_entries().items() + }, "receipt_entries": {k: list(v) for k, v in RECEIPT_ENTRIES.items()}, "account_entries": {k: list(v) for k, v in ACCOUNT_ENTRIES.items()}, "other_income_category_entry": list(OTHER_INCOME_CATEGORY_ENTRY), @@ -1006,6 +1142,15 @@ def qualify_current_asec_income_routing(preparation): k: [*v[:5], list(v[5])] for k, v in ALLOCATION_ENTRIES.items() }, "fields_without_published_allocation_flag": list(UNFLAGGED_FIELDS), + "declared_niu_normalized_to_zero": [ + name + for name, entry in printed_amount_entries().items() + if entry.nonmoney_codes + ], + "published_allocation_flag_by_field": dict( + PUBLISHED_ALLOCATION_FLAG_BY_FIELD + ), + "ambiguous_allocation_flag_coverage": dict(AMBIGUOUS_FLAG_COVERAGE), "printed_universe_questions": { "DST_VAL1": "printed universe is DST_SC1 = 1 although the label names " "the source-1 distribution amount; retained verbatim", @@ -1025,6 +1170,11 @@ def qualify_current_asec_income_routing(preparation): "source_member_sha256": digest, "read_columns": list(READ_COLUMNS), "complete_current_source_rows": rows, + "joined_person_years": [CURRENT_INCOME_YEAR], + "restatement_note": ( + "the money owner restates non-2024 cohorts to the pinned price basis, " + "so the literal identity join is only taken on the 2024 rows" + ), "selected_rows": len(out), "acs_channel_rows": int(channel.eq("acs").sum()), "families": list(FAMILIES), diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py new file mode 100644 index 000000000..ea8956a4f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -0,0 +1,308 @@ +"""Actual source readers over constructed, privately pinned invented bytes. + +No native microdata, no country engine and no full PUF donor fixture: the +current ASEC income routing qualifier only needs the bounded survey preparation +fixture plus invented member literals. +""" + +import ast +import hashlib +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from test_us_asec_coverage_authentication import _changed_parent +from test_us_survey_population_preparation import fixture + +from microcosm.build.frame_checkpoint import load_frame_checkpoint +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + +# Invented current-year people. Ages 55/14 come from the shared fixture; the +# older adult exercises the printed age-58 distribution route. +CURRENT_PEOPLE = (105, 106, 107, 108) +AGES = {105: 55, 106: 14, 107: 70, 108: 14} +AMOUNTS = { + # PNSN ANN DSTV1 DSTV1Y DSTV2 DSTV2Y RNT FRSE OI + 105: (12000.0, -1.0, 0.0, 7000.0, 0.0, 0.0, -400.0, 0.0, 3000.0), + 106: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0), + 107: (9000.0, 0.0, 5000.0, 0.0, 0.0, 0.0, 500.0, -9000.0, 250.0), + 108: (0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 700.0), +} +AMOUNT_ORDER = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "RNT_VAL", + "FRSE_VAL", + "OI_VAL", +) +LITERALS = { + 105: { + "PEN_YN": "1", + "ANN_YN": "0", + "DST_YN": "0", + "DST_YN_YNG": "1", + "DST_SC1_YNG": "4", + "RNT_YN": "1", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "20", + }, + 106: {}, + 107: { + "PEN_YN": "2", + "ANN_YN": "1", + "DST_YN": "1", + "DST_SC1": "9", + "RNT_YN": "0", + "FRSE_YN": "1", + "ERN_YN": "1", + "OI_YN": "1", + "OI_OFF": "0", + }, + 108: {"OI_YN": "0", "ERN_YN": "", "FRMOTR": ""}, +} +DEFAULT_LITERAL = { + **{name: "0" for name in owner.RECEIPT_ENTRIES}, + **{name: "0" for name in owner.ACCOUNT_ENTRIES}, + "OI_OFF": "0", + **{name: "0" for name in owner.ALLOCATION_ENTRIES}, +} + + +def _member_literals(person_id): + return {**DEFAULT_LITERAL, **LITERALS.get(person_id, {})} + + +def routing_arguments(tmp_path, monkeypatch, *, reverse=True, ages=None): + """Extend the invented member before any owner issues a preparation. + + Only fixture registry pins change. No source issuer, ready() or native + loader is replaced, and the person-income attachment is rebuilt from the + changed member bytes exactly as production would. + """ + arguments = fixture(tmp_path, monkeypatch) + asec = arguments["source_dir"] / "asec" + parent_path, attachment = asec / "parent.h5", asec / "household-attachment.h5" + person = load_frame_checkpoint(parent_path).frame.person + ids = person.person_id.to_numpy() + ages = AGES if ages is None else ages + changes = {} + for offset, field in enumerate(AMOUNT_ORDER): + data = person[field].to_numpy(dtype="float64", copy=True) + for person_id in CURRENT_PEOPLE: + positions = np.flatnonzero(ids == person_id) + assert len(positions) == 1 + data[positions[0]] = AMOUNTS[person_id][offset] + changes[field] = data + age_column = person.A_AGE.to_numpy(copy=True) + for person_id in CURRENT_PEOPLE: + age_column[np.flatnonzero(ids == person_id)[0]] = ages[person_id] + changes["A_AGE"] = age_column + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = asec / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for field in (*AMOUNT_ORDER, "A_AGE"): + raw[field] = [str(int(updated.loc[key, field])) for key in raw.PERIDNUM] + keys = [int(key) for key in raw.PERIDNUM] + current = dict(zip(keys, updated.loc[raw.PERIDNUM].person_id, strict=True)) + for name in DEFAULT_LITERAL: + raw[name] = [_member_literals(int(current[key]))[name] for key in keys] + assert raw.notna().all().all() + ordered = raw.iloc[::-1] if reverse else raw + ordered.to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(raw), + len(data), + ) + ) + paths[year] = path + for module in (coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "routing-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, asec / "person-income-attachment.h5" + ) + return arguments + + +def _qualified(tmp_path, monkeypatch, **kwargs): + arguments = routing_arguments(tmp_path, monkeypatch, **kwargs) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + return prepared, owner.qualify_current_asec_income_routing(prepared) + + +def test_actual_invented_member_joins_the_real_preparation_by_native_keys( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert set(person.index) == set(CURRENT_PEOPLE) + # Borrowed money, native identity and member literal agree row by row. + assert person.loc[105, "pension_annuity_pension_source_total"] == 12000.0 + assert person.loc[105, "pension_annuity_pension_reporting_status"] == ( + "known_receipt" + ) + assert person.loc[105, "pension_annuity_pension_known_amount"] == 12000.0 + assert person.source_age.to_dict() == {k: float(v) for k, v in AGES.items()} + literals = qualified.asec_literals.set_index("PERIDNUM") + for person_id in CURRENT_PEOPLE: + key = str(person_id - 100).zfill(22) + assert int(literals.loc[key, "PNSN_VAL"]) == AMOUNTS[person_id][0] + assert int(literals.loc[key, "A_AGE"]) == AGES[person_id] + assert person.amount_validity_PNSN_VAL.eq(1).all() + assert ( + hashlib.sha256(qualified.person.to_json(orient="table").encode()).hexdigest() + == qualified.evidence["projection_sha256"] + ) + assert not qualified.evidence["source_admission_issued"] + assert not qualified.evidence["release_eligible"] + prepared.checked_view() + + +def test_pension_and_annuity_totals_stay_separate_and_unsplit(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + # PNSN_VAL is the printed combined total over all pension sources. Neither + # a private share nor a taxable share is derived from it. + assert person.loc[105, "pension_annuity_combined_total_scope"] == ( + owner.PENSION_TOTAL_SCOPE + ) + assert not person.pension_annuity_private_share_applied.any() + assert not person.pension_annuity_taxable_amount_known.any() + assert not person.pension_annuity_pension_total_has_published_flag.any() + # ANN_VAL's printed -1 is a NIU code, never a one dollar annuity loss. + assert np.isnan(person.loc[105, "pension_annuity_annuity_source_total"]) + assert person.loc[105, "pension_annuity_annuity_amount_kind"] == "declared_niu" + assert person.loc[105, "pension_annuity_annuity_reporting_status"] == "niu" + # A yes answer with a zero gross amount is ambiguous, not a known zero. + assert person.loc[107, "pension_annuity_annuity_reporting_status"] == ( + "ambiguous_recipient_zero" + ) + assert np.isnan(person.loc[107, "pension_annuity_annuity_known_amount"]) + # A no answer against a positive total is a retained contradiction. + assert person.loc[107, "pension_annuity_pension_reporting_status"] == ( + "contradictory_no_nonzero" + ) + assert person.loc[107, "pension_annuity_pension_source_total"] == 9000.0 + assert np.isnan(person.loc[107, "pension_annuity_pension_known_amount"]) + + +def test_distribution_routing_preserves_slots_ages_and_unresolved_accounts( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "retirement_distribution_route"] == "under_age58" + assert person.loc[107, "retirement_distribution_route"] == "age58_and_over" + # Regular IRA is account code 4; the slot amount is not a taxable amount. + assert person.loc[105, "retirement_distribution_slot1_young_account_code"] == 4 + assert person.loc[105, "retirement_distribution_slot1_young_account_label"] == ( + "Regular IRA" + ) + assert person.loc[105, "retirement_distribution_regular_ira_amount"] == 7000.0 + assert person.loc[105, "retirement_distribution_regular_ira_slots"] == 1 + assert person.loc[105, "retirement_distribution_source_total"] == 7000.0 + assert not person.retirement_distribution_taxable_amount_known.any() + assert bool(person.loc[107, "retirement_distribution_route_has_published_flag"]) + assert not bool(person.loc[105, "retirement_distribution_route_has_published_flag"]) + # An account literal outside the printed range leaves the composition and + # the regular IRA share unresolved rather than defaulting either way. + assert person.loc[107, "retirement_distribution_slot1_account_literal"] == "9" + assert person.loc[107, "retirement_distribution_slot1_account_literal_status"] == ( + "outside_printed_range" + ) + assert person.loc[107, "retirement_distribution_slot1_slot_status"] == ( + "unresolved_slot_account" + ) + assert pd.isna(person.loc[107, "retirement_distribution_slot1_account_code"]) + assert np.isnan(person.loc[107, "retirement_distribution_regular_ira_amount"]) + assert pd.isna(person.loc[107, "retirement_distribution_regular_ira_slots"]) + # The slot total is still the retained literal sum for the applicable route. + assert person.loc[107, "retirement_distribution_source_total"] == 5000.0 + # The printed distribution universes name only the age 58 split, so + # coverage below age 15 is a source question, not a resolved answer. + assert pd.isna(person.loc[106, "retirement_distribution_source_reporting_universe"]) + assert person.loc[106, "retirement_distribution_reporting_status"] == ( + "unresolved_reporting_universe" + ) + + +def test_signed_property_and_farm_totals_keep_losses_and_net_zero_receipt( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "net_property_source_total"] == -400.0 + assert person.loc[105, "net_property_known_amount"] == -400.0 + assert person.loc[105, "net_property_reporting_status"] == "known_receipt" + assert bool(person.loc[105, "net_property_is_net_loss"]) + # The receipt question is wider than the amount question, so the total is + # not independently labelled rental and no component split is claimed. + assert person.loc[105, "net_property_receipt_scope"] == ( + owner.NET_PROPERTY_RECEIPT_SCOPE + ) + assert person.loc[105, "net_property_amount_scope"] == ( + owner.NET_PROPERTY_AMOUNT_SCOPE + ) + assert not person.net_property_component_split_known.any() + # A NIU receipt literal against a positive amount stays a contradiction. + assert person.loc[107, "net_property_reporting_status"] == ( + "contradictory_niu_nonzero" + ) + assert person.loc[107, "net_property_source_total"] == 500.0 + assert np.isnan(person.loc[107, "net_property_known_amount"]) + # Farm receipt with a net zero is distinct from absence, NIU and unknown. + assert person.loc[105, "farm_reporting_status"] == "receipt_with_net_zero" + assert person.loc[105, "farm_known_amount"] == 0.0 + assert person.loc[107, "farm_source_total"] == -9000.0 + assert person.loc[107, "farm_reporting_status"] == "known_receipt" + assert bool(person.loc[107, "farm_is_net_loss"]) + assert not person.farm_is_nonfarm_self_employment.any() + # The farm universe comes from ERN_YN/FRMOTR evidence, never from age. + assert bool(person.loc[105, "farm_source_reporting_universe"]) + assert pd.isna(person.loc[108, "farm_source_reporting_universe"]) + assert person.loc[108, "farm_reporting_status"] == "unresolved_reporting_universe" + + +def test_other_income_keeps_reported_alimony_apart_from_residual_rules( + tmp_path, monkeypatch +): + _, qualified = _qualified(tmp_path, monkeypatch) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "other_income_category_code"] == 20 + assert person.loc[105, "other_income_category_label"] == "alimony" + assert person.loc[105, "other_income_routing_status"] == "reported_category" + assert bool(person.loc[105, "other_income_is_reported_alimony"]) + assert person.loc[105, "other_income_source_total"] == 3000.0 + # A receipt without a category is retained as an unresolved pattern; no + # residual rule assigns it to alimony or to miscellaneous income. + assert person.loc[107, "other_income_routing_status"] == "receipt_without_category" + assert not bool(person.loc[107, "other_income_is_reported_alimony"]) + assert not person.other_income_residual_rule_applied.any() + # A NIU receipt with a positive amount is a contradiction, not a nonfiler. + assert person.loc[108, "other_income_reporting_status"] == ( + "contradictory_outside_reporting_universe" + ) + assert person.loc[108, "other_income_source_total"] == 700.0 + assert np.isnan(person.loc[108, "other_income_known_amount"]) From b8380c475b514a2f86b2414c6ad38caba5b14f01 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:27:30 -0400 Subject: [PATCH 135/255] Reject null best-iterate metadata at the strict snapshot codec --- .../src/microcosm/calibrate/target_snapshots.py | 2 +- .../microcosm-calibrate/tests/test_target_snapshots.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py index e6a0811dd..5afeb99a1 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -458,7 +458,7 @@ def normalize_best_retained( :func:`normalize_metadata` does, and ``coerce=False`` additionally requires the triple to be complete rather than defaulted. """ - if value is None: + if value is None and coerce: return {"available": False, "epoch": None, "loss": None} if not isinstance(value, Mapping): raise TargetSnapshotError( diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index 869ec2955..ed4094dcb 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -1012,7 +1012,15 @@ def test_history_publication_still_refuses_to_overwrite_an_existing_chunk( def test_codec_refuses_the_impossible_payloads_the_review_reproduced(): + # The emitter's omitted best iterate is a complete triple; the public + # codec must not silently turn a serialized null into that default. + assert _payload()["best_retained"] == { + "available": False, + "epoch": None, + "loss": None, + } cases = { + "null_best_metadata": {"best_retained": None}, "epoch_exceeds_epochs": {"epoch": 99, "epochs": 1}, "invalid_timestamp": {"created_at": "not-a-time"}, "naive_timestamp": {"created_at": "2026-09-12T00:00:00"}, From b43369dc49e175803f62020cc1e72fa53926aed8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:30:46 -0400 Subject: [PATCH 136/255] Document target snapshot usage and remaining country integration --- PROGRESS-908-target-snapshot-fixes.md | 6 +++ PROGRESS-908-target-snapshots.md | 6 +++ docs/calibration-target-snapshots.md | 60 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 docs/calibration-target-snapshots.md diff --git a/PROGRESS-908-target-snapshot-fixes.md b/PROGRESS-908-target-snapshot-fixes.md index a2d54ad1c..708880f6d 100644 --- a/PROGRESS-908-target-snapshot-fixes.md +++ b/PROGRESS-908-target-snapshot-fixes.md @@ -3,6 +3,12 @@ Lane journal for branch `calibration-target-snapshots-908-fixes-20260912`. Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. +Historical note, 12 September 2026: this journal records the earlier repair +lane. The lane closed at `9a325c8b0`; root independently repaired the remaining +strict-null codec case in `b8380c475`, with 52 tests passing. Current scope and +remaining integration work are in [the maintained guide](docs/calibration-target-snapshots.md). +The earlier work-in-progress and not-pushed statements below are historical. + ## State Merged the completed slice (`bae1887ff`) onto `origin/main` `116d46ee9`, resolved diff --git a/PROGRESS-908-target-snapshots.md b/PROGRESS-908-target-snapshots.md index 35b8b6d9e..7a7f9dedc 100644 --- a/PROGRESS-908-target-snapshots.md +++ b/PROGRESS-908-target-snapshots.md @@ -3,6 +3,12 @@ Lane journal for branch `calibration-target-snapshots-908-20260912`. Journals are history, not state (see CLAUDE.md): check git/GitHub for current truth. +Historical note, 12 September 2026: the initial implementation below was +subsequently merged onto main `116d46ee9` and repaired through `b8380c475`. +Its original review and source-pin claims describe earlier revisions. +[The maintained guide](docs/calibration-target-snapshots.md) records current +component scope; country integration and issue 908 remain incomplete. + ## State Bounded first slice implemented, tested and committed on this branch. Not diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md new file mode 100644 index 000000000..5df3054ec --- /dev/null +++ b/docs/calibration-target-snapshots.md @@ -0,0 +1,60 @@ +# Calibration target snapshots + +An opt-in observer exposes target estimates already computed during an Adam +solve. It also records the estimates on the weights actually returned. Use it +to inspect fit over time without adding another model evaluation or changing +the optimization steps. + +```python +from pathlib import Path + +from microcosm.calibrate import ( + TargetSnapshotCadence, + TargetSnapshotObserver, + TargetSnapshotWriter, + calibrate, +) + +observer = TargetSnapshotObserver( + sink=TargetSnapshotWriter(Path("runs/example/targets"), history_limit=256), + run_id="example", + cadence=TargetSnapshotCadence(every=25), +) +result = calibrate(frame, targets, method="adam", target_snapshots=observer) +``` + +The caller supplies its existing frame and targets. No observer is enabled by +default. The example cadence is a caller choice, not a measured country-build +default. A sink exception propagates, matching the existing progress callback. + +Each snapshot has an ordered target identity, target values, achieved estimates, +relative errors, sequence, phase and iterate labels. Duplicate target names +remain distinguishable by ordered position. Honest nonfinite intermediate +estimates are represented by nulls and counts. Intermediate float32 estimates +and final float64 estimates are labeled with their actual iterate semantics. +Budget probes and subsequent selection/refit phases share one emission sequence. + +Metadata accepts bounded flat scalar mappings and checked string identifiers; +it does not accept nested payloads or record vectors. `best_retained` must be a +complete `{available, epoch, loss}` mapping in a serialized snapshot. The emitter +can fill that mapping when no best iterate exists; the public validator rejects +a serialized null in its place. Each delivered payload is detached from the +caller's metadata and subsequent snapshots. + +The local writer atomically replaces `latest.json`. History chunks become +visible only after their complete bytes are written and synced, and an existing +chunk cannot be overwritten. Retention and dropped-history counts are explicit. +Use a fresh run directory unless intentionally adopting existing history. + +This implements the solver and local storage portion of +[issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Country-host +wiring, staging upload, a dashboard consumer and native cadence/performance +acceptance remain separate work. The grouped US solver currently lives on the +launch integration branch and needs its own integration of this observer. + +The reviewed repair branch passed 307 calibration tests before the final strict +null correction. All 52 snapshot tests then passed after that one-line correction, +including passive solver parity tests. Independent review closed the metadata, +detachment, partial-publication and codec findings. See the +[invented benchmark evidence](../experiments/908-target-snapshot-bench-receipts.md) +for overhead measurements; those are not native US or UK benchmarks. From 7e2c7e77b6bcbe04ec0c019564244f8e608fb302 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:37:30 -0400 Subject: [PATCH 137/255] Cover the income routing qualifier with invented-source regressions Adds the pure reporting/routing regression table, the bounded member reader refusals, the money literal join including ANN_VAL's normalized NIU zero, the pinned domains-artifact agreement, row-order invariance, changed-bytes and wrong-pin refusals, row-for-row parent correspondence, repeated-qualification stability, and source-only portability. The published allocation roster gains OI_YN: the dictionary publishes only I_OIVAL for that family, so the other-income receipt literal carries no allocation evidence either. Co-Authored-By: Claude Opus 5 --- .../current_asec_income_routing_source.py | 1 + .../test_us_current_asec_income_routing.py | 429 ++++++++++++++++++ 2 files changed, 430 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 15b14e8cb..22ed7109d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -249,6 +249,7 @@ def printed_amount_entries(): "FRSE_VAL", "FRSE_YN", "OI_OFF", + "OI_YN", "DST_VAL1_YNG", "DST_VAL2_YNG", "DST_YN_YNG", diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index ea8956a4f..d9713321b 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -306,3 +306,432 @@ def test_other_income_keeps_reported_alimony_apart_from_residual_rules( ) assert person.loc[108, "other_income_source_total"] == 700.0 assert np.isnan(person.loc[108, "other_income_known_amount"]) + + +def test_row_order_of_the_source_member_cannot_change_the_projection( + tmp_path, monkeypatch +): + roots = {} + for name in ("forward", "reverse"): + roots[name] = tmp_path / name + roots[name].mkdir() + with pytest.MonkeyPatch.context() as forward: + _, ordered = _qualified(roots["forward"], forward, reverse=False) + with pytest.MonkeyPatch.context() as backward: + _, reversed_ = _qualified(roots["reverse"], backward, reverse=True) + pd.testing.assert_frame_equal(ordered.person, reversed_.person, check_exact=True) + assert ( + ordered.evidence["projection_sha256"] == reversed_.evidence["projection_sha256"] + ) + # The member digest legitimately differs; the projection must not. + assert ( + ordered.evidence["source_member_sha256"] + != reversed_.evidence["source_member_sha256"] + ) + + +def test_repeated_qualification_is_stable_and_transport_is_not_authority( + tmp_path, monkeypatch +): + prepared, qualified = _qualified(tmp_path, monkeypatch) + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + # Mutating the returned transport cannot reach a fresh live projection. + qualified.person.loc[qualified.person.index[0], "farm_source_total"] = 99.0 + qualified.asec_literals.loc[qualified.asec_literals.index[0], "OI_OFF"] = "99" + fresh = owner.qualify_current_asec_income_routing(prepared) + assert not fresh.person.farm_source_total.eq(99.0).any() + assert not fresh.asec_literals.OI_OFF.eq("99").any() + pd.testing.assert_frame_equal(fresh.person, again.person, check_exact=True) + + +@pytest.mark.parametrize("defect", ("digest", "member", "rows", "bytes")) +def test_changed_member_bytes_or_pin_refuse_before_any_projection( + tmp_path, monkeypatch, defect +): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + pins = list(coverage._MEMBER_PINS) + if defect == "bytes": + member = next(p[1] for p in pins if p[0] == 2024) + path = arguments["source_dir"] / "asec" / member + path.write_bytes(path.read_bytes().replace(b"12000", b"12001", 1)) + else: + index = next(i for i, p in enumerate(pins) if p[0] == 2024) + year, name, archive, digest, rows, size = pins[index] + pins[index] = { + "digest": (year, name, archive, "f" * 64, rows, size), + "member": (year, "pppub99.csv", archive, digest, rows, size), + "rows": (year, name, archive, digest, rows + 1, size), + }[defect] + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + with pytest.raises(ValueError): + owner.qualify_current_asec_income_routing(prepared) + + +@pytest.mark.parametrize("swap", ("coordinate", "amount")) +def test_borrowed_member_must_match_the_parent_row_for_row(tmp_path, monkeypatch, swap): + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = owner._read_capture + + def tampered(path, *, rows): + frame = real(path, rows=rows) + column = "A_LINENO" if swap == "coordinate" else "PNSN_VAL" + values = frame[column].to_numpy(copy=True) + frame[column] = np.concatenate([values[1:], values[:1]]) + return frame + + monkeypatch.setattr(owner, "_read_capture", tampered) + expected = ( + "PARENT_COORDINATE_IDENTITY" + if swap == "coordinate" + else "CURRENT_AMOUNT_SOURCE_IDENTITY" + ) + with pytest.raises(ValueError, match=expected): + owner.qualify_current_asec_income_routing(prepared) + + +def test_a_tampered_preparation_refuses_before_any_projection(tmp_path, monkeypatch): + """A replaced checked method is refused by the owner, not worked around.""" + arguments = routing_arguments(tmp_path, monkeypatch) + prepared = owner.source.prepare_authenticated_survey_population(**arguments) + real = type(prepared)._checked + calls = [] + + def late(self): + entry = real(self) + calls.append(entry) + return entry if len(calls) == 1 else (entry[0], b"{}", entry[2]) + + monkeypatch.setattr(type(prepared), "_checked", late) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED|SOURCE_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + + +def test_the_final_requalification_cannot_be_replaced_or_skipped(tmp_path, monkeypatch): + """The owner refuses a substituted final requalification, so this qualifier + cannot be made to return a projection that skipped it.""" + prepared, qualified = _qualified(tmp_path, monkeypatch) + real = owner.source._pure_final + seen = [] + + def spy(state): + seen.append(state) + return real(state) + + # Scoped so undoing the substitution leaves the fixture pins in place. + with pytest.MonkeyPatch.context() as replaced: + replaced.setattr(owner.source, "_pure_final", spy) + with pytest.raises(ValueError, match="FINAL_AUTHORITY_CHANGED"): + owner.qualify_current_asec_income_routing(prepared) + # Unreplaced, the same preparation still yields the identical projection. + again = owner.qualify_current_asec_income_routing(prepared) + assert ( + again.evidence["projection_sha256"] == qualified.evidence["projection_sha256"] + ) + assert ( + hashlib.sha256(again.person.to_json(orient="table").encode()).hexdigest() + == again.evidence["projection_sha256"] + ) + + +def test_the_qualifier_imports_only_the_accepted_source_neighbourhood(): + """Source-only portability: no graph, fit, engine or modelled stage import.""" + path = Path(owner.__file__) + tree = ast.parse(path.read_text()) + absolute, relative = set(), set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + absolute.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + relative.update( + alias.name for alias in node.names + ) if node.module is None else relative.add(node.module) + else: + absolute.add(node.module.split(".")[0]) + assert absolute <= { + "__future__", + "csv", + "hashlib", + "json", + "re", + "tempfile", + "dataclasses", + "functools", + "importlib", + "pathlib", + "numpy", + "pandas", + } + assert relative == { + "asec_coverage_authentication", + "asec_current_money", + "source_csv_builtin", + "survey_population_preparation", + "support_provenance", + } + # The legacy modelled assumptions are deliberately not reachable from here. + text = path.read_text() + for forbidden in ("microcosm.graph", "policyengine", "0.590", "0.59"): + assert forbidden not in text + + +def _member_row(**overrides): + values = { + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + **{name: "0" for name in owner.AMOUNT_FIELDS}, + **DEFAULT_LITERAL, + **overrides, + } + return [values[name] for name in owner.READ_COLUMNS] + + +def _write_member(path, rows): + lines = [",".join(owner.READ_COLUMNS)] + lines.extend(",".join(row) for row in rows) + path.write_text("\n".join(lines) + "\n") + return path + + +@pytest.mark.parametrize( + "universe,token,kind,net,label", + [ + (True, "1", "nonzero", False, "known_receipt"), + (True, "1", "zero", False, "ambiguous_recipient_zero"), + (True, "1", "zero", True, "receipt_with_net_zero"), + (True, "1", "nonzero", True, "known_receipt"), + (True, "2", "zero", False, "known_nonreceipt"), + (True, "2", "nonzero", False, "contradictory_no_nonzero"), + (True, "0", "zero", False, "niu"), + (True, "0", "nonzero", False, "contradictory_niu_nonzero"), + (True, "0", "declared_niu", False, "niu"), + (True, "1", "declared_niu", False, "contradictory_declared_niu_amount"), + (True, "1", "missing", False, "missing_amount"), + (True, "", "zero", False, "missing_receipt_literal"), + (True, "9", "zero", False, "unrecognized_receipt_literal"), + (True, "01", "zero", False, "unrecognized_receipt_literal"), + (False, "0", "zero", False, "outside_reporting_universe"), + (False, "0", "missing", False, "outside_reporting_universe"), + (False, "1", "nonzero", False, "contradictory_outside_reporting_universe"), + (False, "2", "zero", False, "contradictory_outside_reporting_universe"), + (None, "1", "nonzero", False, "unresolved_reporting_universe"), + (None, "0", "zero", True, "unresolved_reporting_universe"), + ], +) +def test_receipt_and_amount_knownness_stay_separate(universe, token, kind, net, label): + receipt = owner.literal_code(token, owner.RECEIPT_CODES, width=1) + assert owner.receipt_status(universe, receipt, kind, net_measure=net) == label + # Only three statuses establish a dollar reading; nothing else completes. + assert (label in owner.KNOWN_AMOUNT_STATUSES) == ( + label in ("known_receipt", "receipt_with_net_zero", "known_nonreceipt") + ) + + +@pytest.mark.parametrize( + "status,value,kind,dollars", + [ + (owner.money.CodebookStatus.AMOUNT_NONZERO, 120.0, "nonzero", 120.0), + (owner.money.CodebookStatus.AMOUNT_NONZERO, -900.0, "nonzero", -900.0), + (owner.money.CodebookStatus.ZERO_NONE_OR_NIU, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.ZERO_DOLLARS_AS_CODED, 0.0, "zero", 0.0), + (owner.money.CodebookStatus.DECLARED_NIU, 0.0, "declared_niu", None), + (owner.money.CodebookStatus.MISSING_NULL, np.nan, "missing", None), + ], +) +def test_amount_meaning_comes_from_the_parent_status_axis(status, value, kind, dollars): + read_kind, read_dollars = owner.amount_state(value, status) + assert read_kind == kind + if dollars is None: + assert np.isnan(read_dollars) + else: + assert read_dollars == dollars + # A normalized NIU zero must never be read back as a zero dollar amount. + if status == owner.money.CodebookStatus.DECLARED_NIU: + assert read_kind != "zero" + + +@pytest.mark.parametrize( + "token,width,code,status", + [ + ("4", 1, 4, "in_printed_range"), + ("20", 2, 20, "in_printed_range"), + ("", 1, None, "missing"), + ("01", 1, None, "malformed"), + (" 4", 1, None, "malformed"), + ("4.0", 1, None, "malformed"), + ("NA", 1, None, "malformed"), + ("9", 1, None, "outside_printed_range"), + ("21", 2, None, "outside_printed_range"), + ], +) +def test_routing_literals_stay_unknown_rather_than_recoded(token, width, code, status): + allowed = owner.ACCOUNT_CODES if width == 1 else owner.OTHER_INCOME_CATEGORIES + assert owner.literal_code(token, allowed, width=width) == (code, status) + + +@pytest.mark.parametrize( + "defect,match", + [ + ("duplicate_key", "DUPLICATE_OR_INVALID_COORDINATE"), + ("duplicate_coordinate", "DUPLICATE_OR_INVALID_COORDINATE"), + ("short_key", "PERSON_KEY"), + ("bad_age", "COORDINATE:A_AGE"), + ("unsigned_negative", "AMOUNT_TOKEN:PNSN_VAL"), + ("wide_amount", "AMOUNT_TOKEN:OI_VAL"), + ("oversized_token", "TOKEN_BOUND:OI_OFF"), + ("missing_column", "HEADER"), + ("row_count", "ROW_COUNT"), + ], +) +def test_bounded_member_reader_refuses_structural_defects(tmp_path, defect, match): + path = tmp_path / "invented.csv" + rows = [_member_row()] + if defect == "duplicate_key": + rows.append(_member_row(PH_SEQ="2")) + elif defect == "duplicate_coordinate": + rows.append(_member_row(PERIDNUM="0000000000000000000002")) + elif defect == "short_key": + rows = [_member_row(PERIDNUM="1")] + elif defect == "bad_age": + rows = [_member_row(A_AGE="123")] + elif defect == "unsigned_negative": + rows = [_member_row(PNSN_VAL="-5")] + elif defect == "wide_amount": + rows = [_member_row(OI_VAL="1234567")] + elif defect == "oversized_token": + rows = [_member_row(OI_OFF="9" * 65)] + _write_member(path, rows) + if defect == "missing_column": + text = path.read_text().split("\n") + keep = [i for i, c in enumerate(owner.READ_COLUMNS) if c != "OI_OFF"] + path.write_text( + "\n".join(",".join(line.split(",")[i] for i in keep) for line in text[:-1]) + + "\n" + ) + expected = 2 if defect.startswith("duplicate") else 1 + if defect == "row_count": + expected = 2 + with pytest.raises(ValueError, match=match): + owner._read_capture(path, rows=expected) + + +def test_bounded_member_reader_accepts_signed_and_missing_amount_literals(tmp_path): + path = _write_member( + tmp_path / "invented.csv", + [_member_row(RNT_VAL="-9999", FRSE_VAL="-9999999", ANN_VAL="-1", OI_VAL="")], + ) + frame = owner._read_capture(path, rows=1) + assert frame.iloc[0].RNT_VAL == "-9999" + assert frame.iloc[0].FRSE_VAL == "-9999999" + assert frame.iloc[0].ANN_VAL == "-1" + assert frame.iloc[0].OI_VAL == "" + + +def _field(name, amounts, statuses, validity): + return owner.money.MoneyField( + name, + np.array(amounts, dtype=" Date: Sat, 12 Sep 2026 11:42:32 -0400 Subject: [PATCH 138/255] Document the current ASEC income routing source contract Record what the qualifier establishes, the reporting status vocabulary, the printed source-level questions preserved as evidence, the allocation provenance classes, where the constants come from, and the ACS and canonical attachment decisions that remain open. Co-Authored-By: Claude Opus 5 --- ...urrent-asec-income-routing-source.added.md | 1 + docs/us-current-asec-income-routing-source.md | 202 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 changelog.d/us-current-asec-income-routing-source.added.md create mode 100644 docs/us-current-asec-income-routing-source.md diff --git a/changelog.d/us-current-asec-income-routing-source.added.md b/changelog.d/us-current-asec-income-routing-source.added.md new file mode 100644 index 000000000..b4d469ed8 --- /dev/null +++ b/changelog.d/us-current-asec-income-routing-source.added.md @@ -0,0 +1 @@ +Qualify the current ASEC pension/annuity, retirement distribution, net property, farm and other-income source families into a typed reporting/routing projection, keeping printed totals, receipt universes, account and category routing, and allocation provenance distinct from any modelled taxable, private or residual component. diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md new file mode 100644 index 000000000..81ab7950f --- /dev/null +++ b/docs/us-current-asec-income-routing-source.md @@ -0,0 +1,202 @@ +# Current ASEC income reporting and routing source + +`microcosm.build.us_runtime.current_asec_income_routing_source` qualifies the +exact retained 2025 ASEC person member against the authenticated current-money +owner and projects five original-channel income families the current survey +graph does not yet supply: + +| Family | Printed amounts | Printed receipt / routing | +| --- | --- | --- | +| `pension_annuity` | `PNSN_VAL`, `ANN_VAL` | `PEN_YN`, `ANN_YN` | +| `retirement_distribution` | `DST_VAL1`, `DST_VAL1_YNG`, `DST_VAL2`, `DST_VAL2_YNG` | `DST_YN`, `DST_YN_YNG`, `DST_SC1`, `DST_SC1_YNG`, `DST_SC2`, `DST_SC2_YNG` | +| `net_property` | `RNT_VAL` | `RNT_YN` | +| `farm` | `FRSE_VAL` | `FRSE_YN`, `ERN_YN`, `FRMOTR` | +| `other_income` | `OI_VAL` | `OI_YN`, `OI_OFF` | + +Thirteen published allocation flags are read alongside them. + +`qualify_current_asec_income_routing(preparation)` takes a live +`AuthenticatedSurveyPopulationPreparation`, requalifies it, captures the pinned +2024 member once, joins it to the money parent by exact native keys, and returns +`CurrentAsecIncomeRoutingValues(person, asec_literals, evidence)`. The returned +values are descriptive transport. Constructing or mutating them issues no source +authority: a consuming host re-runs this qualifier and compares the result +around its own I/O, exactly as the accepted unemployment qualifier requires. + +## What the qualifier establishes + +- **The exact source member.** The 2024 pin is matched against the retained + native coverage receipt (member name, archive digest, member digest, row count + and byte size) before the capture, and the captured file's stat identity and + SHA-256 are re-checked after it. +- **Row-for-row parent correspondence.** `PH_SEQ`, `A_LINENO` and `A_AGE` from + the member must equal the money parent's `source_household_id`, `A_LINENO` and + `A_AGE` at the 2024 scope positions. The join is by native key, so member row + order cannot change the projection. +- **Literal/amount identity.** Every projected amount is the money owner's own + value, checked against the member literal, with the parent's `statuses`, + `validity` and `zero_origin` carried through as `amount_status_*`, + `amount_validity_*` and `zero_origin_*` columns. +- **A single bounded pass.** One capture, one read, one projection. There is no + per-family re-capture, no nested run issuer, and no PUF borrow. + +## What it deliberately does not establish + +- `PNSN_VAL` is printed as the *total combined amount of pension income received + from all pension sources* — `PEN_SC1`/`PEN_SC2` enumerate company, union, + federal, state, local, military and railroad pensions. It is not an observed + private or taxable amount. The archived `TAXABLE_PENSION_FRACTION = 0.590` + split is a modelled assumption and is not applied or referenced here; + `pension_annuity_private_share_applied` and + `pension_annuity_taxable_amount_known` are constant `False`. +- Account code 4 is `Regular IRA`. That identity does not observe any taxable + fraction, so `retirement_distribution_taxable_amount_known` is constant + `False` and no taxable component is derived. +- `RNT_YN` asks about land, property rented to others, royalties, roomers or + boarders, and estates or trusts, while `RNT_VAL` asks only about *income from + rent after expenses*. Both scopes are carried on every row and + `net_property_component_split_known` is constant `False`, so the total is not + independently labelled rental and no subdivision is separately modelled. +- `FRSE_VAL` is farm self-employment, not nonfarm `SEMP_VAL`; + `farm_is_nonfarm_self_employment` is constant `False`. +- `OI_OFF` code 20 is the reported alimony category. Nothing maps any other + code — including 19, `anything else` — onto alimony or onto a miscellaneous + residual; `other_income_residual_rule_applied` is constant `False`. +- ACS-channel people are not projected at all. `acs_components_modeled` is + `False` in the evidence and the projection covers only ASEC-channel rows. + +## Reporting status vocabulary + +`receipt_status` is the single classifier. Only `known_receipt`, +`receipt_with_net_zero` and `known_nonreceipt` establish a dollar reading +(`KNOWN_AMOUNT_STATUSES`); every other status leaves the canonical amount +unknown rather than completing it with zero. + +| Status | Meaning | +| --- | --- | +| `known_receipt` | in universe, receipt yes, amount non-zero (a signed loss included) | +| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`) | +| `ambiguous_recipient_zero` | in universe, receipt yes, amount zero on a gross entry whose printed zero reads "none or niu" | +| `known_nonreceipt` | in universe, receipt no, amount zero | +| `niu` | in universe, receipt 0, amount zero or a declared non-money code | +| `missing_amount` / `missing_receipt_literal` | one side of the pair is absent | +| `unrecognized_receipt_literal` | the receipt literal is malformed or outside the printed range | +| `contradictory_no_nonzero`, `contradictory_niu_nonzero`, `contradictory_declared_niu_amount`, `contradictory_outside_reporting_universe` | retained source contradictions, excluded from every canonical amount | +| `outside_reporting_universe` | the printed universe excludes the row and the literals agree | +| `unresolved_reporting_universe` | the printed universe cannot be resolved from the retained literals | + +Universes are per family and never reduced to age alone. `PEN_YN`, `ANN_YN`, +`RNT_YN` and `OI_YN` print `All Persons aged 15+`. `FRSE_YN` prints +`ERN_YN=1 or FRMOTR=1`, so the farm universe is `True` when either literal is +`1`, `False` when both are known and neither is, and unresolved otherwise — +a nonfiler is never treated as a non-recipient. + +## Source-level questions preserved as evidence + +These are printed-dictionary facts, kept distinguishable from modelling +judgments and recorded verbatim under +`evidence["dictionary"]["printed_universe_questions"]`: + +- `DST_VAL1`'s printed universe is `DST_SC1 = 1` although its label names the + source-1 distribution amount; taken literally that would restrict it to 401k + accounts. Retained verbatim, not silently corrected. +- `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no + dictionary entry. +- `I_DSTVAL1COMP`'s printed `Universe:` line is empty. +- `DST_YN` and `DST_YN_YNG` print only the age-58 split and, unlike every other + family in this slice, do not print the 15+ floor. Coverage below age 15 is + therefore recorded as unresolved rather than resolved either way. +- `OI_YN`'s printed zero label is `none or niu` where `PEN_YN`, `ANN_YN`, + `DST_YN` and `RNT_YN` print `niu`, so a zero receipt literal is not + interchangeable across families. +- `DST_SC1` is gated on `DST_VAL1 > 0 and a_age ≥ 58` while `DST_SC1_YNG` is + gated on `DST_YN_YNG = 1 and a_age < 58` — the two routes are not symmetric. + +## Amount semantics from the money owner + +The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored +zero and records `CodebookStatus.DECLARED_NIU`. `amount_state` therefore reads +the dollar meaning of a cell from the parent's status axis, never from the +stored number: re-deriving NIU from the number would read that cell as a zero +dollar annuity. `evidence["declared_niu_normalized_to_zero"]` lists the affected +entries. + +The money owner restates non-2024 cohorts to the pinned price basis, so the +literal-identity join is taken only on `person_years == 2024`; widening it to +the pooled 2022/2023 cohorts would require handling that restatement first. +`evidence["restatement_note"]` records this. + +## Allocation provenance + +`*_allocation_origin` is derived from published flags only: + +- `publisher_allocated` — some applicable flag is non-zero. +- `publisher_no_allocation` — every applicable flag is populated and zero, and + every field in the family carries a published flag. +- `publisher_no_allocation_on_published_flags_only` — the same, but the family + also holds a field the dictionary does not flag. +- `allocation_flag_not_populated` — a flag literal is absent. Every flag here + prints a conditional universe, so this is not a defect and not "no + allocation". +- `unresolved_allocation_provenance` — a flag literal is malformed or outside + its printed range. + +`PUBLISHED_ALLOCATION_FLAG_BY_FIELD` names which flag covers which field. +`UNFLAGGED_FIELDS` records the nine fields for which the 2025 dictionary +publishes no flag: `PNSN_VAL`, `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `OI_YN`, +`DST_VAL1_YNG`, `DST_VAL2_YNG`, `DST_YN_YNG`, `DST_SC1_YNG`, `DST_SC2_YNG`. +`AMBIGUOUS_FLAG_COVERAGE` records that `I_DSTSCCOMP`'s label names `DST_SC(2)` +while its universe names both routes, so its coverage of the under-58 source +codes is unresolved. No row is ever labelled a raw respondent value. + +## Constants + +The nine printed money entries are **not** retyped here: they are read on demand +from the packaged `asec_current_money_domains_v1.json` under its pinned digest +(`money.RESOURCE_PINS[0]`), which already attests printed position, length, +page, range, universe and values per vintage. The routing code systems +(`RECEIPT_CODES`, `ACCOUNT_CODES`, `OTHER_INCOME_CATEGORIES`, +`ALLOCATION_ENTRIES`) are not in that artifact and are defined here from the +pinned dictionary. + +Two adjacent definitions exist in tree and are deliberately **not** imported, so +this qualifier keeps the accepted source-only import neighbourhood +(`asec_coverage_authentication`, `asec_current_money`, `source_csv_builtin`, +`survey_population_preparation`, `support_provenance`) and stays free of the +modelled stage machinery: + +- `retirement_distributions.US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS` + — the same eight `DST_SC*`/`DST_VAL*` column names. +- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` — the same category code 20. + +`test_routing_code_systems_agree_with_the_existing_domain_constants` asserts +both agreements, per `docs/shared-constants.md` rule 5. If a later change makes +the modelled modules importable from a source qualifier, import those +definitions directly and drop the local copies. + +`RINT_SC1`/`RINT_SC2` reuse the identical 0:7 retirement-account enumeration for +retirement *interest*. They are out of this slice; a future consumer mapping +account codes to outputs must decide whether they share this code system. + +## Remaining work + +This lane delivers source qualification only. Still open, and explicitly not +decided here: + +1. **Canonical attachment.** Which projected columns become graph leaves, on + which clone, and under what name. These families overlap PUF-owned values, so + the unemployment/health both-clone policy cannot simply be reused. Root's + active amount owner owns `graph_us_survey_enrichment`. +2. **ACS clone0 components.** ACS publishes aggregate anchors + (`acs_retirement_income`, `acs_interest_dividend_rental_income`) rather than + these components. A conditional model plus a documented reconciliation rule + is needed, and the existing interest/dividend draws and ACS `INTP` overlap + mean rent cannot be added independently without checking that aggregate. +3. **Tax composition.** The pension private/taxable split and the distribution + taxable fraction remain unobserved. Any split must be approved explicitly + against the source, not inherited from the archived assumptions. +4. **Net property decomposition.** The receipt/amount scope mismatch must be + resolved before rent, royalties, roomers/boarders and estate/trust income are + modelled as separate components. +5. **Other-income residual.** Categories other than 20 are preserved but + unrouted; a residual-to-miscellaneous rule is a separate, reviewed decision. From d19e97602d177ee5729b5fe007edafe1a45d9fad Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:42:56 -0400 Subject: [PATCH 139/255] Record the income routing lane outcome and review corrections Co-Authored-By: Claude Opus 5 --- PROGRESS-current-asec-income-routing.md | 42 ++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md index adde17838..4ea5f5494 100644 --- a/PROGRESS-current-asec-income-routing.md +++ b/PROGRESS-current-asec-income-routing.md @@ -33,14 +33,46 @@ issuer, no engine execution, no native cell writes, no release claim. verbatim Universe and Values) from the verified dictionary, and cross-checked each against `asec_current_money_domains_v1.json`: exact match on position, length, universe and values for all nine money fields. -- [ ] Module `current_asec_income_routing_source.py` -- [ ] Tests `test_us_current_asec_income_routing.py` -- [ ] Source contract note + changelog fragment -- [ ] Ruff, CI inventory `--verify`, bounded pytest +- [x] Module + `packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py` + (commit 12ac3a75b, corrected in aa7bde7d4 and 7e2c7e77b). +- [x] Tests `packages/microcosm-build/tests/test_us_current_asec_income_routing.py` + — 64 passing, no engine, no PUF fixture. +- [x] Source contract note `docs/us-current-asec-income-routing-source.md` + plus `changelog.d/us-current-asec-income-routing-source.added.md`. +- [x] `ruff check .` and `ruff format --check` clean; + `tools/ci_test_groups.py --verify` reports `verification=ok` and the new + file lands in `[fast] rest` and `[engine] us-am`, not `[defaulted]`. +- [x] Mutation-checked the regressions: default-zero completion of an ambiguous + recipient zero, reading NIU from the raw number instead of the parent + status axis, collapsing receipt-with-net-zero into known nonreceipt, and + resolving an unknown slot account into a known non-IRA zero each turn the + suite red; reverting each returns it green. + +## Corrections made after independent review of the first draft + +1. The money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and + records `DECLARED_NIU` (`asec_current_money.py:974-977`). The first draft + re-derived NIU from the stored number, which would have read that cell as a + zero dollar annuity and also failed the literal-identity join. The amount + reading now comes from the parent's status axis. +2. The nine printed money entries were retyped by hand; they are already + attested per vintage in `asec_current_money_domains_v1.json`. They are now + read from that packaged artifact under `money.RESOURCE_PINS[0]`. +3. The published-allocation roster wrongly listed `DST_VAL1`, `DST_VAL2`, + `DST_YN`, `DST_SC1`, `DST_SC2` and `FRMOTR` as unflagged; all six are + flagged. `OI_YN` is unflagged and was missing from the list. +4. Three printed universes were transcribed with ASCII `>=` where the + dictionary prints U+2265; they now carry the printed character. ## Next -Write the qualifier and its invented-source tests; run red-then-green. +Root review and integration into `graph_us_survey_enrichment`. The open +decisions are listed under "Remaining work" in the contract note: canonical +attachment and clone policy for these PUF-overlapping leaves, the ACS clone0 +conditional model and its reconciliation against the ACS aggregate anchors, the +unobserved pension and distribution tax composition, the net property +decomposition, and any other-income residual rule. ## Verified source literals (2025 dictionary, pages 43-49 + allocation pages) From c859e34e8e464638791f4650b5abb6f4c6e916ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:49:14 -0400 Subject: [PATCH 140/255] Retain both distribution recipiency literals with the age route The age route selects which of DST_YN and DST_YN_YNG applies, but an answered off-route recipiency is a source contradiction rather than a missing answer. Keep both literals as typed columns and flag an off-route answer beside the existing off-route amount flag, so the complete slot and receipt pattern stays inspectable. Co-Authored-By: Claude Opus 5 --- docs/us-current-asec-income-routing-source.md | 8 +++ .../current_asec_income_routing_source.py | 25 ++++++++++ .../test_us_current_asec_income_routing.py | 50 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index 81ab7950f..90db39d52 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -112,6 +112,14 @@ judgments and recorded verbatim under - `DST_SC1` is gated on `DST_VAL1 > 0 and a_age ≥ 58` while `DST_SC1_YNG` is gated on `DST_YN_YNG = 1 and a_age < 58` — the two routes are not symmetric. +Both printed recipiency literals are retained per row +(`retirement_distribution_receipt_58_*` and `..._receipt_young_*`) alongside all +four slots. The age route selects which pair applies; +`retirement_distribution_offroute_receipt` and +`retirement_distribution_offroute_nonzero` flag an answered recipiency or a +non-zero amount on the route that does not apply, so the complete slot/receipt +pattern stays inspectable rather than being reduced to the applicable half. + ## Amount semantics from the money owner The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 22ed7109d..3b007bb86 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -600,6 +600,31 @@ def _retirement_distribution(raw, ages): ira_slots.append(len(matched)) else: ira_slots.append(None) + # Both printed recipiency literals are retained. The route selects which one + # applies; the other stays inspectable, because an answered off-route + # recipiency is a source contradiction rather than a missing answer. + route_codes = {} + for suffix, field in (("58", "DST_YN"), ("young", "DST_YN_YNG")): + frame, codes, statuses = _codes_frame( + "retirement_distribution_receipt_" + suffix, + raw[field], + RECEIPT_CODES, + RECEIPT_CODES, + ) + out = pd.concat([out, frame], axis=1) + route_codes[field] = (codes, statuses) + offroute_receipt = [ + ( + route_codes["DST_YN_YNG" if route[i] == "age58_and_over" else "DST_YN"][0][ + i + ] + not in (0, None) + ) + for i in range(rows) + ] + out["retirement_distribution_offroute_receipt"] = pd.array( + offroute_receipt, dtype="boolean" + ) receipt_tokens = [ raw["DST_YN"][i] if route[i] == "age58_and_over" else raw["DST_YN_YNG"][i] for i in range(rows) diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index d9713321b..885f8bd97 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -735,3 +735,53 @@ def test_routing_code_systems_agree_with_the_existing_domain_constants(): assert set(owner.PUBLISHED_ALLOCATION_FLAG_BY_FIELD.values()) <= set( owner.ALLOCATION_ENTRIES ) + + +def test_both_distribution_recipiency_literals_are_retained_with_the_route(): + n = 2 + raw = { + "amounts": {name: np.zeros(n) for name in owner.AMOUNT_FIELDS}, + "statuses": { + name: np.full(n, int(owner.money.CodebookStatus.ZERO_NONE_OR_NIU), "u1") + for name in owner.AMOUNT_FIELDS + }, + "allocations": { + name: ([0] * n, ["in_printed_range"] * n) + for name in owner.ALLOCATION_ENTRIES + }, + } + for name in (*owner.RECEIPT_ENTRIES, *owner.ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ["0"] * n + # A 70 year old whose under-58 recipiency answers yes, and a 40 year old + # whose 58-and-over recipiency answers yes: both are off-route answers. + raw["DST_YN_YNG"] = ["1", "0"] + raw["DST_YN"] = ["0", "1"] + out = owner.project_income_routing(raw, np.array([70.0, 40.0])) + assert out.retirement_distribution_route.tolist() == [ + "age58_and_over", + "under_age58", + ] + assert out.retirement_distribution_receipt_58_literal.tolist() == ["0", "1"] + assert out.retirement_distribution_receipt_young_literal.tolist() == ["1", "0"] + assert out.retirement_distribution_offroute_receipt.tolist() == [True, True] + # The applicable literal still drives the reporting status. + assert out.retirement_distribution_reporting_status.tolist() == ["niu", "niu"] + # A clean pair reports no off-route answer. Here each person answers yes on + # their own route with every applicable slot at the printed "none or niu" + # zero, which stays ambiguous rather than becoming a known zero. + raw["DST_YN_YNG"], raw["DST_YN"] = ["0", "1"], ["1", "0"] + clean = owner.project_income_routing(raw, np.array([70.0, 40.0])) + assert clean.retirement_distribution_offroute_receipt.tolist() == [False, False] + assert clean.retirement_distribution_reporting_status.tolist() == [ + "ambiguous_recipient_zero", + "ambiguous_recipient_zero", + ] + assert clean.retirement_distribution_known_amount.isna().all() + # Answering no on the applicable route does give a known zero. + raw["DST_YN_YNG"], raw["DST_YN"] = ["0", "2"], ["2", "0"] + declined = owner.project_income_routing(raw, np.array([70.0, 40.0])) + assert declined.retirement_distribution_reporting_status.tolist() == [ + "known_nonreceipt", + "known_nonreceipt", + ] + assert declined.retirement_distribution_known_amount.tolist() == [0.0, 0.0] From a0cb39ff1a22daad0bcd2153a2003c272630decb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:49:53 -0400 Subject: [PATCH 141/255] Assert the income routing evidence receipt round-trips as JSON A consuming host records this receipt, so pin that it serializes unchanged and that it names the 2024 join year, the unmodelled ACS rows, the closed family roster and the pinned domains artifact digest. Co-Authored-By: Claude Opus 5 --- .../tests/test_us_current_asec_income_routing.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index 885f8bd97..c5d7cc05b 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -7,6 +7,7 @@ import ast import hashlib +import json import shutil from pathlib import Path @@ -177,6 +178,17 @@ def test_actual_invented_member_joins_the_real_preparation_by_native_keys( ) assert not qualified.evidence["source_admission_issued"] assert not qualified.evidence["release_eligible"] + # A host records this receipt, so it must round-trip as JSON unchanged. + receipt = json.dumps(qualified.evidence, sort_keys=True) + assert json.loads(receipt) == qualified.evidence + assert qualified.evidence["joined_person_years"] == [2024] + assert qualified.evidence["acs_channel_rows"] > 0 + assert not qualified.evidence["acs_components_modeled"] + assert set(qualified.evidence["families"]) == set(owner.FAMILIES) + assert ( + qualified.evidence["dictionary"]["amount_entries_source"]["sha256"] + == owner.money.RESOURCE_PINS[0] + ) prepared.checked_view() From e64d5a8850ab301b7e819a6b972f235c51688568 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:53:56 -0400 Subject: [PATCH 142/255] Emit target snapshots from the grouped Adam solver The merge alone left grouped runs uninstrumented: _optimize returns early into _optimize_grouped before reaching any of the snapshot hooks the ordinary Adam loop carries, so a caller passing target_snapshots to a grouped calibrate got exactly one closing snapshot and no trajectory. Forward the bound observer into _optimize_grouped and emit there, after the existing progress callback and before backward(), from the exact float32 estimate tensor the epoch's loss was already computed from. No _apply_constraint call, no problem.estimates call, no gated draw and no optimizer/projection step is added or moved, so a grouped run's returned weights, trajectory and RNG state are bit-identical with the observer on and off. Grouped Adam is a closing-state algorithm, so every grouped snapshot reports best_retained {available: false, epoch: null, loss: null} and the closing snapshot is stamped epoch == epochs. Retain-best detection at the final emitter is now read off the solver actually used rather than inferred from an empty iterate_selection_receipt, so a later change that populated a grouped receipt cannot make the reused emitter claim the grouped solver retained a best iterate. The receipt itself stays empty, as before. Grouped snapshots carry three bounded aggregate scalars in `selection` (rule, constraint_mode, grouped_preserve_zeros) mirroring options["iterate_selection"], because "this solver retains no best iterate" and "retain-best was switched off" are otherwise indistinguishable to a consumer. The private _post_projection_observer proof seam is untouched and stays entirely separate: its record-length weights, household IDs, group map and bound vector never reach a snapshot. The closing snapshot still runs before the stored-result admission, so the accepted-weight byte equality, fixed-zero and ordered-household-ID checks remain the last thing that happens. Co-Authored-By: Claude Opus 5 --- ...ouped-target-snapshot-integration.added.md | 1 + docs/calibration-target-snapshots.md | 43 ++++++++- .../src/microcosm/calibrate/solve.py | 94 ++++++++++++++++++- 3 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 changelog.d/grouped-target-snapshot-integration.added.md diff --git a/changelog.d/grouped-target-snapshot-integration.added.md b/changelog.d/grouped-target-snapshot-integration.added.md new file mode 100644 index 000000000..c5df41593 --- /dev/null +++ b/changelog.d/grouped-target-snapshot-integration.added.md @@ -0,0 +1 @@ +Instrument the grouped/fixed-zero Adam solver with the opt-in calibration target snapshot observer, reusing the estimate tensor each epoch's loss was computed from and the closing float64 diagnostics estimate. diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md index 5df3054ec..3afbae9f0 100644 --- a/docs/calibration-target-snapshots.md +++ b/docs/calibration-target-snapshots.md @@ -46,11 +46,50 @@ visible only after their complete bytes are written and synced, and an existing chunk cannot be overwritten. Retention and dropped-history counts are explicit. Use a fresh run directory unless intentionally adopting existing history. +## The grouped US solver + +The grouped/fixed-zero Adam path (`grouped_upper_bounds`, optionally +`grouped_preserve_zeros`) is instrumented at the same two seams and with the +same guarantees: the in-loop snapshot reads the exact float32 estimate tensor +the epoch's loss was computed from, and the closing `selected` snapshot reads +the same float64 `problem.estimates` the final diagnostics use. Enabling the +observer on a grouped run adds no matrix evaluation, no model evaluation and no +RNG advance, and returns bit-identical weights and trajectory. + +Two grouped specifics a consumer must read correctly: + +- Grouped Adam is a **closing-state** algorithm. It never runs the retain-best + rule, so every grouped snapshot carries + `best_retained = {"available": false, "epoch": null, "loss": null}` and the + closing snapshot is stamped `epoch == epochs`. That is not the same as an + ordinary run whose retain-best rule was switched off, so grouped snapshots + also carry bounded `selection` labels — `rule: "closing_state"`, + `constraint_mode: "grouped_upper_bounds"` and `grouped_preserve_zeros` — + which mirror the run's `options["iterate_selection"]`. +- In-loop snapshots are **pre-update loss evaluations**, the same convention the + ungrouped Adam loop uses. They are deliberately not the post-update projected + accepted vector that the private `_post_projection_observer` proof seam + reports at the same epoch number: those are different vectors, and producing + target totals for the accepted one would require an extra matrix evaluation. + That private seam carries record-length weights, household IDs, the group map + and the absolute-bound vector; none of it reaches a snapshot, and the public + codec rejects record-level key names outright. + +Grouped runs keep every existing guard: fixed zeros stay in full-population +coordinates, the accepted-weight byte equality and ordered-household-ID checks +still run *after* the closing snapshot sink, and the unsupported grouped modes +(scalar cap, conserved mass, prox, L0/exact-k, gate initialization) are still +refused before the optimizer is constructed. Snapshot support does not turn any +of them into a permitted numeric path. + This implements the solver and local storage portion of [issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Country-host wiring, staging upload, a dashboard consumer and native cadence/performance -acceptance remain separate work. The grouped US solver currently lives on the -launch integration branch and needs its own integration of this observer. +acceptance remain separate work. In particular +`us_runtime/graph_fiscal_dense_calibration.py` still calls `calibrate` without +an observer: a solver-side integration alone produces no snapshot files for +that path, and wiring it needs an explicit host-owned instrumentation seam +rather than a callback or filesystem path serialized into graph node params. The reviewed repair branch passed 307 calibration tests before the final strict null correction. All 52 snapshot tests then passed after that one-line correction, diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index 4d733bc0f..a402de65c 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -861,6 +861,7 @@ def _optimize( progress_callback=progress_callback, progress_context=progress_context, observer=_post_projection_observer, + snapshots=snapshots, preserve_zeros=grouped_preserve_zeros, ) if return_gate_open_probabilities: @@ -1145,6 +1146,24 @@ def _check_grouped_household_ids(frame: Frame, expected: tuple[int | str, ...]) raise ValueError("stored group household IDs must preserve the ordered IDs") +def _grouped_snapshot_selection(preserve_zeros: bool) -> dict[str, object]: + """The bounded aggregate labels identifying a grouped snapshot's solver. + + Grouped Adam is a closing-state algorithm: it never runs the retain-best + rule, so every grouped snapshot carries ``best_retained.available: False``. + Without a label saying which solver produced it, that is indistinguishable + from an ordinary run whose retain-best rule happened to be off, and a + consumer would read the two the same way. These are three JSON scalars + describing the run's mode -- no household IDs, no group membership, no + bounds vector, and nothing of record length. + """ + return { + "rule": "closing_state", + "constraint_mode": "grouped_upper_bounds", + "grouped_preserve_zeros": bool(preserve_zeros), + } + + def _optimize_grouped( matrix: torch.Tensor, targets: torch.Tensor, @@ -1163,9 +1182,20 @@ def _optimize_grouped( progress_callback: Callable[[dict[str, object]], None] | None, progress_context: Mapping[str, object] | None, observer: Callable[[dict[str, object]], None] | None, + snapshots: BoundTargetSnapshots | None = None, preserve_zeros: bool = False, ) -> tuple[np.ndarray, np.ndarray]: - """Free-mass Adam on positive coordinates, with full float64 accepted state.""" + """Free-mass Adam on positive coordinates, with full float64 accepted state. + + ``snapshots`` is the public aggregate observer (microcosm#908), off by + default and strictly separate from the private ``observer`` proof seam: + the latter carries record-length accepted weights, household IDs, the + group map and the bounds vector, none of which a public snapshot may ever + hold. Snapshots read the estimate tensor this epoch's loss was already + computed from, so enabling them adds no matrix evaluation, no model + evaluation and no RNG advance, and the returned weights and trajectory are + bit-identical to the same run with snapshots off. + """ if ( matrix.dtype != torch.float32 or targets.dtype != torch.float32 @@ -1232,6 +1262,9 @@ def observe(epoch: int, corrected: int) -> None: ) anchor_t = torch.tensor(anchor[active], dtype=torch.float64) trajectory = np.empty(epochs, dtype=np.float64) + selection_labels = ( + None if snapshots is None else _grouped_snapshot_selection(preserve_zeros) + ) for epoch in range(epochs): optimizer.zero_grad() # This is an internal candidate, not an accepted population vector. @@ -1268,6 +1301,32 @@ def observe(epoch: int, corrected: int) -> None: "loss": trajectory[epoch], } ) + if snapshots is not None and snapshots.emits(epoch + 1, epochs): + # The exact estimate tensor this epoch's loss was computed from, + # detached rather than recomputed: recomputing would add a matrix + # evaluation, and the truthful numerical basis for trajectory + # [epoch] is the float32 forward pass that produced it, even + # though the accepted state is float64. + # + # This is a PRE-update loss-evaluation snapshot, the same + # convention the ordinary Adam loop above uses. It is deliberately + # not the post-update projected accepted vector the private + # observer reports at observe(epoch + 1): those are different + # vectors, and producing target totals for the accepted one would + # need the extra evaluation this design forbids. + snapshots.emit( + estimate.detach().cpu().numpy(), + epoch=epoch + 1, + epochs=epochs, + iterate=ITERATE_CURRENT, + precision="float32", + loss=trajectory[epoch], + # Grouped Adam retains no best iterate at all, so this is the + # closed "no retain-best rule is running" triple, not a + # not-yet-recorded incumbent. + best_retained={"available": False, "epoch": None, "loss": None}, + selection=selection_labels, + ) total_loss.backward() optimizer.step() with torch.no_grad(): @@ -2073,6 +2132,16 @@ def calibrate( loss was already computed from. The closing ``selected`` snapshot describes the weights calibration actually returns. + ``grouped_upper_bounds`` runs are instrumented the same way and at + the same seams. Because grouped Adam is a closing-state algorithm, + every grouped snapshot reports ``best_retained.available: False`` + and carries the bounded ``selection`` labels ``rule``, + ``constraint_mode`` and ``grouped_preserve_zeros`` so a consumer + can tell "this solver retains no best iterate" from "retain-best + was switched off". The private ``_post_projection_observer`` proof + seam stays entirely separate: its record-length weights, household + IDs, group map and bounds never reach a snapshot. + Returns: A :class:`CalibrationResult` with the calibrated frame, per-target diagnostics, loss trajectory, and any skipped targets. @@ -2482,13 +2551,23 @@ def grouped_observer(payload: dict[str, object]) -> None: # read off the same float64 estimates the final diagnostics use — not # the last in-loop iterate, which the optimizer may have discarded in # favour of an earlier better one or changed by a closing projection. - selected_epoch = iterate_selection_receipt.get("selected_epoch") + # + # Grouped Adam is a closing-state algorithm. It returns the accepted + # vector of its last projection and never runs the retain-best rule, + # so its retain-best answer is read off the solver it actually used, + # not inferred from an empty receipt. Inferring it would make a later + # change that populated a grouped receipt silently claim the grouped + # solver had retained a best iterate. + grouped_run = grouped_upper_bounds is not None + selected_epoch = ( + None if grouped_run else iterate_selection_receipt.get("selected_epoch") + ) # A non-empty receipt means the optimizer ran the retain-best rule. It # records the epoch it selected, which IS the best iterate's epoch when # an earlier iterate won; when the closing iterate won, no separate best # epoch was recorded, so the snapshot says "retained, epoch unrecorded" # instead of inventing one. - retained_best = bool(iterate_selection_receipt) + retained_best = (not grouped_run) and bool(iterate_selection_receipt) best_is_earlier = ( retained_best and isinstance(selected_epoch, int) @@ -2513,7 +2592,14 @@ def grouped_observer(payload: dict[str, object]) -> None: else None ), }, - selection=dict(iterate_selection_receipt) or None, + # The grouped receipt stays empty by design, so the grouped + # selection identity is the bounded mode label instead. Both are + # aggregate scalars; neither carries record-level content. + selection=( + _grouped_snapshot_selection(grouped_preserve_zeros) + if grouped_run + else (dict(iterate_selection_receipt) or None) + ), ) effective_target_loss_weights = ( np.ones(problem.target_vector.shape, dtype=np.float64) From 1c6422369e3de9ace529d4cb3c5a7550013813d9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:54:44 -0400 Subject: [PATCH 143/255] Separate an unresolved other-income receipt from a reported category A printed category alongside a missing or unrecognized OI_YN literal was labelled a reported category, which let a row with an unreadable receipt read as reported alimony. That pair is now its own unresolved routing status and no longer sets the alimony flag. Co-Authored-By: Claude Opus 5 --- docs/us-current-asec-income-routing-source.md | 6 +++ .../current_asec_income_routing_source.py | 4 ++ .../test_us_current_asec_income_routing.py | 46 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index 90db39d52..6f7e214fe 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -62,6 +62,12 @@ around its own I/O, exactly as the accepted unemployment qualifier requires. - `OI_OFF` code 20 is the reported alimony category. Nothing maps any other code — including 19, `anything else` — onto alimony or onto a miscellaneous residual; `other_income_residual_rule_applied` is constant `False`. + `other_income_routing_status` separates `reported_category` from + `receipt_without_category`, `category_without_receipt`, + `unresolved_receipt_routing`, `missing_category_literal`, + `unrecognized_category_literal` and `niu_category`, and + `other_income_is_reported_alimony` is set only on a `reported_category` row, + so an unreadable receipt literal never yields a reported alimony receipt. - ACS-channel people are not projected at all. `acs_components_modeled` is `False` in the evidence and the projection covers only ASEC-channel rows. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 3b007bb86..003aaa754 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -800,6 +800,10 @@ def _other_income(raw, ages): routing.append("missing_category_literal") elif code is None: routing.append("unrecognized_category_literal") + elif codes[i] is None: + # The category is printed but the receipt literal cannot be read, so + # the pair is unresolved and the category is not a reported receipt. + routing.append("unresolved_receipt_routing") elif codes[i] == 1 and code == 0: routing.append("receipt_without_category") elif codes[i] in (0, 2) and code != 0: diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index c5d7cc05b..05f4168e8 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -797,3 +797,49 @@ def test_both_distribution_recipiency_literals_are_retained_with_the_route(): "known_nonreceipt", ] assert declined.retirement_distribution_known_amount.tolist() == [0.0, 0.0] + + +@pytest.mark.parametrize( + "receipt,category,routing,alimony", + [ + ("1", "20", "reported_category", True), + ("1", "19", "reported_category", False), + ("1", "0", "receipt_without_category", False), + ("2", "20", "category_without_receipt", False), + ("0", "20", "category_without_receipt", False), + ("2", "0", "niu_category", False), + ("", "20", "unresolved_receipt_routing", False), + ("9", "20", "unresolved_receipt_routing", False), + ("1", "", "missing_category_literal", False), + ("1", "21", "unrecognized_category_literal", False), + ("1", "NA", "unrecognized_category_literal", False), + ], +) +def test_other_income_routing_never_invents_a_reported_category( + receipt, category, routing, alimony +): + n = 1 + raw = { + "amounts": {name: np.zeros(n) for name in owner.AMOUNT_FIELDS}, + "statuses": { + name: np.full(n, int(owner.money.CodebookStatus.ZERO_NONE_OR_NIU), "u1") + for name in owner.AMOUNT_FIELDS + }, + "allocations": { + name: ([0] * n, ["in_printed_range"] * n) + for name in owner.ALLOCATION_ENTRIES + }, + } + for name in (*owner.RECEIPT_ENTRIES, *owner.ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ["0"] * n + raw["amounts"]["OI_VAL"] = np.array([1500.0]) + raw["statuses"]["OI_VAL"] = np.full( + n, int(owner.money.CodebookStatus.AMOUNT_NONZERO), "u1" + ) + raw["OI_YN"], raw["OI_OFF"] = [receipt], [category] + out = owner.project_income_routing(raw, np.array([45.0])) + assert out.other_income_routing_status.iloc[0] == routing + assert bool(out.other_income_is_reported_alimony.iloc[0]) is alimony + assert out.other_income_source_total.iloc[0] == 1500.0 + # The reported category never becomes a residual assignment. + assert not bool(out.other_income_residual_rule_applied.iloc[0]) From b19a5b6a8f56c0253ba7205845eb8ddbdd2b287c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:38:02 -0400 Subject: [PATCH 144/255] Qualify observed ACS income anchors from retained source --- ...-current-acs-income-anchor-source.added.md | 3 + .../us-current-acs-income-anchors-20260912.md | 67 +++ .../current_acs_income_anchor_source.py | 441 ++++++++++++++++++ ...est_us_current_acs_income_anchor_source.py | 310 ++++++++++++ 4 files changed, 821 insertions(+) create mode 100644 changelog.d/us-current-acs-income-anchor-source.added.md create mode 100644 experiments/us-current-acs-income-anchors-20260912.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py diff --git a/changelog.d/us-current-acs-income-anchor-source.added.md b/changelog.d/us-current-acs-income-anchor-source.added.md new file mode 100644 index 000000000..d8d6fce32 --- /dev/null +++ b/changelog.d/us-current-acs-income-anchor-source.added.md @@ -0,0 +1,3 @@ +Qualify original ACS property and retirement income anchors through the retained +survey source owner, preserving literal missingness, allocation, age universe and +signed values while verifying raw identities and adjusted amount bits. diff --git a/experiments/us-current-acs-income-anchors-20260912.md b/experiments/us-current-acs-income-anchors-20260912.md new file mode 100644 index 000000000..17fc09e1d --- /dev/null +++ b/experiments/us-current-acs-income-anchors-20260912.md @@ -0,0 +1,67 @@ +# Qualifying observed ACS income anchors + +`qualify_current_acs_income_anchors` reads the original ACS person archive through +the retained survey preparation owner. It returns the selected ACS people's +original `INTP`, `RETP`, `ADJINC`, `AGEP`, `FINTP`, and `FRETP` literals, joined by +the original `SERIALNO` and normalized `SPORDER`. Household serial numbers are +checked through the native person's household link. The selected native person +axis and the preparation's stacked person axis remain explicit. + +These are broad survey anchors. They are not separately observed tax inputs. + +| Anchor | Source meaning | Qualification | +| --- | --- | --- | +| `INTP` | Interest, dividends, net rental, royalty, estate and trust income over the preceding 12 months | Preserve signed values, published zero, blank and malformed literals separately. | +| `RETP` | Broad retirement, survivor and disability income over the preceding 12 months, excluding Social Security | Preserve the nonnegative source amount without choosing a pension or retirement-account decomposition. | +| `ADJINC` | Factor for expressing income in the release's dollar year | Match the mapper's multiplication and division order and every retained nonmissing float64 bit. This adjustment does not convert the rolling reference period into a calendar year. | +| `AGEP` | Original source age | Use the original age for the age-15 income universe, before any downstream top-code mapping. | +| `FINTP`, `FRETP` | Allocation flags | Keep allocation separate from amount validity. A valid allocated amount remains known and explicitly labeled. | + +The [2024 ACS questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf), +[2024 PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +(printed pages 38, 43, 127 and 129), and +[2024 subject definitions](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +establish these meanings. The released INTP domain contains zero, -10,000 through +-4, and 4 through 999,999; RETP contains zero and 4 through 999,999. Those disclosure +bounds describe released records, not bounds on latent income. + +The output distinguishes observed, missing, malformed, outside-domain, invalid +adjustment, and outside-universe records. It supplies no analytical zero for an +under-15 person or for an adult with a blank amount. The original literal survives +every classification, including missing or unrecognized allocation flags. Numeric +coercion reproduces the existing source mapper only for the retained-storage +comparison; it never supplies observation knownness. + +The qualifier exhausts a privately captured archive, checks its recorded hash and +complete source row count, and compares selected raw fields and adjusted anchors +to the retained native frame. After capture cleanup, it rechecks the actual +preparation and catalogue owners. A final physical seal also detects mutations +that table JSON's decimal precision cannot represent. Returned values, projection +bytes and evidence are descriptive: they cannot replace the original preparation +or grant source authority. A consuming graph host must retain that owner and the +physical value seal, check them immediately before consumption, and requalify +after its last relevant I/O before returning a successor. + +The 35-case invented-source suite passed with no failures, errors or skips in +56.261 seconds wall time (55.239 CPU seconds; 487,407,616 bytes peak RSS). It covers +the real preparation issuer, household and group-quarter records, selected-row +reordering and ID changes, signed and missing amounts, allocation states, exact +adjustment bits, a copied preparation, source mutation after capture I/O, and a +returned amount mutation hidden by JSON precision. The guarded run used 982 +source files and 15 allowed resources, with all 990 source-plus-owned hashes and +all resource hashes unchanged, no children, no unexpected refusals, and Torch +threads fixed at 1/1. The normal guard-induced dateutil zoneinfo warning occurred. + +The source SHA256 is +`94cb447fe209e8b801cd6080151a93c80bfb06bb736b068affb44d9d595c50a2`; +the test SHA256 is +`9bffafce6cdf04d63940e15052306037e0bee31049996ec561d46d8cb9e1a2bc`. +The local `codex-acs-income-anchor-20260912/source-v4/source-v4.json` receipt SHA256 +is `9154b624cf98def81bcf84fb814faf72d4a866d29243eacbeef12a6607c737b6`. +Earlier failed and superseded fixture evidence remains preserved. + +This change does not fit or reconcile components, add an enrichment host, choose +between ASEC interest definitions, or resolve the retirement donor bridge. It +does not execute native microdata, a country engine or a model. Those steps need +their own source and graph acceptance before this qualification can support a +release candidate. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py new file mode 100644 index 000000000..b206f709c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py @@ -0,0 +1,441 @@ +"""Observed ACS income anchors from the retained original source archive. + +Values and projections are descriptive, not source issuers. Consumers retain the +original preparation and requalify after their last relevant I/O. This module +does not decompose income, create tax inputs, or fill any analytical zeros. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import graph_full_puf_enrichment as physical +from . import source_csv_builtin +from . import support_provenance as provenance +from . import survey_population_preparation as preparation + +PROTOCOL = "microcosm.us.current-acs-income-anchors.v1" +COLUMNS = ("SERIALNO", "SPORDER", "INTP", "RETP", "ADJINC", "AGEP", "FINTP", "FRETP") +ANCHORS = ( + ("INTP", "FINTP", "property_income", "acs_interest_dividend_rental_income"), + ("RETP", "FRETP", "retirement_income", "acs_retirement_income"), +) +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +def require(condition, reason): + if not condition: + raise ValueError("ACS_INCOME_ANCHOR_" + reason) + + +def _literal(value): + require(type(value) is str and len(value) <= 64, "LITERAL_TYPE_OR_BOUND") + return value + + +def parse_anchor( + token: str, *, age: str, adjustment: str, allocation: str, field: str +) -> dict: + """Classify source literals separately from native numeric storage. + + Allocation is provenance, not amount validity. Published domains describe + released records; their disclosure bounds are not latent income bounds. + """ + require(field in ("INTP", "RETP"), "FIELD") + token, age, adjustment, allocation = map( + _literal, (token, age, adjustment, allocation) + ) + require(re.fullmatch(r"[0-9]{1,2}", age, re.ASCII) is not None, "AGE_LITERAL") + parsed_age = int(age) + factor_valid = ( + re.fullmatch(r"[0-9]{1,7}", adjustment, re.ASCII) is not None + and int(adjustment) > 0 + ) + status, amount = "observed", None + if token == "": + status = ( + "outside_universe_blank" if parsed_age < 15 else "missing_source_amount" + ) + elif re.fullmatch(r"-?[0-9]+", token, re.ASCII) is None: + status = "malformed_source_amount" + else: + raw = int(token) + in_domain = ( + raw == 0 or 4 <= raw <= 999999 or (field == "INTP" and -10000 <= raw <= -4) + ) + if not in_domain: + status = "outside_published_domain" + elif parsed_age < 15: + status = "outside_universe_observation" + elif not factor_valid: + status = "invalid_adjustment" + else: + amount = float( + np.float64(raw) * (np.float64(int(adjustment)) / 1_000_000.0) + ) + return { + "amount": amount, + "known": status == "observed", + "status": status, + "allocation_status": { + "0": "not_allocated", + "1": "allocated", + "": "allocation_missing", + }.get(allocation, "allocation_unrecognized"), + "adjustment_known": factor_valid, + "in_income_universe": parsed_age >= 15, + } + + +def _key(row): + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "SOURCE_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "SOURCE_PERSON_KEY", + ) + return row["SERIALNO"], int(row["SPORDER"]) + + +def _scan(stream, wanted, selected, *, maximum): + require(source_csv_builtin.capture_csv_reader(csv) is not None, "CSV_BINDING") + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in COLUMNS] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(COLUMNS, (values[p] for p in positions), strict=True)) + require(all(len(v) <= 64 for v in row.values()), "SOURCE_TOKEN_BOUND") + key = _key(row) + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(state, document): + payload = document["origins"]["persons"] + full = pd.DataFrame(payload["rows"], columns=payload["columns"]) + people = state.frame.person + require( + full.person_id.dtype == np.dtype("int64") + and full.person_id.is_unique + and np.array_equal(full.person_id.to_numpy(), people.person_id.to_numpy()), + "ORIGIN_AXIS", + ) + channel = provenance.support_channel_column("person") + native_id = provenance.spine_source_id_column("person") + require( + np.array_equal(full.source.to_numpy(), people[channel].to_numpy()) + and np.array_equal( + full.selected_receiving_person_id.to_numpy(), people[native_id].to_numpy() + ), + "ORIGIN_FRAME_IDENTITY", + ) + origins = full.loc[full.source.eq("acs")].copy().set_index("person_id", drop=True) + require( + len(origins) > 0 + and origins.source_year.eq(2024).all() + and origins.survey_year.eq(2024).all(), + "ORIGIN_PERIOD", + ) + native = state.source_frames[0].person + require( + np.array_equal( + origins.selected_receiving_person_id.to_numpy(), native.person_id.to_numpy() + ), + "ORIGIN_NATIVE_AXIS", + ) + keys = [] + for _, row in origins.iterrows(): + require( + all( + type(v) is str + for v in ( + row.raw_native_household_id, + row.raw_native_person_id, + row.native_line_numeric_original, + ) + ), + "ORIGIN_LITERAL_TYPE", + ) + key = _key( + { + "SERIALNO": row.raw_native_household_id, + "SPORDER": row.native_line_numeric_original, + } + ) + require(row.raw_native_person_id == str(key[1]), "ORIGIN_NATIVE_PERSON_KEY") + keys.append(key) + require(len(set(keys)) == len(keys), "ORIGIN_DUPLICATE") + origins["anchor_source_key"] = keys + return origins + + +def _raw_table(origins, selected): + require(set(selected) == set(origins.anchor_source_key), "SELECTED_SOURCE_ROSTER") + raw = pd.DataFrame( + [selected[k] for k in origins.anchor_source_key], + index=origins.index.copy(), + columns=COLUMNS, + dtype=object, + ) + require( + all( + _key(row) == key + for row, key in zip( + raw.to_dict("records"), origins.anchor_source_key, strict=True + ) + ), + "SOURCE_COORDINATE_CHANGED", + ) + return raw + + +def _native_numbers(tokens): + # Match the existing mapper's numeric-coercion semantics for identity only; + # this operation does not determine source amount knownness. + return pd.to_numeric(pd.Series(tokens, dtype=object), errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + + +def _same_number(left, right): + if pd.isna(left): + return bool(pd.isna(right)) + if isinstance(left, (bool, np.bool_)) or not isinstance( + left, (int, float, np.integer, np.floating) + ): + return False + return np.float64(left).view("uint64") == np.float64(right).view("uint64") + + +def _compare_retained(raw, origins, native): + people = native.person + require( + raw.index.equals(origins.index) + and np.array_equal( + origins.selected_receiving_person_id.to_numpy(), people.person_id.to_numpy() + ), + "RETAINED_AXIS", + ) + households = native.table("household").set_index("household_id", drop=False) + require( + households.index.is_unique + and np.array_equal( + origins.selected_receiving_household_id.to_numpy(), + people.person_household_id.to_numpy(), + ), + "RETAINED_HOUSEHOLD_ID", + ) + serials = households.SERIALNO.reindex(people.person_household_id.to_numpy()) + require( + np.array_equal(raw.SERIALNO.to_numpy(), serials.to_numpy()) + and np.array_equal(raw.SPORDER.map(int).to_numpy(), people.SPORDER.to_numpy()), + "RETAINED_SOURCE_ID", + ) + numbers = {c: _native_numbers(raw[c]) for c in ("INTP", "RETP", "ADJINC", "AGEP")} + for column, expected_numbers in numbers.items(): + for literal, stored, expected in zip( + raw[column], people[column], expected_numbers, strict=True + ): + same = ( + stored == literal + if type(stored) is str + else _same_number(stored, expected) + ) + require(bool(same), "RETAINED_RAW_" + column) + adjustment = numbers["ADJINC"] + for column, _flag, _prefix, output in ANCHORS: + require(people[output].dtype == np.dtype("float64"), "RETAINED_ADJUSTED_DTYPE") + expected = numbers[column] * (adjustment / 1_000_000.0) + actual = people[output].to_numpy(copy=False) + require( + np.array_equal(np.isnan(expected), np.isnan(actual)) + and np.array_equal( + expected[~np.isnan(expected)].view("uint64"), + actual[~np.isnan(actual)].view("uint64"), + ), + "RETAINED_ADJUSTED_BITS", + ) + + +def _parsed_table(raw, origins): + result = raw.copy(deep=True) + result["native_person_id"] = origins.selected_receiving_person_id.to_numpy( + copy=True + ) + result["source_year"] = 2024 + result["dollar_year"] = 2024 + for column, flag, prefix, _output in ANCHORS: + values = [ + parse_anchor( + r[column], + age=r["AGEP"], + adjustment=r["ADJINC"], + allocation=r[flag], + field=column, + ) + for r in raw.to_dict("records") + ] + parsed = pd.DataFrame(values, index=raw.index) + for name in parsed: + dtype = ( + "Float64" + if name == "amount" + else ( + bool + if name in ("known", "adjustment_known", "in_income_universe") + else STRING_DTYPE + ) + ) + result[prefix + "_" + name] = pd.array(parsed[name], dtype=dtype) + return result + + +@dataclass(frozen=True) +class QualifiedAcsIncomeAnchors: + """Detached source observations, not a substitute for the preparation.""" + + anchors: pd.DataFrame + projection: bytes + evidence: dict + + +def income_anchor_seal(qualified: QualifiedAcsIncomeAnchors) -> tuple: + require(type(qualified) is QualifiedAcsIncomeAnchors, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.anchors.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + physical._table_stamp(qualified.anchors), + qualified.projection, + json.dumps( + qualified.evidence, sort_keys=True, separators=(",", ":"), allow_nan=False + ), + ) + + +def _capture_person(root, pin, wanted, total_rows): + """Return literal values from an owned capture, after cleanup I/O completes. + + Caller-supplied pins grant no authority here; the qualifier binds them to + the retained catalogue and rechecks that original owner after this returns. + """ + _role, name, digest, size = pin + selected, count = {}, 0 + with tempfile.TemporaryDirectory( + prefix="microcosm-acs-income-anchor-" + ) as directory: + captured = Path(directory) / name + require( + housing._copy(root / "acs" / name, captured, size, exact_size=size) + == digest, + "ACS_CAPTURE_DIGEST", + ) + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + wanted, + selected, + maximum=total_rows - count, + ) + else: + while stream.read(65536): + pass + require( + count == total_rows and housing._persisted_sha(captured, size) == digest, + "ACS_CAPTURE_CHANGED", + ) + return selected + + +def qualify_current_acs_income_anchors(source_preparation) -> QualifiedAcsIncomeAnchors: + """Read exact original ACS members and recheck the real owner before return.""" + require( + type(source_preparation) + is preparation.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = source_preparation._checked() + state, document = entry[2], json.loads(entry[1]) + origins = _origins(state, document) + owned = preparation.acs_catalogue._lookup(state.catalogues[0]) + catalogue = json.loads(owned.receipt) + pins = tuple(p for p in owned.pins if p[0] == "person") + require( + len(pins) == 1 and catalogue["source_year"] == catalogue["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + selected = _capture_person( + state.root, + pins[0], + set(origins.anchor_source_key), + catalogue["counts"]["people"], + ) + raw = _raw_table(origins, selected) + _compare_retained(raw, origins, state.source_frames[0]) + anchors = _parsed_table(raw, origins) + projection = anchors.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(owned.receipt).hexdigest(), + "acs_person_archive_sha256": pins[0][2], + "selected_persons": len(anchors), + "source_year": 2024, + "dollar_year": 2024, + "income_period": "rolling_12_months", + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "analytical_universe_zeros_supplied": False, + "decomposition_performed": False, + "release_eligible": False, + } + result = QualifiedAcsIncomeAnchors(anchors, projection, evidence) + seal = income_anchor_seal(result) + require(source_preparation._checked() is entry, "PREPARATION_CHANGED") + preparation._pure_final(state) + require( + preparation._ISSUED.get(id(source_preparation)) is entry + and preparation.acs_catalogue._lookup(state.catalogues[0]) is owned, + "FINAL_OWNER", + ) + require(income_anchor_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py new file mode 100644 index 000000000..00d1c2269 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py @@ -0,0 +1,310 @@ +"""Actual retained ACS source owners over privately pinned invented bytes.""" + +import copy +import io + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_acs_income_anchor_source as owner + + +@pytest.mark.parametrize( + "field,token,age,status,known", + [ + ("INTP", "-10000", "30", "observed", True), + ("INTP", "-4", "30", "observed", True), + ("INTP", "0", "30", "observed", True), + ("RETP", "0", "30", "observed", True), + ("RETP", "999999", "30", "observed", True), + ("INTP", "-3", "30", "outside_published_domain", False), + ("RETP", "-4", "30", "outside_published_domain", False), + ("INTP", "1000000", "30", "outside_published_domain", False), + ("INTP", "", "14", "outside_universe_blank", False), + ("INTP", "", "15", "missing_source_amount", False), + ("RETP", "4", "14", "outside_universe_observation", False), + ("INTP", "4.0", "30", "malformed_source_amount", False), + ("INTP", " 4", "30", "malformed_source_amount", False), + ("INTP", "NA", "30", "malformed_source_amount", False), + ], +) +def test_published_anchor_domain_is_distinct_from_storage_and_universe( + field, token, age, status, known +): + value = owner.parse_anchor( + token, age=age, adjustment="1000133", allocation="1", field=field + ) + assert value["status"] == status and value["known"] is known + assert value["allocation_status"] == "allocated" + if known: + expected = np.float64(int(token)) * (np.float64(1000133) / 1_000_000.0) + assert np.float64(value["amount"]).view("uint64") == expected.view("uint64") + else: + assert value["amount"] is None + + +@pytest.mark.parametrize( + "flag,status", + [ + ("0", "not_allocated"), + ("1", "allocated"), + ("", "allocation_missing"), + ("x", "allocation_unrecognized"), + ], +) +def test_allocation_does_not_decide_amount_knownness(flag, status): + value = owner.parse_anchor( + "4", age="15", adjustment="1000133", allocation=flag, field="RETP" + ) + assert value["known"] and value["allocation_status"] == status + + +@pytest.mark.parametrize("adjustment", ["", "0", "-1", "1.1", "inf"]) +def test_unknown_or_invalid_adjustment_never_supplies_an_amount(adjustment): + value = owner.parse_anchor( + "4", age="15", adjustment=adjustment, allocation="0", field="INTP" + ) + assert not value["known"] and value["amount"] is None + assert value["status"] == "invalid_adjustment" + + +def test_column_numeric_identity_keeps_scalar_mapper_bits_and_unknowns(): + tokens = ["-800", "0", "bad", "", "4.0", "-0.0", "NA", "999999", "1000133"] + expected = np.array( + [ + pd.to_numeric(pd.Series([v], dtype=object), errors="coerce").iloc[0] + for v in tokens + ], + dtype=np.float64, + ) + actual = owner._native_numbers(tokens) + assert np.array_equal(np.isnan(actual), np.isnan(expected)) + assert np.array_equal( + actual[~np.isnan(actual)].view("uint64"), + expected[~np.isnan(expected)].view("uint64"), + ) + + +def _literal_rows(): + return [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": "1", + "INTP": "-800", + "RETP": "444", + "ADJINC": "1000133", + "AGEP": "30", + "FINTP": "0", + "FRETP": "1", + }, + { + "SERIALNO": "2024HU0000001", + "SPORDER": "2", + "INTP": "0", + "RETP": "", + "ADJINC": "1000133", + "AGEP": "15", + "FINTP": "1", + "FRETP": "", + }, + { + "SERIALNO": "2024HU0000002", + "SPORDER": "1", + "INTP": "bad", + "RETP": "4", + "ADJINC": "1000133", + "AGEP": "80", + "FINTP": "", + "FRETP": "x", + }, + *( + { + "SERIALNO": serial, + "SPORDER": "1", + "INTP": "0", + "RETP": "0", + "ADJINC": "1000000", + "AGEP": age, + "FINTP": "0", + "FRETP": "0", + } + for serial, age in (("2024GQ0000001", "40"), ("2024GQ0000002", "50")) + ), + ] + + +def _scan(rows, *, wanted=None): + import csv + + text = io.StringIO(newline="") + writer = csv.DictWriter(text, fieldnames=owner.COLUMNS) + writer.writeheader() + writer.writerows(rows) + selected = {} + wanted = ( + wanted + if wanted is not None + else {(r["SERIALNO"], int(r["SPORDER"])) for r in rows} + ) + count = owner._scan( + io.BytesIO(text.getvalue().encode()), wanted, selected, maximum=100 + ) + return count, selected + + +def test_literal_scan_is_keyed_and_keeps_every_original_token(): + count, values = _scan(_literal_rows()[::-1]) + assert count == 5 + assert values["2024HU0000001", 2]["RETP"] == "" + assert values["2024HU0000002", 1]["INTP"] == "bad" + + +@pytest.mark.parametrize("defect", ["duplicate", "foreign_bad_key", "short_row"]) +def test_literal_reader_refuses_duplicate_selected_and_invalid_tail(defect): + rows = _literal_rows() + if defect == "duplicate": + rows.append(rows[0]) + elif defect == "foreign_bad_key": + rows.append({**rows[0], "SERIALNO": "bad"}) + else: + stream = io.BytesIO((",".join(owner.COLUMNS) + "\n1,2\n").encode()) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._scan(stream, set(), {}, maximum=100) + return + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + _scan(rows) + + +def _arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + + original = fixture._person + lookup = {(r["SERIALNO"], int(r["SPORDER"])): r for r in _literal_rows()} + + def person(*args, **kwargs): + row = original(*args, **kwargs) + key = (row["SERIALNO"], int(row["SPORDER"])) + row.update( + { + k: v + for k, v in lookup.get(key, {}).items() + if k not in ("SERIALNO", "SPORDER") + } + ) + row.setdefault("FINTP", "0") + row.setdefault("FRETP", "0") + return row + + monkeypatch.setattr(fixture, "_person", person) + return fixture.fixture(tmp_path, monkeypatch, zero=False) + + +def test_actual_qualifier_binds_originals_and_adjusted_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + before = owner.preparation._frame_identity(original) + qualified = owner.qualify_current_acs_income_anchors(prepared) + assert qualified.evidence["source_admission_issued"] is False + assert qualified.evidence["decomposition_performed"] is False + assert len(qualified.anchors) == 5 + first = qualified.anchors.set_index(["SERIALNO", "SPORDER"]) + assert first.loc[("2024HU0000001", "1"), "property_income_amount"] < 0 + assert first.loc[("2024HU0000001", "2"), "property_income_amount"] == 0 + assert ( + first.loc[("2024HU0000001", "2"), "retirement_income_status"] + == "missing_source_amount" + ) + assert ( + first.loc[("2024HU0000002", "1"), "property_income_status"] + == "malformed_source_amount" + ) + assert pd.isna(first.loc[("2024HU0000002", "1"), "property_income_amount"]) + assert owner.preparation._frame_identity(original) == before + fresh = owner.qualify_current_acs_income_anchors(prepared) + assert owner.income_anchor_seal(fresh) == owner.income_anchor_seal(qualified) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(copy.copy(prepared)) + + +@pytest.mark.parametrize( + "defect", ["row_order", "person_id", "raw_amount", "adjusted_bit"] +) +def test_exact_retained_comparison_refuses_source_or_amount_mutation( + tmp_path, monkeypatch, defect +): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + document = owner.json.loads(entry[1]) + origins = owner._origins(entry[2], document) + _, selected = _scan(_literal_rows()) + raw = owner._raw_table(origins, selected) + native = owner.preparation._copy_source(entry[2].source_frames[0]) + if defect == "row_order": + origins = origins.iloc[::-1].copy() + elif defect == "person_id": + native.person.loc[native.person.index[0], "person_id"] += 100 + elif defect == "raw_amount": + native.person.loc[native.person.index[0], "RETP"] = 888 + else: + column = "acs_interest_dividend_rental_income" + native.person.loc[native.person.index[0], column] = np.nextafter( + native.person[column].iloc[0], np.inf + ) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._compare_retained(raw, origins, native) + + +def test_final_source_io_mutation_cannot_return_qualified_value(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + capture = owner._capture_person + mutated = [] + + def mutate_after_io(*args): + value = capture(*args) + original.person.loc[original.person.index[0], "age"] += 1 + mutated.append(True) + return value + + monkeypatch.setattr(owner, "_capture_person", mutate_after_io) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(prepared) + assert mutated == [True] + + +def test_final_source_fence_seals_exact_returned_amount_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original_seal = owner.income_anchor_seal + changed = [] + + def change_after_seal(qualified): + seal = original_seal(qualified) + if not changed: + changed.append(True) + table = qualified.anchors + index = table.index[ + table.SERIALNO.eq("2024HU0000001") & table.SPORDER.eq("1") + ][0] + table.loc[index, "property_income_amount"] = np.nextafter( + table.loc[index, "property_income_amount"], np.inf + ) + # Portable JSON's precision is not an exact physical mutation seal. + assert ( + qualified.projection + == table.reset_index().to_json(orient="table", index=False).encode() + ) + return seal + + monkeypatch.setattr(owner, "income_anchor_seal", change_after_seal) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_acs_income_anchors(prepared) + assert changed == [True] From 8901ed3a140dfafa8aaaeb48682a7ec099782493 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:49:33 -0400 Subject: [PATCH 145/255] Add deterministic reconciliation to signed component totals --- .../signed-income-reconciliation.added.md | 2 + .../signed-income-reconciliation-20260912.md | 85 +++++++ .../microcosm/fit/signed_reconciliation.py | 188 ++++++++++++++ .../tests/test_signed_reconciliation.py | 240 ++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 changelog.d/signed-income-reconciliation.added.md create mode 100644 experiments/signed-income-reconciliation-20260912.md create mode 100644 packages/microcosm-fit/src/microcosm/fit/signed_reconciliation.py create mode 100644 packages/microcosm-fit/tests/test_signed_reconciliation.py diff --git a/changelog.d/signed-income-reconciliation.added.md b/changelog.d/signed-income-reconciliation.added.md new file mode 100644 index 000000000..428496f2c --- /dev/null +++ b/changelog.d/signed-income-reconciliation.added.md @@ -0,0 +1,2 @@ +Add a deterministic weighted projection onto a signed total with explicit +nonnegative component bounds, adjustment diagnostics and floating-point checks. diff --git a/experiments/signed-income-reconciliation-20260912.md b/experiments/signed-income-reconciliation-20260912.md new file mode 100644 index 000000000..4d5b6453e --- /dev/null +++ b/experiments/signed-income-reconciliation-20260912.md @@ -0,0 +1,85 @@ +# Deterministic reconciliation to a signed income total + +`microcosm.fit.signed_reconciliation.reconcile_signed_total` projects a joint +draw onto a caller-qualified total. It is a separate numeric operation for a +future graph node. No existing model, source qualifier, financial attachment, +PUF host or native candidate invokes it in this change. + +For draw `q`, anchor `A`, strictly positive scales `s`, and a declared set `B` of +nonnegative components, it solves the strictly convex problem + +```text +minimize sum_j ((z_j - q_j) / s_j)^2 +subject to sum_j z_j = A + z_j >= 0 for j in B +``` + +At least one component must remain unrestricted, making every finite signed +anchor feasible in exact arithmetic. Component names, the bound mask and scales +are explicit parameters. Ordinary interest and retirement-account interest can +therefore be separate components; this operator assigns neither a source meaning +nor tax treatment to either. It introduces no nonzero-presence restriction. + +Define `w_j = (s_j / max(s))^2`. Uniform scale normalization preserves the +minimizer. Given an active bound set `C`, the equality solution for the remaining +free coordinates `F` is + +```text +t = (A - sum_{j in F} q_j) / sum_{j in F} w_j +z_j = q_j + t*w_j for j in F +z_j = 0 for j in C +``` + +Start with every coordinate free. Fix any bounded coordinate with a negative +candidate to zero, then solve again. Removing negative candidates can only lower +`t`, so a fixed coordinate cannot need releasing later. An unrestricted +coordinate always remains free. The algorithm therefore terminates after at +most `len(B) + 1` solves, without a general optimizer or random draws. The final +free-coordinate stationarity and active-bound dual feasibility are checked. + +The signed anchor is never clamped or used as a divisor. A zero total does not +force its components to zero. For example, `[8, 4, -3]` with unit scales and a +zero anchor becomes `[5, 1, -6]`. Positive income and offsetting property loss +remain possible for negative, zero or positive totals. An exactly feasible draw +is preserved, including its float64 signed-zero bits. + +Draws have shape `(..., k)` and anchors have exactly the batch shape `(...)`. +Scales and the boolean bound mask each have shape `(k,)`; broadcasting an anchor +across records is deliberately not implicit. Empty batches are supported. +Inputs are finite real numeric values converted into detached float64 arrays. +The result preserves the raw draw, anchor, component roster and scales, together +with projected values, adjustment vectors, signed sum residuals, active bounds, +normalized shifts, objectives and KKT diagnostics. These arrays are descriptive, +mutable values, not an authority or immutable execution receipt. + +Callers must supply finite nonnegative `atol` and `rtol`. The sum check uses +`abs(residual) <= atol + rtol*abs(A)`. Component stationarity and dual checks use +the same form with the maximum absolute draw, result and proposed adjustment. +The sum uses `math.fsum`; no residual is patched into a final component and no +display rounding changes the optimum. Nonfinite input, intermediate or objective +overflow, a normalized scale weight lost to underflow, or failure of the stated +tolerances causes refusal. Floating-point refusal is possible even when an +exact-arithmetic solution exists. + +The 45-case guarded suite passed with zero failures/errors/skips in 4.470 seconds +wall time, 2.899 CPU seconds and 405,143,552 bytes peak RSS. This includes 160 +randomized small problems compared against an independent oracle that enumerates +bound faces, solves dense block KKT systems and selects the minimum objective. +It also verifies negative and zero-net totals, loss offsets, unequal scales, +scale-rescaling invariance, active bounds, exact feasible identity, detached +storage, alternate component rosters, multidimensional and empty batches, invalid +inputs and unstable arithmetic. All 990 source-plus-owned hashes and 15 allowed +resource hashes were unchanged; no child or unexpected refusal occurred. The +normal guard-induced dateutil zoneinfo warning occurred. + +Production source SHA256: +`d56c21959121cc7500f06cd747c74bea51561deba537ce2c58266d5966ae84b2`. +Test SHA256: `b5ee23c6ff4f633434c96a9e17c8b82b254435757b6eb24061b1835f57bdaa77`. +The local `codex-signed-reconciliation-20260912/numeric-v2/numeric-v2.json` receipt +SHA256 is `2f071f5116489c1febb4f594c97781c9b8d88d68cf125e94aec446d80e7c3321`. +The earlier guard preflight failure is preserved; it executed no numeric tests. + +This numerical acceptance does not establish ACS/ASEC measurement equivalence, +resolve the property or retirement donor bridge, approve scale choices, fit a +model, or certify a release. Source qualification, donor definitions, a named +reconciliation node and dependent model replay remain separate work. diff --git a/packages/microcosm-fit/src/microcosm/fit/signed_reconciliation.py b/packages/microcosm-fit/src/microcosm/fit/signed_reconciliation.py new file mode 100644 index 000000000..08e61e905 --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/signed_reconciliation.py @@ -0,0 +1,188 @@ +"""Weighted projection onto an observed signed total and declared lower bounds. + +This numeric operation has no source, model, tax or execution authority. The +caller chooses the component meanings and scales and qualifies the anchor. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import fsum + +import numpy as np + + +@dataclass(frozen=True) +class SignedTotalReconciliation: + """Detached numeric results and diagnostics, with no input storage aliases.""" + + components: tuple[str, ...] + nonnegative: np.ndarray + scales: np.ndarray + raw_draws: np.ndarray + anchors: np.ndarray + values: np.ndarray + adjustments: np.ndarray + residuals: np.ndarray + active_bounds: np.ndarray + normalized_shift: np.ndarray + objective: np.ndarray + stationarity_residuals: np.ndarray + active_dual_violations: np.ndarray + atol: float + rtol: float + + +def _require(condition, reason): + if not condition: + raise ValueError("SIGNED_RECONCILIATION_" + reason) + + +def _numbers(value, name): + _require(not np.ma.isMaskedArray(value), name + "_MASKED") + array = np.asarray(value) + _require(array.dtype.kind in "fiu", name + "_NUMERIC") + with np.errstate(over="raise", invalid="raise"): + result = np.array(array, dtype=np.float64, copy=True) + _require(np.isfinite(result).all(), name + "_FINITE") + return result + + +def _tolerance(atol, rtol, magnitude): + result = atol + rtol * magnitude + _require(np.isfinite(result).all(), "TOLERANCE_OVERFLOW") + return result + + +def _row(q, anchor, weights, nonnegative): + active = np.zeros(len(q), dtype=bool) + # Removing a negative bounded candidate can only lower the common shift. + # A removed coordinate therefore cannot need releasing later. At least one + # unrestricted coordinate remains free for every signed anchor. + for _ in range(int(nonnegative.sum()) + 1): + free = ~active + shift = (anchor - fsum(q[free])) / fsum(weights[free]) + _require(np.isfinite(shift), "SHIFT_OVERFLOW") + candidate = q.copy() if shift == 0 else q + shift * weights + _require(np.isfinite(candidate).all(), "VALUE_OVERFLOW") + newly_active = free & nonnegative & (candidate < 0) + if not newly_active.any(): + return np.where(active, 0.0, candidate), active, shift + active |= newly_active + raise ValueError("SIGNED_RECONCILIATION_ACTIVE_SET_FAILED") + + +def reconcile_signed_total( + draws, + anchors, + *, + components: tuple[str, ...], + nonnegative, + scales, + atol: float, + rtol: float, +) -> SignedTotalReconciliation: + """Minimize ``sum(((z - q) / scales)**2)`` with ``sum(z) = anchor``. + + ``draws`` has shape ``(..., k)`` and ``anchors`` exactly ``draws.shape[:-1]``; + one vector and scalar anchor, multidimensional batches and empty batches + are supported. Scales and the boolean lower-bound mask each have shape + ``(k,)``. The unique component roster is explicit. At least one coordinate + must be unrestricted; no nonzero-presence or sign restriction is inferred + from a draw or an anchor. Scales must be finite and strictly positive. + + The active-set solve uses ``weights = (scales / max(scales))**2``. This + common normalization preserves the objective's minimizer without squaring + very large scales. A free coordinate is ``q + normalized_shift * weights``; + a coordinate whose declared zero bound is active is zero. The signed anchor + is never clamped or used as a divisor. Exact feasible inputs are unchanged. + + Float64 results must pass the explicit ``atol + rtol * magnitude`` checks: + ``abs(anchor)`` for the sum residual; the maximum absolute draw, result and + proposed adjustment for component stationarity and active dual feasibility. + Nonfinite inputs, overflowing intermediates/objectives, weights lost to + underflow, or results outside these tolerances are refused. No last-coordinate + residual patch or rounding changes the computed optimum. Returned arrays + are detached diagnostics, not immutable authority or source observations. + """ + try: + with np.errstate(over="raise", invalid="raise", divide="raise"): + return _reconcile( + draws, anchors, components, nonnegative, scales, atol, rtol + ) + except (FloatingPointError, OverflowError) as error: + raise ValueError("SIGNED_RECONCILIATION_NUMERIC_OVERFLOW") from error + + +def _reconcile(draws, anchors, components, nonnegative, scales, atol, rtol): + q = _numbers(draws, "DRAWS") + a = _numbers(anchors, "ANCHORS") + s = _numbers(scales, "SCALES") + _require(q.ndim >= 1 and q.shape[-1] > 0, "DRAW_SHAPE") + k = q.shape[-1] + _require(a.shape == q.shape[:-1], "ANCHOR_SHAPE") + _require(s.shape == (k,) and (s > 0).all(), "SCALE_SHAPE_OR_SIGN") + _require( + type(components) is tuple + and len(components) == k + and all(type(c) is str and c and c.strip() == c for c in components) + and len(set(components)) == k, + "COMPONENT_ROSTER", + ) + _require(not np.ma.isMaskedArray(nonnegative), "BOUND_MASKED") + bound = np.array(nonnegative, copy=True) + _require(bound.shape == (k,) and bound.dtype == np.dtype(bool), "BOUND_MASK") + _require((~bound).any(), "SIGNED_COMPONENT_REQUIRED") + tolerances = [_numbers(v, "TOLERANCE") for v in (atol, rtol)] + _require(all(v.shape == () and v >= 0 for v in tolerances), "TOLERANCE") + atol, rtol = (float(v) for v in tolerances) + weights = (s / s.max()) ** 2 + _require((weights > 0).all(), "SCALE_RATIO_UNDERFLOW") + batch_shape = q.shape[:-1] + rows, totals = q.reshape(-1, k), a.reshape(-1) + values = np.empty_like(rows) + active = np.empty(rows.shape, dtype=bool) + shifts = np.empty(len(rows)) + residuals = np.empty(len(rows)) + stationarity = np.empty(len(rows)) + dual_violation = np.empty(len(rows)) + for i, (row, anchor) in enumerate(zip(rows, totals, strict=True)): + z, fixed, shift = _row(row, anchor, weights, bound) + residual = fsum(z) - anchor + _require(np.isfinite(residual), "RESIDUAL_OVERFLOW") + _require(abs(residual) <= _tolerance(atol, rtol, abs(anchor)), "SUM_TOLERANCE") + proposed = shift * weights + magnitude = np.maximum.reduce([abs(row), abs(z), abs(proposed)]) + limit = _tolerance(atol, rtol, magnitude) + free_error = abs((z - row) - proposed) + active_error = np.maximum(row + proposed, 0.0) + _require( + (z[bound] >= 0).all() + and (free_error[~fixed] <= limit[~fixed]).all() + and (active_error[fixed] <= limit[fixed]).all(), + "KKT_TOLERANCE", + ) + values[i], active[i], shifts[i], residuals[i] = z, fixed, shift, residual + stationarity[i] = np.max(free_error[~fixed], initial=0.0) + dual_violation[i] = np.max(active_error[fixed], initial=0.0) + values = values.reshape(q.shape) + adjustment = values - q + objective = np.sum((adjustment / s) ** 2, axis=-1) + _require(np.isfinite(objective).all(), "OBJECTIVE_OVERFLOW") + return SignedTotalReconciliation( + components=components, + nonnegative=bound, + scales=s, + raw_draws=q, + anchors=a, + values=values, + adjustments=adjustment, + residuals=residuals.reshape(batch_shape), + active_bounds=active.reshape(q.shape), + normalized_shift=shifts.reshape(batch_shape), + objective=objective, + stationarity_residuals=stationarity.reshape(batch_shape), + active_dual_violations=dual_violation.reshape(batch_shape), + atol=atol, + rtol=rtol, + ) diff --git a/packages/microcosm-fit/tests/test_signed_reconciliation.py b/packages/microcosm-fit/tests/test_signed_reconciliation.py new file mode 100644 index 000000000..c6013e502 --- /dev/null +++ b/packages/microcosm-fit/tests/test_signed_reconciliation.py @@ -0,0 +1,240 @@ +"""Invented signed-total controls and an independent exhaustive KKT oracle.""" + +import itertools +from math import fsum + +import numpy as np +import pytest + +from microcosm.fit.signed_reconciliation import reconcile_signed_total + + +def project(q, anchor, **options): + parameters = dict( + components=("interest", "dividends", "property"), + nonnegative=[True, True, False], + scales=[1.0, 1.0, 1.0], + atol=1e-10, + rtol=1e-12, + ) + parameters.update(options) + return reconcile_signed_total(q, anchor, **parameters) + + +@pytest.mark.parametrize( + "q,anchor,expected", + [ + ([10.0, 4.0, 2.0], -6.0, [1.0, 0.0, -7.0]), + ([8.0, 4.0, -3.0], 0.0, [5.0, 1.0, -6.0]), + ([8.0, 4.0, -9.0], 6.0, [9.0, 5.0, -8.0]), + ([-3.0, 2.0, 1.0], 5.0, [0.0, 3.0, 2.0]), + ([1.0, 2.0, 8.0], -10.0, [0.0, 0.0, -10.0]), + ], +) +def test_signed_and_zero_net_anchors_preserve_offsets(q, anchor, expected): + result = project(q, anchor) + np.testing.assert_array_equal(result.values, expected) + np.testing.assert_array_equal(result.raw_draws, q) + np.testing.assert_array_equal(result.adjustments, result.values - q) + assert fsum(result.values) == anchor + assert result.residuals == 0 + np.testing.assert_array_equal(result.active_bounds, result.values == 0) + + +def test_unequal_scales_allocate_adjustment_by_squared_scale(): + result = project([10, 10, 10], 44.0, scales=[1, 2, 3]) + np.testing.assert_allclose(result.values, [11, 14, 19], atol=1e-12) + assert result.objective == pytest.approx(14.0) + assert result.normalized_shift == pytest.approx(9.0) + + +def test_exactly_feasible_input_is_bitwise_identity_and_detached(): + q = np.array([-0.0, 2.0, -1.0]) + a, s, bound = ( + np.array(1.0), + np.array([1.0, 2.0, 3.0]), + np.array([True, True, False]), + ) + original = [v.copy() for v in (q, a, s, bound)] + result = project(q, a, scales=s, nonnegative=bound, atol=0.0, rtol=0.0) + np.testing.assert_array_equal(result.values.view("uint64"), q.view("uint64")) + assert result.objective == 0 and result.residuals == 0 + for before, after in zip(original, (q, a, s, bound), strict=True): + np.testing.assert_array_equal(before, after) + for returned, supplied in ( + (result.raw_draws, q), + (result.values, q), + (result.anchors, a), + (result.scales, s), + (result.nonnegative, bound), + ): + assert not np.shares_memory(returned, supplied) + result.raw_draws[0] = 50 + result.nonnegative[0] = False + assert q[0] == 0 and bound[0] + + +def test_roster_is_explicit_and_can_split_interest_without_tax_assumptions(): + result = project( + [4, 6, 2, -9], + 3, + components=("ordinary_interest", "account_interest", "dividend", "property"), + nonnegative=[True, True, True, False], + scales=[1, 2, 3, 4], + ) + np.testing.assert_array_equal(result.values, [4, 6, 2, -9]) + assert result.components[1] == "account_interest" + + +def test_batches_match_individual_rows_including_multidimensional_batch(): + q = np.array([[[8, 4, 2], [-3, 2, 1]], [[8, 4, -3], [1, 2, 8]]], dtype=float) + anchors = np.array([[-6, 5], [0, -10]], dtype=float) + batch = project(q, anchors) + assert batch.values.shape == (2, 2, 3) and batch.residuals.shape == (2, 2) + for index in np.ndindex(anchors.shape): + row = project(q[index], anchors[index]) + np.testing.assert_array_equal(batch.values[index], row.values) + assert batch.objective[index] == row.objective + + +@pytest.mark.parametrize("shape", [(0, 3), (2, 0, 3)]) +def test_empty_batches_preserve_shapes(shape): + result = project(np.empty(shape), np.empty(shape[:-1])) + for value in ( + result.values, + result.raw_draws, + result.adjustments, + result.active_bounds, + ): + assert value.shape == shape + for value in (result.residuals, result.objective, result.normalized_shift): + assert value.shape == shape[:-1] + + +def test_any_declared_coordinate_can_be_signed_and_all_can_be_unrestricted(): + permuted = project( + [2, 10, 4], + -6, + nonnegative=[False, True, True], + components=("property", "interest", "dividends"), + ) + np.testing.assert_array_equal(permuted.values, [-7, 1, 0]) + free = project([3, -7, 1], 0, nonnegative=[False, False, False]) + np.testing.assert_array_equal(free.values, [4, -6, 2]) + + +def test_rescaling_all_scales_does_not_change_the_solution(): + base = project([8, 2, 3], -6, scales=[1, 2, 3]) + large = project([8, 2, 3], -6, scales=np.array([1, 2, 3]) * 1e100) + np.testing.assert_allclose(base.values, large.values, atol=1e-12, rtol=1e-12) + assert large.objective == pytest.approx(base.objective / 1e200, rel=1e-12, abs=0) + + +@pytest.mark.parametrize( + "q,anchor,options", + [ + ([np.nan, 1, 1], 0, {}), + ([np.inf, 1, 1], 0, {}), + ([1, 1, 1], np.inf, {}), + ([1, 1, 1], np.nan, {}), + ([1, 1, 1], 0, {"scales": [0, 1, 1]}), + ([1, 1, 1], 0, {"scales": [-1, 1, 1]}), + ([1, 1, 1], 0, {"scales": [np.inf, 1, 1]}), + ([1, 1, 1], 0, {"scales": [1, 1]}), + ([1, 1, 1], 0, {"scales": [True, True, True]}), + ([1, 1, 1], 0, {"nonnegative": [1, 1, 0]}), + ([1, 1, 1], 0, {"nonnegative": [True, True, True]}), + ([1, 1, 1], 0, {"components": ("i", "i", "p")}), + ([1, 1, 1], 0, {"components": ("i", "d", " ")}), + ([1, 1, 1], 0, {"components": ["i", "d", "p"]}), + ([1, 1, 1], 0, {"atol": -1}), + ([1, 1, 1], 0, {"rtol": np.inf}), + ([1, 1, 1], 0, {"atol": [1e-6]}), + ([1, 1, 1], [0], {}), + ([[1, 1, 1]], 0, {}), + ([], 0, {}), + (1, 0, {}), + (["1", "2", "3"], 0, {}), + ([1j, 1, 1], 0, {}), + ([True, False, True], 0, {}), + (np.ma.array([1, 2, 3], mask=[False, True, False]), 0, {}), + ([1, 2, 3], 0, {"nonnegative": np.ma.array([True, True, False])}), + ], +) +def test_malformed_nonfinite_and_undeclared_inputs_refuse(q, anchor, options): + with pytest.raises(ValueError, match="SIGNED_RECONCILIATION_"): + project(q, anchor, **options) + + +@pytest.mark.parametrize( + "q,anchor,options", + [ + ([1e308, 1e308, -1e308], 1, {}), + ([1, 2, 3], 0, {"scales": [1e-200, 1, 1]}), + ([1, 2, 3], 0, {"scales": [1e-200, 1e-200, 1e-200]}), + ([1, 2, 3], 10, {"rtol": 1e308}), + ([1e16, 1e16, -1e16], 1, {"atol": 1e-12, "rtol": 0}), + ], +) +def test_unstable_overflow_or_unrepresentable_sum_refuses(q, anchor, options): + with pytest.raises(ValueError, match="SIGNED_RECONCILIATION_"): + project(q, anchor, **options) + + +def _exhaustive_kkt_oracle(q, anchor, scales, bound): + """Enumerate bound faces and solve each equality-constrained quadratic. + + This uses dense block KKT solves and objective comparison, independently of + the production active-set order and common-shift arithmetic. + """ + best = None + bounded = np.flatnonzero(bound) + for fixed_values in itertools.product((False, True), repeat=len(bounded)): + fixed = np.zeros(len(q), dtype=bool) + fixed[bounded] = fixed_values + free = ~fixed + hessian = np.diag(1 / scales[free] ** 2) + count = int(free.sum()) + matrix = np.block( + [[hessian, np.ones((count, 1))], [np.ones((1, count)), np.zeros((1, 1))]] + ) + rhs = np.concatenate([hessian @ q[free], [anchor]]) + solved = np.linalg.solve(matrix, rhs) + z = np.zeros(len(q)) + z[free] = solved[:-1] + if np.any(z[bound] < -1e-9): + continue + objective = fsum(((z - q) / scales) ** 2) + if best is None or objective < best[0]: + best = objective, z, solved[-1] + assert best is not None + return best + + +def test_random_cases_match_independent_convex_kkt_oracle(): + rng = np.random.default_rng(8471) + for _ in range(160): + count = int(rng.integers(2, 6)) + q = rng.normal(0, 40, size=count) + anchor = float(rng.normal(0, 50)) + scales = np.exp(rng.uniform(-2, 2, size=count)) + bound = rng.uniform(size=count) > 0.25 + bound[int(rng.integers(count))] = False + result = reconcile_signed_total( + q, + anchor, + components=tuple(f"part_{i}" for i in range(count)), + nonnegative=bound, + scales=scales, + atol=1e-10, + rtol=1e-12, + ) + objective, expected, multiplier = _exhaustive_kkt_oracle( + q, anchor, scales, bound + ) + np.testing.assert_allclose(result.values, expected, atol=1e-8, rtol=2e-10) + assert float(result.objective) == pytest.approx(objective, rel=2e-10, abs=1e-8) + gradient = (result.values - q) / scales**2 + multiplier + np.testing.assert_allclose(gradient[~result.active_bounds], 0, atol=1e-8) + assert (gradient[result.active_bounds] >= -1e-8).all() + assert abs(fsum(result.values) - anchor) <= 1e-10 + 1e-12 * abs(anchor) From 8efbf5ba643730ee353bcafe9297a830151d7c20 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:59:58 -0400 Subject: [PATCH 146/255] Correct four printed-dictionary claims found by adversarial review I_FRMYN prints an empty Values block, so its codes were wrongly described as following I_ANNVAL; it is now accepted on its printed (0:9) range header alone and the missing value list is recorded as a printed ambiguity, alongside the DST_SC(2) notation that I_DSTSC shares with I_DSTSCCOMP. FARM_AMOUNT_SCOPE dropped the printed composite clause naming ERN_VAL and FRM_VAL, so it is no longer the complete printed label it claimed to be. The distribution universes were said to lack the 15+ floor that every other family prints; the farm family prints no age floor at all, being gated on ERN_YN and FRMOTR instead. Printed question scopes and code labels were stored under a key naming printed universes, and the NIU normalization list was documented at a path it did not occupy. Both now sit where the contract note says they do. Co-Authored-By: Claude Opus 5 --- docs/us-current-asec-income-routing-source.md | 31 +++++---- .../current_asec_income_routing_source.py | 63 ++++++++++++++----- .../test_us_current_asec_income_routing.py | 56 +++++++++++++++++ 3 files changed, 122 insertions(+), 28 deletions(-) diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index 6f7e214fe..f9df50b0c 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -100,8 +100,9 @@ a nonfiler is never treated as a non-recipient. ## Source-level questions preserved as evidence These are printed-dictionary facts, kept distinguishable from modelling -judgments and recorded verbatim under -`evidence["dictionary"]["printed_universe_questions"]`: +judgments. The first four are recorded under +`evidence["dictionary"]["printed_universe_questions"]`; the rest under +`evidence["dictionary"]["printed_scope_and_code_questions"]`: - `DST_VAL1`'s printed universe is `DST_SC1 = 1` although its label names the source-1 distribution amount; taken literally that would restrict it to 401k @@ -109,9 +110,12 @@ judgments and recorded verbatim under - `DST_SC2_YNG`'s printed universe names `DST_VAL_YNG`, a field with no dictionary entry. - `I_DSTVAL1COMP`'s printed `Universe:` line is empty. -- `DST_YN` and `DST_YN_YNG` print only the age-58 split and, unlike every other - family in this slice, do not print the 15+ floor. Coverage below age 15 is - therefore recorded as unresolved rather than resolved either way. +- `DST_YN` and `DST_YN_YNG` print only the age-58 split. They print no 15+ floor + as the four age-universe families (`PEN_YN`, `ANN_YN`, `RNT_YN`, `OI_YN`) do, + and they are not gated on other literals as the farm family is — `FRSE_YN` + prints `ERN_YN=1 or FRMOTR=1` and so carries no age floor either. Coverage + below age 15 is therefore recorded as unresolved rather than resolved either + way. - `OI_YN`'s printed zero label is `none or niu` where `PEN_YN`, `ANN_YN`, `DST_YN` and `RNT_YN` print `niu`, so a zero receipt literal is not interchangeable across families. @@ -132,8 +136,8 @@ The authenticated money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and records `CodebookStatus.DECLARED_NIU`. `amount_state` therefore reads the dollar meaning of a cell from the parent's status axis, never from the stored number: re-deriving NIU from the number would read that cell as a zero -dollar annuity. `evidence["declared_niu_normalized_to_zero"]` lists the affected -entries. +dollar annuity. `evidence["declared_niu_normalized_to_zero"]` (a top-level key) lists the +affected entries. The money owner restates non-2024 cohorts to the pinned price basis, so the literal-identity join is taken only on `person_years == 2024`; widening it to @@ -156,12 +160,17 @@ the pooled 2022/2023 cohorts would require handling that restatement first. its printed range. `PUBLISHED_ALLOCATION_FLAG_BY_FIELD` names which flag covers which field. -`UNFLAGGED_FIELDS` records the nine fields for which the 2025 dictionary +`UNFLAGGED_FIELDS` records the ten fields for which the 2025 dictionary publishes no flag: `PNSN_VAL`, `FRSE_VAL`, `FRSE_YN`, `OI_OFF`, `OI_YN`, `DST_VAL1_YNG`, `DST_VAL2_YNG`, `DST_YN_YNG`, `DST_SC1_YNG`, `DST_SC2_YNG`. -`AMBIGUOUS_FLAG_COVERAGE` records that `I_DSTSCCOMP`'s label names `DST_SC(2)` -while its universe names both routes, so its coverage of the under-58 source -codes is unresolved. No row is ever labelled a raw respondent value. +`AMBIGUOUS_FLAG_COVERAGE` records three printed ambiguities: `I_DSTSCCOMP`'s +label names `DST_SC(2)` while its universe names both routes, so its coverage of +the under-58 source codes is unresolved; `I_DSTSC`'s label uses the same +`DST_SC(2)` notation, so mapping it to both `DST_SC1` and `DST_SC2` reads the +parenthesis as a slot count that the dictionary does not state; and `I_FRMYN` +prints an empty `Values:` block, so only its `(0:9)` range header is published +and no meaning is claimed for its codes. No row is ever labelled a raw +respondent value. ## Constants diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 003aaa754..56502ea36 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -198,10 +198,15 @@ def printed_amount_entries(): OTHER_INCOME_CATEGORY_ENTRY = (2, 547, 47, "6C-26", "OI_YN = 1") # Published allocation flags. Values 0-9 follow I_ANNVAL; the DST composites -# follow I_INTYN (0, 10, 11); I_DSTSC prints its own 0/1/9 set. +# follow I_INTYN (0, 10, 11); I_DSTSC prints its own 0/1/9 set. I_FRMYN prints +# an empty Values block, so only its (0:9) range header is published and the +# meaning of its codes is not; it is accepted on the printed range alone. ALLOCATION_ANNVAL_CODES = tuple(range(10)) ALLOCATION_COMPOSITE_CODES = (0, 10, 11) ALLOCATION_DSTSC_CODES = (0, 1, 9) +# I_FRMYN's printed Values block is empty; its (0:9) range header is all that is +# published, so no code meaning is claimed for it. +ALLOCATION_PRINTED_RANGE_CODES = tuple(range(10)) # name: (length, position, pdf_page, printed_page, universe, codes) ALLOCATION_ENTRIES = { "I_ANNVAL": (1, 802, 53, "6C-32", "ANN_YN =1", ALLOCATION_ANNVAL_CODES), @@ -219,7 +224,7 @@ def printed_amount_entries(): "I_DSTVAL2COMP": (2, 825, 55, "6C-34", "DST_VAL2> 0", ALLOCATION_COMPOSITE_CODES), "I_DSTYNCOMP": (2, 827, 55, "6C-34", "DST_YN > 0", ALLOCATION_COMPOSITE_CODES), "I_ERNYN": (1, 833, 55, "6C-34", "ERN_YN > 0", ALLOCATION_ANNVAL_CODES), - "I_FRMYN": (1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_ANNVAL_CODES), + "I_FRMYN": (1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_PRINTED_RANGE_CODES), "I_OIVAL": (1, 843, 56, "6C-35", "OI_VAL > 0", ALLOCATION_ANNVAL_CODES), "I_PENYN": (1, 854, 57, "6C-36", "PEN_YN > 0", ALLOCATION_ANNVAL_CODES), "I_RNTVAL": (1, 861, 57, "6C-36", "RNT_VAL > 0", ALLOCATION_ANNVAL_CODES), @@ -256,10 +261,17 @@ def printed_amount_entries(): "DST_SC1_YNG", "DST_SC2_YNG", ) +# The DST_SC(2) notation is read here as "the two DST_SC slots", which is an +# inference from the printed parenthesis rather than a printed statement. AMBIGUOUS_FLAG_COVERAGE = { + "I_DSTSC": "printed label names DST_SC(2); mapping it to both DST_SC1 and " + "DST_SC2 reads the parenthesis as a slot count, which the dictionary does " + "not state", "I_DSTSCCOMP": "printed label names DST_SC(2) while its printed universe " "names DST_YN = 1 or DST_YNG_YN = 1; whether it covers the under-58 source " - "codes is unresolved" + "codes is unresolved", + "I_FRMYN": "printed Values block is empty, so the meaning of its codes is " + "not published; only the (0:9) range header is", } READ_COLUMNS = ( @@ -280,7 +292,10 @@ def printed_amount_entries(): "own any land, property, rented to others, or receive income from royalties, " "roomers or boarders, or from estates or trusts" ) -FARM_AMOUNT_SCOPE = "total amount of farm self-employment earnings" +FARM_AMOUNT_SCOPE = ( + "total amount of farm self-employment earnings (combined amounts in " + "ERN_VAL, if ERN_SRCE=3, and FRM_VAL)" +) FAMILIES = ( "pension_annuity", @@ -633,9 +648,11 @@ def _retirement_distribution(raw, ages): "retirement_distribution_receipt", receipt_tokens, RECEIPT_CODES, RECEIPT_CODES ) out = pd.concat([out, frame], axis=1) - # The two printed DST universes name only the age-58 split; unlike every - # other family in this slice they do not print the 15+ floor. Whether a - # person under 15 is inside them is a source question, left unresolved. + # The two printed DST universes name only the age-58 split. Unlike the four + # age-universe families here (PEN_YN, ANN_YN, RNT_YN, OI_YN) they print no + # 15+ floor, and unlike the farm family they are not gated on other + # literals either, so whether a person under 15 is inside them is a source + # question left unresolved. universes = [True if a >= 15 else None for a in ages] labels, canonical, _, _ = _classify( totals, @@ -1172,11 +1189,6 @@ def qualify_current_asec_income_routing(preparation): k: [*v[:5], list(v[5])] for k, v in ALLOCATION_ENTRIES.items() }, "fields_without_published_allocation_flag": list(UNFLAGGED_FIELDS), - "declared_niu_normalized_to_zero": [ - name - for name, entry in printed_amount_entries().items() - if entry.nonmoney_codes - ], "published_allocation_flag_by_field": dict( PUBLISHED_ALLOCATION_FLAG_BY_FIELD ), @@ -1187,11 +1199,23 @@ def qualify_current_asec_income_routing(preparation): "DST_SC2_YNG": "printed universe names DST_VAL_YNG, which has no " "dictionary entry; retained verbatim", "I_DSTVAL1COMP": "printed universe line is empty", - "DST_YN": "printed universes name only the a_age 58 split and do " - "not print the 15+ floor the other families print; coverage below " - "age 15 is unresolved", - "RNT_YN": NET_PROPERTY_RECEIPT_SCOPE, - "RNT_VAL": NET_PROPERTY_AMOUNT_SCOPE, + "DST_YN": "printed universes name only the a_age 58 split; they " + "print no 15+ floor as the four age-universe families here do, " + "and they are not gated on other literals as the farm family is, " + "so coverage below age 15 is unresolved", + }, + "printed_scope_and_code_questions": { + "RNT_YN": "printed question scope: " + NET_PROPERTY_RECEIPT_SCOPE, + "RNT_VAL": "printed question scope: " + NET_PROPERTY_AMOUNT_SCOPE, + "FRSE_VAL": "printed label: " + FARM_AMOUNT_SCOPE, + "PNSN_VAL": "printed label: " + PENSION_TOTAL_SCOPE, + "OI_YN": "printed zero label is 'none or niu' where PEN_YN, " + "ANN_YN, DST_YN and RNT_YN print 'niu'", + "DST_SC1": "gated on DST_VAL1 > 0 and a_age \u2265 58 while " + "DST_SC1_YNG is gated on DST_YN_YNG = 1 and a_age < 58; the two " + "routes are not symmetric", + "FRSE_YN": "printed universe is ERN_YN=1 or FRMOTR=1, so the farm " + "family prints no age floor at all", }, }, "preparation_sha256": _sha(entry[1]), @@ -1200,6 +1224,11 @@ def qualify_current_asec_income_routing(preparation): "source_member_sha256": digest, "read_columns": list(READ_COLUMNS), "complete_current_source_rows": rows, + "declared_niu_normalized_to_zero": [ + name + for name, entry in printed_amount_entries().items() + if entry.nonmoney_codes + ], "joined_person_years": [CURRENT_INCOME_YEAR], "restatement_note": ( "the money owner restates non-2024 cohorts to the pinned price basis, " diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index 05f4168e8..21fb7702b 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -189,6 +189,26 @@ def test_actual_invented_member_joins_the_real_preparation_by_native_keys( qualified.evidence["dictionary"]["amount_entries_source"]["sha256"] == owner.money.RESOURCE_PINS[0] ) + assert qualified.evidence["declared_niu_normalized_to_zero"] == ["ANN_VAL"] + printed = qualified.evidence["dictionary"] + assert set(printed["printed_universe_questions"]) == { + "DST_VAL1", + "DST_SC2_YNG", + "I_DSTVAL1COMP", + "DST_YN", + } + assert set(printed["printed_scope_and_code_questions"]) == { + "RNT_YN", + "RNT_VAL", + "FRSE_VAL", + "PNSN_VAL", + "OI_YN", + "DST_SC1", + "FRSE_YN", + } + assert set(printed["ambiguous_allocation_flag_coverage"]) == set( + owner.AMBIGUOUS_FLAG_COVERAGE + ) prepared.checked_view() @@ -843,3 +863,39 @@ def test_other_income_routing_never_invents_a_reported_category( assert out.other_income_source_total.iloc[0] == 1500.0 # The reported category never becomes a residual assignment. assert not bool(out.other_income_residual_rule_applied.iloc[0]) + + +def test_printed_flag_and_scope_claims_match_the_dictionary_as_printed(): + # I_FRMYN prints an empty Values block, so only its (0:9) range header is + # published; no code meaning is claimed for it. + assert owner.ALLOCATION_ENTRIES["I_FRMYN"][5] == tuple(range(10)) + assert owner.ALLOCATION_ENTRIES["I_FRMYN"][5] is not owner.ALLOCATION_ANNVAL_CODES + assert owner.ALLOCATION_ENTRIES["I_DSTSC"][5] == (0, 1, 9) + for name in ("I_DSTVAL1COMP", "I_DSTVAL2COMP", "I_DSTYNCOMP"): + assert owner.ALLOCATION_ENTRIES[name][5] == (0, 10, 11) + # I_DSTVAL1COMP's printed universe line is empty in the dictionary. + assert owner.ALLOCATION_ENTRIES["I_DSTVAL1COMP"][4] == "" + # The DST_SC(2) notation and the empty Values block stay recorded as + # printed ambiguities rather than resolved silently. + assert set(owner.AMBIGUOUS_FLAG_COVERAGE) == { + "I_DSTSC", + "I_DSTSCCOMP", + "I_FRMYN", + } + assert len(owner.UNFLAGGED_FIELDS) == 10 + # Printed labels are complete, including the farm composite clause. + assert owner.FARM_AMOUNT_SCOPE.endswith( + "(combined amounts in ERN_VAL, if ERN_SRCE=3, and FRM_VAL)" + ) + assert owner.PENSION_TOTAL_SCOPE.endswith("from all pension sources") + assert "estates or trusts" in owner.NET_PROPERTY_RECEIPT_SCOPE + assert "estates" not in owner.NET_PROPERTY_AMOUNT_SCOPE + # The two printed-question dictionaries stay separate: one is about printed + # universes, the other about printed scopes and code labels. + assert set(owner.ACCOUNT_ENTRIES["DST_SC1"][4:]) == {"DST_VAL1 > 0 and a_age ≥ 58"} + assert owner.ACCOUNT_ENTRIES["DST_SC1_YNG"][4] == ("DST_YN_YNG = 1 and a_age < 58") + assert owner.ACCOUNT_ENTRIES["DST_SC2_YNG"][4] == ("DST_VAL_YNG > 0 and a_age < 58") + assert owner.RECEIPT_ENTRIES["OI_YN"][5] == "none or niu" + assert owner.RECEIPT_ENTRIES["FRSE_YN"][4] == "ERN_YN=1 or FRMOTR=1" + for name in ("PEN_YN", "ANN_YN", "RNT_YN", "OI_YN"): + assert owner.RECEIPT_ENTRIES[name][4] == "All Persons aged 15+" From 7ac2b864a93977ffa98fa86674c09e04ea6b20b5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:04:57 -0400 Subject: [PATCH 147/255] Recompute the source-derived identities the merged solver moves solve.py is an attested kernel module (_DIRECT_KERNEL_MODULES at spec_engine/seeds.py), so editing it moves the seed protocol digest, every resolved-spec digest that folds it in, and the calibrate kernel's implementation hash. Both branches pinned their own tree's values; neither describes this one, so every value here is recomputed against the merged checkout with the isolated 3.14 interpreter, not chosen ours/theirs. seed_protocol 7c67a5a3 (G) / d052fd87 (S) -> 57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988 seed_map be2d24ad (G) / d5a9694a (S) -> 4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878 us spec f03e4d08 (G) / ff2c9703 (S) -> 15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8 loader golden 77f043a1 (G) / 8a240898 (S) -> 3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7 calibrate impl 59ca046b (G) / 46b52a20 (S) -> f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d calibrate node faaccb27 (G) / 970720b7 (S) -> ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102 docs/evidence/spec-engine/us-f0-coverage.json is regenerated by tools/spec_engine_coverage.py and its --check now exits 0. The calibrate parity pin is re-recorded by tools/graph_parity_repin.py calibrate, which produces the local platform key by running the graph; direct.csv is unchanged because the kernel's outputs did not move, only its implementation hash. The country-bundle digests in test_spec_engine_country_bundles.py move for the same reason and are recomputed too (am/be/uk); the us value there is the same 15e61194 the coverage report and the multispine pin now carry. Deliberately NOT repinned: the simulate parity pin and the fit.qrf pin. Neither hashes microcosm.calibrate.solve, and both are byte-unchanged. Co-Authored-By: Claude Opus 5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- .../tests/test_spec_engine_country_bundles.py | 6 +++--- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- .../fixtures/parity/kernels/calibrate/pins.json | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index c7032335f..9a16b3b31 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", - "protocol_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47" + "map_sha256": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878", + "protocol_sha256": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988" }, "failures": [], "observed": { - "map_sha256": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", - "protocol_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47" + "map_sha256": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878", + "protocol_sha256": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", + "implementation_sha256": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", + "implementation_sha256": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405" + "spec_sha256": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405" + "spec_sha256": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 9a32bb9cd..cfa783b54 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", - "seed_protocol": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", + "seed_map": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878", + "seed_protocol": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index ed52cc979..d9c38c0e9 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c" +AM_SPEC_SHA256 = "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", + "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", + "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 90b0c458e..0d0ab7368 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3" + "3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 723f18d56..b59b4cb6c 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2517,7 +2517,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405", + "spec_sha256": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8", }, } diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index 1b196a434..1dec5500f 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74","kernel":"calibrate.adam@1","node":"calibrate","node_key":"faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d","kernel":"calibrate.adam@1","node":"calibrate","node_key":"ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102"}},"seed":0} From fbb109987e30f09ccc87fc402c47190d7c3ecfd2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:07:27 -0400 Subject: [PATCH 148/255] Fix snapshot iterate and selected probe identity --- .../914-snapshot-iterate-identity.fixed.md | 1 + docs/calibration-target-snapshots.md | 19 +++++ docs/evidence/spec-engine/us-f0-coverage.json | 16 ++-- .../build/spec_engine/inventory_coverage.py | 4 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- .../src/microcosm/calibrate/solve.py | 23 ++++-- .../microcosm/calibrate/target_snapshots.py | 23 +++--- .../tests/test_target_snapshots.py | 80 +++++++++++++++++-- .../parity/kernels/calibrate/pins.json | 2 +- 10 files changed, 137 insertions(+), 35 deletions(-) create mode 100644 changelog.d/914-snapshot-iterate-identity.fixed.md diff --git a/changelog.d/914-snapshot-iterate-identity.fixed.md b/changelog.d/914-snapshot-iterate-identity.fixed.md new file mode 100644 index 000000000..2563244d6 --- /dev/null +++ b/changelog.d/914-snapshot-iterate-identity.fixed.md @@ -0,0 +1 @@ +Use completed optimizer updates consistently in target snapshots, retain the selected budget probe, and encode overflowing relative errors as counted nulls. diff --git a/docs/calibration-target-snapshots.md b/docs/calibration-target-snapshots.md index 5df3054ec..58e19072f 100644 --- a/docs/calibration-target-snapshots.md +++ b/docs/calibration-target-snapshots.md @@ -33,6 +33,13 @@ remain distinguishable by ordered position. Honest nonfinite intermediate estimates are represented by nulls and counts. Intermediate float32 estimates and final float64 estimates are labeled with their actual iterate semantics. Budget probes and subsequent selection/refit phases share one emission sequence. +`epoch` counts completed optimizer updates: zero is the starting state. An +in-loop estimate precedes that iteration's update; its selected counterpart uses +the same convention. Cadence counts loss evaluations starting at one, so an +every-25 cadence can emit epoch 24. The selected budget-search snapshot retains +the winning probe's identity, even when another probe ran afterward. If computing +a relative error overflows, the error is null and `non_finite_rows` counts it; +finite operands remain visible. Metadata accepts bounded flat scalar mappings and checked string identifiers; it does not accept nested payloads or record vectors. `best_retained` must be a @@ -45,6 +52,9 @@ The local writer atomically replaces `latest.json`. History chunks become visible only after their complete bytes are written and synced, and an existing chunk cannot be overwritten. Retention and dropped-history counts are explicit. Use a fresh run directory unless intentionally adopting existing history. +Atomic history publication requires a filesystem that supports hard links and +directory synchronization. Unsupported filesystem errors propagate to the caller; +the writer does not expose partially written history as a fallback. This implements the solver and local storage portion of [issue 908](https://github.com/PolicyEngine/microcosm/issues/908). Country-host @@ -58,3 +68,12 @@ including passive solver parity tests. Independent review closed the metadata, detachment, partial-publication and codec findings. See the [invented benchmark evidence](../experiments/908-target-snapshot-bench-receipts.md) for overhead measurements; those are not native US or UK benchmarks. + +Fable's later PR review identified a mixed epoch convention, a missing winning +budget-probe label and a relative-error overflow case. Four new counterexamples +failed first; all 56 snapshot tests and all 311 calibration tests then passed +after those corrections. The 26 affected build identity and graph parity checks +also passed after recalculating the calibration source pins; fit and simulation +pins were preserved. These changes affect diagnostics only. The public writer still validates each incoming +payload because callers can supply serialized snapshots independently of an +observer. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index ccf6fde29..460ee2e99 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", - "protocol_sha256": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3" + "map_sha256": "429fd3ae3de9b67cfc077b9592ad5287b3569bf9f52d4d68fd00051a7bb118dd", + "protocol_sha256": "7cd0a767961aab8681cd9e89707f9c9ff3e66414693f6858ba4f3cc36be3c706" }, "failures": [], "observed": { - "map_sha256": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", - "protocol_sha256": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3" + "map_sha256": "429fd3ae3de9b67cfc077b9592ad5287b3569bf9f52d4d68fd00051a7bb118dd", + "protocol_sha256": "7cd0a767961aab8681cd9e89707f9c9ff3e66414693f6858ba4f3cc36be3c706" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", + "implementation_sha256": "7cd0a767961aab8681cd9e89707f9c9ff3e66414693f6858ba4f3cc36be3c706", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", + "implementation_sha256": "7cd0a767961aab8681cd9e89707f9c9ff3e66414693f6858ba4f3cc36be3c706", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88" + "spec_sha256": "43212f8137927dc65e70f26af9efe85ce227fa2267b7ce0c824431e5471af31a" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88" + "spec_sha256": "43212f8137927dc65e70f26af9efe85ce227fa2267b7ce0c824431e5471af31a" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 17e57e056..481f58405 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", - "seed_protocol": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", + "seed_map": "429fd3ae3de9b67cfc077b9592ad5287b3569bf9f52d4d68fd00051a7bb118dd", + "seed_protocol": "7cd0a767961aab8681cd9e89707f9c9ff3e66414693f6858ba4f3cc36be3c706", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 2e3da170d..a428ac516 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "8a24089831b3a8f0cfe1be094769af7a30b7b74dbc7d4009e26e9f56444cba4c" + "0e2f4a1494185dadf40cee23ae9eff6f1598e6db4e12d733d544363e5e461eb9" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 8a76b3230..dc0876547 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2504,7 +2504,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88", + "spec_sha256": "43212f8137927dc65e70f26af9efe85ce227fa2267b7ce0c824431e5471af31a", }, } diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index b9a20cc86..75cea4e01 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -930,7 +930,9 @@ def _optimize( retained = retain_best and best_log_w is not None snapshots.emit( estimate.detach().numpy(), - epoch=epoch + 1, + # Number of completed updates, matching the selection receipt. + # Cadence still uses the 1-based evaluation ordinal above. + epoch=epoch, epochs=epochs, iterate=( ITERATE_BEST_RETAINED @@ -941,7 +943,7 @@ def _optimize( loss=trajectory[epoch], best_retained={ "available": bool(retain_best), - "epoch": (best_epoch + 1) if retained else None, + "epoch": best_epoch if retained else None, "loss": best_loss if retained else None, }, ) @@ -1113,7 +1115,7 @@ def _optimize_proximal( # closing state), so every in-loop snapshot here is `current`. snapshots.emit( estimate.detach().numpy(), - epoch=epoch + 1, + epoch=epoch, epochs=epochs, iterate=ITERATE_CURRENT, precision="float32", @@ -1369,6 +1371,7 @@ def evaluate( # (0 for a feasible/unconstrained probe, 1 for an infeasible one; distance # to the budget): feasible probes always beat infeasible ones. best_key: tuple[int, int] | None = None + best_evaluation: int | None = None probes: list[dict[str, object]] = [] # Sentinel: a probe whose penalty over-pruned past the cap-feasible floor # (the conserve+cap projection raised). It is *more* pruning than feasible, @@ -1381,7 +1384,7 @@ def evaluate( _steer_stop = "stop" def consider(lam: float) -> tuple[int, str]: - nonlocal best, best_key + nonlocal best, best_key, best_evaluation try: weights, trajectory, n_nonzero, gate_open_probabilities = evaluate(lam) except ValueError as exc: @@ -1446,6 +1449,7 @@ def consider(lam: float) -> tuple[int, str]: gate_open_probabilities, ) best_key = key + best_evaluation = evaluation return n_nonzero, verdict def steer(n_nonzero: int, verdict: str) -> str: @@ -1512,6 +1516,7 @@ def settled() -> bool: "acceptable_within_tolerance" if settled() else "budget_exhausted" ), "selected_l0_lambda": None if best is None else float(best[2]), + "selected_budget_iteration": best_evaluation, "selected_measure": None if best is None else int(best[3]), "selected_feasible": ( None @@ -1531,6 +1536,7 @@ def settled() -> bool: "evaluations": int(evaluation), "probes": probes, "selected_l0_lambda": None if best is None else float(best[2]), + "selected_budget_iteration": best_evaluation, "selected_measure": None if best is None else int(best[3]), "selected_feasible": ( None @@ -2173,7 +2179,14 @@ def calibrate( and isinstance(selected_epoch, int) and selected_epoch < epochs ) - snapshots.emit( + selected_snapshots = snapshots + if budget_search is not None: + selected_snapshots = snapshots.with_search( + budget_iteration=budget_search["selected_budget_iteration"], + budget_iters=budget_search["budget_iters"], + l0_lambda=budget_search["selected_l0_lambda"], + ) + selected_snapshots.emit( final_estimates, # The epoch whose iterate was actually selected, not the last one # executed: a retained-best run returns an earlier iterate, and diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py index 5afeb99a1..06e9b1bf5 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py @@ -295,7 +295,7 @@ def signed_relative_error(estimate: float | None, target: float | None) -> float return None if target == 0.0: return estimate - target - result = estimate - target if target == 0.0 else (estimate - target) / target + result = (estimate - target) / target return _finite_or_none(result) @@ -750,7 +750,8 @@ def snapshot( zip(self.names, self.targets, strict=True) ): estimate = _finite_or_none(values[index]) - if estimate is None or target is None: + error = signed_relative_error(estimate, target) + if error is None: non_finite += 1 rows.append( { @@ -758,7 +759,7 @@ def snapshot( "name": name, "target": target, "estimate": estimate, - "relative_error": signed_relative_error(estimate, target), + "relative_error": error, } ) self.counter[0] += 1 @@ -919,19 +920,19 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: stored = _strict_number( row["relative_error"], what=f"relative_error for {name!r}" ) - if estimate is None or target is None: + expected = signed_relative_error(estimate, target) + if expected is None: observed_non_finite += 1 if stored is not None: raise TargetSnapshotError( - f"relative_error for {name!r} must be null when its estimate " - "or target is." + f"relative_error for {name!r} must be null when its operands " + "or computed error are not finite." ) else: - expected = signed_relative_error(estimate, target) - if stored is None or expected is None: + if stored is None: raise TargetSnapshotError( - f"relative_error for {name!r} must be present when both its " - "estimate and target are." + f"relative_error for {name!r} must be present when its " + "computed error is finite." ) if not math.isclose( stored, expected, rel_tol=_RELATIVE_ERROR_RTOL, abs_tol=0.0 @@ -950,7 +951,7 @@ def validate_target_snapshot(payload: Mapping[str, object]) -> None: if observed_non_finite != non_finite_rows: raise TargetSnapshotError( f"non_finite_rows says {non_finite_rows} but {observed_non_finite} rows " - "carry a null estimate or target." + "carry a null estimate, target or computed error." ) digest = target_identity_digest(names, values) if payload["targets_sha256"] != digest: diff --git a/packages/microcosm-calibrate/tests/test_target_snapshots.py b/packages/microcosm-calibrate/tests/test_target_snapshots.py index ed4094dcb..595f14e1f 100644 --- a/packages/microcosm-calibrate/tests/test_target_snapshots.py +++ b/packages/microcosm-calibrate/tests/test_target_snapshots.py @@ -10,6 +10,7 @@ from __future__ import annotations +import copy import json import math from datetime import datetime @@ -342,7 +343,7 @@ def test_every_epoch_emits_one_snapshot_per_epoch_plus_the_selected_one(): seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) calibrate(_frame(), _targets(), epochs=6, seed=0, target_snapshots=observer) epochs = [s["epoch"] for s in seen if s["iterate"] != ITERATE_SELECTED] - assert epochs == [1, 2, 3, 4, 5, 6] + assert epochs == [0, 1, 2, 3, 4, 5] selected = [s for s in seen if s["iterate"] == ITERATE_SELECTED] assert len(selected) == 1 for snapshot in seen: @@ -356,11 +357,11 @@ def test_bounded_cadence_emits_fewer_snapshots_than_every_epoch(): calibrate(_frame(), _targets(), epochs=40, seed=0, target_snapshots=sparse_observer) assert len(sparse) < len(dense) assert [s["epoch"] for s in sparse if s["iterate"] != ITERATE_SELECTED] == [ - 1, - 10, - 20, - 30, - 40, + 0, + 9, + 19, + 29, + 39, ] @@ -589,6 +590,73 @@ def test_the_selected_snapshot_names_the_epoch_it_actually_selected(): assert selected["epoch"] == receipt["selected_epoch"] +def test_retained_and_selected_snapshots_use_completed_update_identity(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), + _targets(), + epochs=30, + seed=0, + learning_rate=0.1, + target_snapshots=observer, + ) + receipt = result.options["iterate_selection_receipt"] + assert receipt["selected_epoch"] < 30 # Exercise an earlier retained state. + retained = [s for s in seen if s["iterate"] == ITERATE_BEST_RETAINED][-1] + selected = seen[-1] + assert retained["epoch"] == selected["epoch"] + assert retained["best_retained"]["epoch"] == selected["best_retained"]["epoch"] + assert retained["loss"] == receipt["selected_loss_float32"] + + +def test_selected_budget_snapshot_identifies_the_winning_probe(): + seen, observer = _collect(cadence=TargetSnapshotCadence(every=EVERY_EPOCH)) + result = calibrate( + _frame(), + _targets(), + epochs=8, + seed=0, + target_records=2, + budget_iters=2, + target_snapshots=observer, + ) + selected = seen[-1] + assert selected["iterate"] == ITERATE_SELECTED + assert selected["search"] is not None + receipt = result.options["budget_search"] + assert selected["search"] == { + "budget_iteration": receipt["selected_budget_iteration"], + "budget_iters": receipt["budget_iters"], + "l0_lambda": receipt["selected_l0_lambda"], + } + matching = [s for s in seen[:-1] if s["search"] == selected["search"]] + assert matching + + +@pytest.mark.parametrize("estimate,target", [(1.0, 5e-324), (1e308, -1e308)]) +def test_overflowing_relative_error_remains_a_counted_null_diagnostic(estimate, target): + _, observer = _collect() + snapshot = observer.bind(names=("tiny",), targets=np.array([target])).snapshot( + np.array([estimate]), + epoch=0, + epochs=1, + iterate=ITERATE_CURRENT, + ) + assert snapshot["targets"][0] == { + "index": 0, + "name": "tiny", + "target": target, + "estimate": estimate, + "relative_error": None, + } + assert snapshot["non_finite_rows"] == 1 + validate_target_snapshot(snapshot) + broken = copy.deepcopy(snapshot) + broken["targets"][0]["relative_error"] = 0.0 + with pytest.raises(TargetSnapshotError, match="relative_error"): + validate_target_snapshot(broken) + + def test_a_store_refuses_to_share_a_directory_with_another_run(): """Sequences restart at 1 per observer, so a shared directory collides.""" import tempfile diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index 44e121b7d..fa3737957 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32","kernel":"calibrate.adam@1","node":"calibrate","node_key":"970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"01531ce6f26c38e3caf0cfcae16af8511a1d891614ec2677632e5f0edb6f0c1d","kernel":"calibrate.adam@1","node":"calibrate","node_key":"275c01005b61afc99862f374e816a045a005ff046cbd6377251bd613eef7db58","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"275c01005b61afc99862f374e816a045a005ff046cbd6377251bd613eef7db58"}},"seed":0} From f92418a859e576ab7fd40919d3e9d75f41bd1ee4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 11:04:28 -0400 Subject: [PATCH 149/255] Add source-qualified survey health coverage graph fragment --- ...us-current-survey-health-coverage.added.md | 4 + .../us-current-health-coverage-20260912.md | 108 +++ .../current_survey_health_coverage.py | 245 +++++++ .../current_survey_health_source.py | 343 ++++++++++ .../us_runtime/graph_current_survey_health.py | 431 ++++++++++++ .../test_us_current_survey_health_coverage.py | 635 ++++++++++++++++++ 6 files changed, 1766 insertions(+) create mode 100644 changelog.d/us-current-survey-health-coverage.added.md create mode 100644 experiments/us-current-health-coverage-20260912.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_health_coverage.py diff --git a/changelog.d/us-current-survey-health-coverage.added.md b/changelog.d/us-current-survey-health-coverage.added.md new file mode 100644 index 000000000..5443947b8 --- /dev/null +++ b/changelog.d/us-current-survey-health-coverage.added.md @@ -0,0 +1,4 @@ +Add a source-qualified health coverage graph fragment for the common US survey +frame and its PUF clones. Preserve literal source and allocation codes, explicit +unknowns, and incoming amount fields; keep broader ACS concepts as documented +gaps instead of manufacturing narrower coverage flags. diff --git a/experiments/us-current-health-coverage-20260912.md b/experiments/us-current-health-coverage-20260912.md new file mode 100644 index 000000000..586f8a8af --- /dev/null +++ b/experiments/us-current-health-coverage-20260912.md @@ -0,0 +1,108 @@ +# Current health coverage on the survey multispine + +This lane adds source-qualified interview-date coverage to the common ACS+ASEC +frame and its PUF clones. It does not estimate eligibility, annual receipt, +insurance costs, or a common observation date. The current ASEC arm is income +year 2024 / survey year 2025; ACS coverage was observed during 2024. Older ASEC +income-year vintages are outside this first attachment. + +## Source definitions and mapping decisions + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf) +defines these current recodes for all persons, with 1=yes and 2=no. Corresponding +`I_NOW_*` flags distinguish reported, hot-deck, logical, and whole-unit imputation. +The [2024 ACS questionnaire, item 16](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf) +asks all respondents about current coverage. The [ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines HINS1–7 (printed pages 37–38) and allocation/coverage-edit flags (126). + +| NATIONAL_CD input | ASEC source | ACS exact source in this first stage | +| --- | --- | --- | +| `has_esi` | `NOW_GRP` | `HINS1` | +| `has_marketplace_health_coverage_at_interview` | `NOW_MRK` | Gap: `HINS2` combines direct-purchase types | +| `has_non_marketplace_direct_purchase_health_coverage_at_interview` | `NOW_NONM` | Same gap | +| `has_medicaid_health_coverage_at_interview` | `NOW_CAID` | Gap: `HINS4` includes CHIP/other means-tested coverage | +| `has_other_means_tested_health_coverage_at_interview` | `NOW_OTHMT` | Same gap | +| `has_tricare_health_coverage_at_interview` | `NOW_MIL` | Gap: `HINS5` includes other military coverage | +| `has_champva_health_coverage_at_interview` | `NOW_CHAMPVA` | No separate ACS item | +| `has_va_health_coverage_at_interview` | `NOW_VACARE` | Keep `HINS6` distinct pending CHAMPVA recode reconciliation | +| `has_indian_health_service_coverage_at_interview` | `NOW_IHSFLG` | `HINS7` | + +The existing `cps_carried._fill_health_coverage_inputs` supplies the field roster +and most ASEC mappings, but its absent-column-to-zero helper is unsuitable here. +Its Medicaid mapping uses `NOW_MCAID`, the broader Medicaid/CHIP/other-means-tested +aggregate, rather than `NOW_CAID`. Both names and their distinction also appear +in the [2023](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar23.pdf) and +[2024](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf) dictionaries. +The old pipeline remains untouched; this stage refuses ownership collisions. + +This narrower mapping is supported by the pinned consumer source, rather than +inferred from the input name alone. PolicyEngine-US 1.819.0 declares the Medicaid +input as person-level current Medicaid coverage, declares other means-tested +coverage separately, and uses the Medicaid input alongside `receives_medicaid` +for California Medi-Cal continuity. The checked files are +`variables/household/expense/health/has_medicaid_health_coverage_at_interview.py` +(SHA256 `0500c4e86120b540e6e2f09fc108123758d8f9863c0e4e4bf94b178dd50e09b8`) +and `variables/gov/states/ca/chhs/is_ca_medicaid_immigration_status_eligible.py` +(`6902d01792b8d0902c2693edb78f5ba2554a34f95d4073bd4e85e4ad69ce41a5`). +Only source text was inspected; this is not an engine execution or a blanket +reinterpretation of earlier datasets. Both `NOW_CAID` and `NOW_MCAID`, with their +allocation flags, remain in the graph. A later CMS comparison must declare +whether its target covers Medicaid alone or Medicaid plus CHIP. + +The [ACS subject definitions, health insurance](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +describe source editing and the 2019 change to current VA enrollment wording. +The [Census programming guide](https://www.census.gov/topics/health/health-insurance/guidance/programming-code/acs-estimates.html) +still groups CHAMPVA with HINS6. Consequently this stage does not assume that +HINS6 is interchangeable with the narrower ASEC VACARE recode. Raw HINS2–6 stay +inspectable; no missing concept becomes a false flag. `HIMRKS` measures subsidized +direct purchase, not all Marketplace enrollment, and cannot repair that split. + +## Graph and authority + +Borrow the original checked `SurveyPuf55Run` and its retained survey preparation. +Capture the exact original ASEC PERSON member and ACS person archive under the +existing native owners' pins. Match source household/person coordinates to the +preparation's original person ledger; never join by row position. Record the +source codes, allocation literals, status, knownness, source year and survey year. + +The fragment exposes a source projection artifact, a CREATE source population, +literal code columns, and nine named per-field recodes. CREATE requires the +existing `survey_population_source` reference from the original source registry; +it introduces no additional source file or issuer. A post-PUF attachment follows the retained +original-person link to both clones, verifies native channel/ID and clone roster, +and adds nullable booleans plus provenance. This stage has no fitted model. +Future disaggregation models need separate declarations and assessment. + +One fixed country enrichment host assembles the amount and health fragments over +the original checked PUF handle. The health fragment receives the explicit +receiving version and the amount attachment's typed artifact edge. Its ordinary +attachment preserves all incoming columns, including amount unknowns. It does +not return a separate whole-population branch or issue a public health handle. + +Portable values do not issue authority. The owning consumer must requalify the +retained parent immediately before consumption and after its last relevant I/O, +then seal the complete successor before returning it. The fragment checks its +retained physical values around each kernel callback; callback success itself +grants no source authority. The host owns source and kernel registries, typed +artifact/store bindings, replay checks, and complete successor verification. + +## Bounded verification + +The 31-case suite passes over invented records. It exercises the real source +catalogues, preparation issuers, original-member qualifier and requalification; +literal yes/no/allocation and unknown handling; exact source-coordinate joins; +clone identity and collision refusal; all prior amount-column preservation; +and a 13-node actual graph cold run plus required replay. Independent complete +population reconstruction matches every observed health-node output. The graph +fixture's qualified value is explicitly descriptive; it does not claim a new +source admission. A separate test obtains the qualified value through the real +issuer path using privately pinned invented source bytes. + +The source-v3 guarded run closed with result 0: 20.61 seconds wall, 19.95 seconds +CPU and 490.9 MB peak RSS. Earlier failed fixtures remain preserved, including +the origin-coordinate type mismatch exposed by the real qualification test. +No native microdata, country engine, public upload or model benchmark was used. +This verifies the fragment, not a launchable population. Actual-source replay, +the combined host's final retained-parent fence and release coverage checks +remain its consumer's responsibilities; the seven unresolved ACS concepts stay +explicit gaps. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py new file mode 100644 index 000000000..066a13553 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py @@ -0,0 +1,245 @@ +"""Interview-date coverage recodes; detached values grant no source authority. + +Only literal source yes/no codes determine coverage. Allocation is separate +provenance, and a broader ACS item never becomes a narrower canonical flag. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import support_provenance as provenance +from .cps_carried import US_REPORTED_COVERAGE_PERSON_INPUTS + +PROTOCOL = "microcosm.us.current-survey-health-coverage.v1" +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +@dataclass(frozen=True) +class CoverageField: + output: str + asec: str + acs: str | None + acs_gap: str | None = None + + @property + def columns(self): + return (self.output, self.output + "__known", self.output + "__source_status") + + +# Separate NOW_CAID from the broader NOW_MCAID legacy mapping. These are +# source recodes, not benefit eligibility or prior-calendar-year coverage. +FIELDS = ( + CoverageField("has_esi", "NOW_GRP", "HINS1"), + CoverageField( + "has_marketplace_health_coverage_at_interview", + "NOW_MRK", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "NOW_NONM", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_medicaid_health_coverage_at_interview", + "NOW_CAID", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_other_means_tested_health_coverage_at_interview", + "NOW_OTHMT", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_tricare_health_coverage_at_interview", + "NOW_MIL", + None, + "military_coverage_type_unresolved", + ), + CoverageField( + "has_champva_health_coverage_at_interview", + "NOW_CHAMPVA", + None, + "champva_not_separately_observed", + ), + CoverageField( + "has_va_health_coverage_at_interview", + "NOW_VACARE", + None, + "va_champva_recode_scope_unresolved", + ), + CoverageField( + "has_indian_health_service_coverage_at_interview", "NOW_IHSFLG", "HINS7" + ), +) +ASEC_VALUE_COLUMNS = tuple(f.asec for f in FIELDS) + ("NOW_MCAID",) +ASEC_ALLOCATION_COLUMNS = tuple("I_" + c for c in ASEC_VALUE_COLUMNS) +ACS_VALUE_COLUMNS = tuple("HINS" + str(i) for i in range(1, 8)) +ACS_ALLOCATION_COLUMNS = tuple("FHINS" + str(i) + "P" for i in range(1, 8)) +ACS_EDIT_COLUMNS = ("FHINS3C", "FHINS4C", "FHINS5C") +RAW_COLUMNS = ( + *ASEC_VALUE_COLUMNS, + *ASEC_ALLOCATION_COLUMNS, + *ACS_VALUE_COLUMNS, + *ACS_ALLOCATION_COLUMNS, + *ACS_EDIT_COLUMNS, +) +SOURCE_PREFIX = "health_source_" + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_HEALTH_" + reason) + + +def _field(name): + require( + set(f.output for f in FIELDS) == set(US_REPORTED_COVERAGE_PERSON_INPUTS), + "PROFILE_ROSTER", + ) + matches = [f for f in FIELDS if f.output == name] + require(len(matches) == 1, "FIELD") + return matches[0] + + +def recode(tokens, allocations, *, survey): + """Keep missing, unrecognized, allocated and reported source states apart. + + This classifier accepts literal strings only. A valid source yes/no value + survives unknown allocation provenance, without claiming a raw response. + Neither blank nor an out-of-dictionary code means no coverage. + """ + require(survey in ("asec", "acs"), "SURVEY") + values, flags = tuple(tokens), tuple(allocations) + require( + len(values) == len(flags) + and all(type(x) is str and len(x) <= 64 for x in (*values, *flags)), + "LITERAL_CONTRACT", + ) + allocation_labels = ( + {"0": "reported", "1": "hotdeck", "2": "logical", "3": "whole_unit"} + if survey == "asec" + else {"0": "not_allocated", "1": "allocated"} + ) + canonical, known, status = [], [], [] + for token, allocation in zip(values, flags, strict=True): + valid = token in ("1", "2") + canonical.append(token == "1" if valid else pd.NA) + known.append(valid) + if not valid: + status.append( + "missing_source_value" if token == "" else "unrecognized_source_value" + ) + else: + method = allocation_labels.get(allocation, "allocation_unknown") + status.append(("source_yes_" if token == "1" else "source_no_") + method) + return pd.DataFrame( + { + "value": pd.array(canonical, dtype="boolean"), + "known": np.asarray(known, dtype=bool), + "status": pd.array(status, dtype=STRING_DTYPE), + } + ) + + +def recode_field(raw, name): + """Transform one field on the full original-person axis, retaining gaps.""" + spec = _field(name) + require( + type(raw) is pd.DataFrame + and raw.index.is_unique + and raw.index.name == "person_id" + and {"source", *RAW_COLUMNS} <= set(raw) + and raw.source.isin(("acs", "asec")).all(), + "RAW_AXIS", + ) + result = pd.DataFrame(index=raw.index.copy()) + result[spec.output] = pd.array([pd.NA] * len(raw), dtype="boolean") + result[spec.output + "__known"] = False + result[spec.output + "__source_status"] = pd.array( + [pd.NA] * len(raw), dtype=STRING_DTYPE + ) + for survey, raw_name in (("asec", spec.asec), ("acs", spec.acs)): + mask = raw.source.eq(survey) + if raw_name is None: + result.loc[mask, spec.output + "__source_status"] = ( + "semantic_gap:" + spec.acs_gap + ) + continue + flag_name = "I_" + raw_name if survey == "asec" else "F" + raw_name + "P" + part = recode(raw.loc[mask, raw_name], raw.loc[mask, flag_name], survey=survey) + for output, value in zip(spec.columns, part, strict=True): + result.loc[mask, output] = part[value].array + return result + + +def source_columns(raw): + """Namespace literal observations so they cannot impersonate output leaves.""" + require(raw.index.name == "person_id" and raw.index.is_unique, "RAW_AXIS") + result = pd.DataFrame(index=raw.index.copy()) + for name in RAW_COLUMNS: + require(all(x is None or type(x) is str for x in raw[name]), "RAW_LITERAL_TYPE") + result[SOURCE_PREFIX + name] = pd.array(raw[name], dtype=STRING_DTYPE) + for name in ("source", "source_year", "survey_year"): + result[SOURCE_PREFIX + name] = ( + pd.array(raw[name], dtype=STRING_DTYPE) + if name == "source" + else raw[name].array.copy() + ) + return result + + +def attach_columns(origins, receiving, columns): + """Fan original observations out to exactly their unchanged two clones. + + The owning host authenticates origins and source columns before calling. + Existing fields are never overwritten, including another family's leaves. + """ + require( + type(origins) is pd.DataFrame + and type(columns) is pd.DataFrame + and origins.index.is_unique + and origins.index.name == "person_id" + and columns.index.equals(origins.index) + and columns.columns.is_unique, + "ATTACH_SOURCE_AXIS", + ) + people = receiving.person + ids = people.person_id + original = people[provenance.support_source_id_column("person")].to_numpy() + clone = people[provenance.support_clone_index_column("person")].to_numpy() + native = people[provenance.spine_source_id_column("person")].to_numpy() + channel = people[provenance.support_channel_column("person")].to_numpy() + require( + ids.dtype == np.dtype("int64") + and ids.is_unique + and original.dtype == clone.dtype == native.dtype == np.dtype("int64") + and len(original) == 2 * len(origins) + and set(original) == set(origins.index), + "CLONE_SOURCE_ROSTER", + ) + expected = origins.reindex(original) + require( + np.array_equal(native, expected.native_person_id.to_numpy()) + and np.array_equal(channel, expected.source.to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + pairs = pd.MultiIndex.from_arrays((original, clone)) + require(pairs.is_unique and np.isin(clone, (0, 1)).all(), "CLONE_PAIR") + require(not set(columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + return { + ("person", name): pd.Series( + columns[name].reindex(original).array.copy(), + index=pd.Index(ids.to_numpy(), name="person_id"), + name=name, + ) + for name in columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py new file mode 100644 index 000000000..f9b31c825 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py @@ -0,0 +1,343 @@ +"""Health-code projection from the retained original ACS and ASEC sources. + +The preparation issues source identity. This module only borrows it, captures +its pinned original members and returns descriptive values. A host must retain +that owner and requalify after its last relevant I/O before returning output. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import asec_coverage_authentication as asec +from . import current_survey_health_coverage as health +from . import source_csv_builtin +from . import survey_population_preparation as source + +require = health.require +ASEC_KEYS = ("PERIDNUM", "PH_SEQ", "A_LINENO") +ACS_KEYS = ("SERIALNO", "SPORDER") +ASEC_COLUMNS = (*ASEC_KEYS, *health.ASEC_VALUE_COLUMNS, *health.ASEC_ALLOCATION_COLUMNS) +ACS_COLUMNS = ( + *ACS_KEYS, + *health.ACS_VALUE_COLUMNS, + *health.ACS_ALLOCATION_COLUMNS, + *health.ACS_EDIT_COLUMNS, +) + + +def _key(row, survey): + if survey == "asec": + require( + re.fullmatch(r"[0-9]{22}", row["PERIDNUM"], re.ASCII) is not None, + "ASEC_PERSON_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,5}", row["PH_SEQ"], re.ASCII) is not None, + "ASEC_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["A_LINENO"], re.ASCII) is not None + and 1 <= int(row["A_LINENO"]) <= 20 + and int(row["PH_SEQ"]) > 0, + "ASEC_LINE_KEY", + ) + return (int(row["PH_SEQ"]), row["PERIDNUM"], int(row["A_LINENO"])) + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "ACS_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "ACS_LINE_KEY", + ) + return (row["SERIALNO"], str(int(row["SPORDER"])), int(row["SPORDER"])) + + +def _scan(stream, *, survey, wanted, selected, maximum): + """Exhaust a byte-bounded literal member; selected records remain strings. + + This helper grants no authority to a file, row count or caller key list. + A complete authenticated source/roster is bound by the enclosing qualifier. + """ + require( + survey in ("asec", "acs") + and source_csv_builtin.capture_csv_reader(csv) is not None, + "SCANNER_OR_CSV_BINDING", + ) + columns = ASEC_COLUMNS if survey == "asec" else ACS_COLUMNS + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(columns) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in columns] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(columns, (values[p] for p in positions), strict=True)) + key = _key(row, survey) + require(all(len(row[c]) <= 64 for c in columns), "SOURCE_TOKEN_BOUND") + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(preparation_frame, document): + payload = document["origins"]["persons"] + original = pd.DataFrame(payload["rows"], columns=payload["columns"]) + required = { + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + } + require( + required <= set(original) + and original.person_id.dtype == np.dtype("int64") + and original.person_id.is_unique, + "ORIGIN_ROSTER", + ) + original = original.set_index("person_id", drop=True) + people = preparation_frame.person + require( + np.array_equal(original.index.to_numpy(), people.person_id.to_numpy()) + and np.array_equal( + original.source.to_numpy(), + people[health.provenance.support_channel_column("person")].to_numpy(), + ) + and np.array_equal( + original.selected_receiving_person_id.to_numpy(), + people[health.provenance.spine_source_id_column("person")].to_numpy(), + ), + "ORIGIN_FRAME_IDENTITY", + ) + require( + original.source.isin(("acs", "asec")).all() + and original.source_year.eq(2024).all() + and original.survey_year.eq( + original.source.map({"acs": 2024, "asec": 2025}) + ).all(), + "ORIGIN_PERIOD", + ) + original["native_person_id"] = original.selected_receiving_person_id + keys = {} + for pid, row in original.iterrows(): + household = row.raw_native_household_id + require( + type(household) is str + and type(row.raw_native_person_id) is str + and type(row.native_line_numeric_original) is str, + "ORIGIN_COORDINATE_TYPE", + ) + line = row.native_line_numeric_original + if row.source == "asec": + key = _key( + { + "PH_SEQ": household, + "PERIDNUM": row.raw_native_person_id, + "A_LINENO": line, + }, + "asec", + ) + else: + key = _key({"SERIALNO": household, "SPORDER": line}, "acs") + require(row.raw_native_person_id == key[1], "ORIGIN_PERSON_KEY") + require((row.source, key) not in keys, "DUPLICATE_ORIGIN") + keys[row.source, key] = pid + return original, keys + + +def _combine(origins, keys, selected): + require(set(selected) == {"acs", "asec"}, "SOURCE_ARMS") + require( + all(set(selected[s]) == {k for arm, k in keys if arm == s} for s in selected), + "SELECTED_SOURCE_ROSTER", + ) + raw = origins.loc[:, ["source", "source_year", "survey_year"]].copy() + for column in health.RAW_COLUMNS: + raw[column] = pd.Series([None] * len(raw), index=raw.index, dtype=object) + for (survey, key), pid in keys.items(): + row = selected[survey][key] + require(_key(row, survey) == key, "SOURCE_COORDINATE_CHANGED") + for column in health.RAW_COLUMNS: + if column in row: + raw.at[pid, column] = row[column] + return raw + + +@dataclass(frozen=True) +class QualifiedSurveyHealthCoverage: + """A detached projection, not a substitute for the retained source owner.""" + + source_frame: object + origins: pd.DataFrame + raw: pd.DataFrame + projection: bytes + evidence: dict + + +def qualify_current_survey_health(preparation): + """Re-read pinned originals without native Frame or engine reconstruction. + + This entry point does read original microdata when called by an authorized + build. Tests can exercise the same path with privately pinned invented bytes. + """ + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + document = json.loads(entry[1]) + origins, keys = _origins(state.frame, document) + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "ASEC_NATIVE_ISSUANCE", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "ASEC_NATIVE_PERIOD", + ) + pin = tuple(p for p in asec._MEMBER_PINS if p[0] == 2024) + require(len(pin) == 1, "ASEC_PIN_ROSTER") + year, member, archive_digest, digest, rows, size = pin[0] + retained = tuple( + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ) + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive_digest), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "ASEC_MEMBER_BINDING", + ) + acs_owned = source.acs_catalogue._lookup(state.catalogues[0]) + acs_doc = json.loads(acs_owned.receipt) + person_pins = tuple(p for p in acs_owned.pins if p[0] == "person") + require( + len(person_pins) == 1 + and acs_doc["source_year"] == acs_doc["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + _role, acs_name, acs_digest, acs_size = person_pins[0] + selected = {"asec": {}, "acs": {}} + with tempfile.TemporaryDirectory(prefix="microcosm-health-source-") as temporary: + directory = Path(temporary) + captured = directory / member + identity = asec._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[asec._BODY_MAX], + ) + with captured.open("rb") as stream: + count = _scan( + stream, + survey="asec", + wanted={k for s, k in keys if s == "asec"}, + selected=selected["asec"], + maximum=rows, + ) + require( + count == rows + and asec._identity(captured.stat(follow_symlinks=False)) == identity + and housing._persisted_sha(captured, size) == digest, + "ASEC_CAPTURE_CHANGED", + ) + captured = directory / acs_name + require( + housing._copy( + state.root / "acs" / acs_name, captured, acs_size, exact_size=acs_size + ) + == acs_digest, + "ACS_CAPTURE_DIGEST", + ) + count = 0 + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + survey="acs", + wanted={k for s, k in keys if s == "acs"}, + selected=selected["acs"], + maximum=acs_doc["counts"]["people"] - count, + ) + else: + while stream.read(65536): + pass # Exhaust auxiliary members so their CRC is checked. + require( + count == acs_doc["counts"]["people"] + and housing._persisted_sha(captured, acs_size) == acs_digest, + "ACS_CAPTURE_CHANGED", + ) + raw = _combine(origins, keys, selected) + projection = raw.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": health.PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(acs_owned.receipt).hexdigest(), + "asec_person_member_sha256": digest, + "acs_person_archive_sha256": acs_digest, + "asec_coverage_observation_year": 2025, + "acs_coverage_observation_year": 2024, + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "selected_rows": len(raw), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "release_eligible": False, + } + # The original issuer checks source bytes after the last capture/cleanup I/O. + require(preparation._checked() is entry, "PREPARATION_CHANGED") + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and source.acs_catalogue._lookup(state.catalogues[0]) is acs_owned, + "FINAL_OWNER", + ) + return QualifiedSurveyHealthCoverage( + state.frame, origins, raw, projection, evidence + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py new file mode 100644 index 000000000..73f4c0b67 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py @@ -0,0 +1,431 @@ +"""Private health fragment for the one country enrichment host. + +The host retains the actual checked PUF run, preparation, kernel registry and +source files. This fragment never issues a run or creates a second receiving +branch. Its final ordinary attachment preserves the incoming population version. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_health_coverage as health +from . import current_survey_health_source as source +from . import graph_full_puf_enrichment as physical +from .graph_survey_population import SOURCE_NAME + +qualify_health_coverage = source.qualify_current_survey_health +require = health.require +SOURCE_NODE = "survey_health.source" +RAW_NODE = "survey_health.raw_columns" +RECODE_PREFIX = "survey_health.recode." +ATTACH_NODE = "survey_health.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_health_projection", 1) +COLUMNS_TYPE = ArtifactType("microcosm.us.current_survey_health_columns", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_health_attachment", 1) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _table_bytes(table): + return table.to_json(orient="table").encode() + + +def health_coverage_seal(qualified): + require(type(qualified) is source.QualifiedSurveyHealthCoverage, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.raw.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + source.source._frame_identity(qualified.source_frame), + physical._table_stamp(qualified.origins), + physical._table_stamp(qualified.raw), + qualified.projection, + _json(qualified.evidence), + ) + + +def _source_edge(): + return ArtifactInput( + "health_projection", SOURCE_NODE, "projection", PROJECTION_TYPE + ) + + +def _column_artifact(node_id, alias): + return ArtifactInput(alias, node_id, "columns", COLUMNS_TYPE) + + +def _dtype(series): + return population_ops.token_for_dtype(series.dtype) + + +def _outputs(table): + return tuple(Owned("person", name, _dtype(table[name])) for name in table) + + +def _completed(qualified): + return pd.concat( + [ + health.source_columns(qualified.raw), + *(health.recode_field(qualified.raw, f.output) for f in health.FIELDS), + ], + axis=1, + ) + + +def _params(qualified): + return { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256(qualified.projection).hexdigest(), + "source_evidence": _json(qualified.evidence).decode(), + } + + +def health_coverage_nodes(qualified, *, receiving_version, after): + health_coverage_seal(qualified) + require( + type(receiving_version) is str + and receiving_version + and type(after) is ArtifactInput, + "FRAGMENT_INPUT", + ) + params = _params(qualified) + raw = health.source_columns(qualified.raw) + create = Node( + SOURCE_NODE, + "us.survey_health.source@1", + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=(Owned("person", "health_native_person_id", "int64"),), + params=params, + artifact_inputs=(after,), + artifact_outputs=(ArtifactOutput("projection", PROJECTION_TYPE),), + description="Borrow original ACS/ASEC person support and retain authenticated current-coverage source projection.", + ) + literal = Node( + RAW_NODE, + "us.survey_health.raw_columns@1", + population=SOURCE_NODE, + outputs=_outputs(raw), + params=params, + artifact_inputs=(_source_edge(),), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description="Expose literal coverage, allocation and edit codes with survey observation years.", + ) + recodes = [] + for field in health.FIELDS: + inputs = [ + health.SOURCE_PREFIX + "source", + health.SOURCE_PREFIX + field.asec, + health.SOURCE_PREFIX + "I_" + field.asec, + ] + if field.acs: + inputs += [ + health.SOURCE_PREFIX + field.acs, + health.SOURCE_PREFIX + "F" + field.acs + "P", + ] + recodes.append( + Node( + RECODE_PREFIX + field.output, + "us.survey_health.recode@1", + population=SOURCE_NODE, + inputs=(Slice("person", tuple(inputs)),), + outputs=_outputs(health.recode_field(qualified.raw, field.output)), + params={**params, "field": field.output}, + artifact_inputs=( + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + ), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description=f"Current coverage: {field.asec}; ACS {field.acs or field.acs_gap}. Missing codes remain unknown.", + ) + ) + identity = tuple( + f("person") + for f in ( + health.provenance.support_source_id_column, + health.provenance.support_clone_index_column, + health.provenance.spine_source_id_column, + health.provenance.support_channel_column, + ) + ) + attach = Node( + ATTACH_NODE, + "us.survey_health.attach@1", + population=receiving_version, + inputs=(Slice("person", identity),), + outputs=_outputs(_completed(qualified)), + params=params, + artifact_inputs=( + after, + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + *( + _column_artifact(n.id, "health_field_" + str(i)) + for i, n in enumerate(recodes) + ), + ), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Attach only exact source-person observations and explicit unknowns to both PUF clones; preserve all incoming fields.", + ) + return (create, literal, *recodes, attach) + + +def _expected_artifacts(qualified): + return { + (SOURCE_NODE, "projection"): qualified.projection, + (RAW_NODE, "columns"): _table_bytes(health.source_columns(qualified.raw)), + **{ + (RECODE_PREFIX + f.output, "columns"): _table_bytes( + health.recode_field(qualified.raw, f.output) + ) + for f in health.FIELDS + }, + } + + +def _check_artifacts(node, artifacts, qualified): + require(set(artifacts) == {a.name for a in node.artifact_inputs}, "ARTIFACT_ROSTER") + expected = _expected_artifacts(qualified) + for edge in node.artifact_inputs: + value = artifacts[edge.name] + require( + type(value) is ArtifactValue + and value.type == edge.type + and type(value.payload) is bytes, + "ARTIFACT_TYPE", + ) + if (edge.producer, edge.artifact) in expected: + require( + value.payload == expected[edge.producer, edge.artifact], + "ARTIFACT_PAYLOAD", + ) + # The incoming amount edge is authenticated by the owning host, which + # retains its node/store keys. The fragment never treats its JSON as an issuer. + + +def _result(qualified, node, people): + if node.id == SOURCE_NODE: + original = source.source._copy_source(qualified.source_frame) + # A projection branch keeps original support/design and structural IDs; + # its coverage cells are introduced visibly by the following nodes. + tables = {} + for entity in original.entities: + columns = [original.schema.entity_id_column(entity)] + if entity == original.schema.person_entity: + columns += [ + original.schema.membership_column(e) + for e in original.schema.group_entities + ] + tables[entity] = original.table(entity).loc[:, columns].copy() + require( + np.array_equal( + qualified.origins.index.to_numpy(), original.person.person_id.to_numpy() + ), + "CREATE_ORIGIN_AXIS", + ) + tables["person"]["health_native_person_id"] = ( + qualified.origins.native_person_id.to_numpy(copy=True) + ) + projected = Frame( + tables, + original.schema, + dict(original._weights), + original.strata, + metadata=original.metadata, + mass_log=original.mass_log, + ) + return KernelResult( + frame=projected, artifacts={"projection": qualified.projection} + ) + if node.id == RAW_NODE: + table = health.source_columns(qualified.raw) + elif node.id.startswith(RECODE_PREFIX): + table = health.recode_field(qualified.raw, node.params["field"]) + else: + require(node.id == ATTACH_NODE, "NODE_ID") + columns = health.attach_columns( + qualified.origins, SimpleNamespace(person=people), _completed(qualified) + ) + return KernelResult( + columns=columns, + artifacts={ + "attachment": _json( + { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256( + qualified.projection + ).hexdigest(), + "receiving_version": node.population, + "rows": len(people), + "source_admission_issued": False, + "release_eligible": False, + } + ) + }, + ) + require( + people is not None + and np.array_equal(people.person_id.to_numpy(), table.index.to_numpy()), + "SOURCE_PERSON_AXIS", + ) + return KernelResult( + columns={("person", c): table[c].copy() for c in table}, + artifacts={"columns": _table_bytes(table)}, + ) + + +def expected_health_population(node_id, incoming, *, qualified, node, artifacts): + """Reconstruct the complete output, independently of executor cache results.""" + require(node_id == node.id, "EXPECTED_NODE") + _check_artifacts(node, artifacts, qualified) + result = _result( + qualified, node, None if incoming is None else incoming.frame.person + ) + if node.structural is StructuralDelta.CREATE: + require(incoming is None, "CREATE_INCOMING") + return population_ops.Population.from_frame(result.frame, node.id) + require(incoming is not None, "ORDINARY_INCOMING") + return population_ops.patch(incoming, node, result) + + +class _HealthKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, qualified, nodes, require_current, *, create=False, ref): + self.ref, self.qualified, self.nodes = ref, qualified, nodes + self.require_current, self.seal = ( + require_current, + health_coverage_seal(qualified), + ) + if create: + self.capabilities = replace( + self.capabilities, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + health, + source, + physical, + source.source, + source.housing, + source.records, + source.asec, + source.source_csv_builtin, + sys.modules[health.__package__ + ".cps_carried"], + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + context.node in self.nodes and context.node.kernel == self.ref, + "KERNEL_NODE", + ) + _check_artifacts(context.node, context.artifacts, self.qualified) + if context.node.id.startswith(RECODE_PREFIX): + expected = health.source_columns(self.qualified.raw) + actual = context.tables["person"].set_index("person_id") + for selection in context.node.inputs: + for column in selection.columns: + require( + population_ops.storage_equal(actual[column], expected[column]), + "RECODE_SOURCE_SLICE", + ) + result = _result(self.qualified, context.node, context.tables.get("person")) + result_seal = ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + frame_seal = ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + artifacts = tuple(sorted(result.artifacts.items())) + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + == result_seal + and ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + == frame_seal + and tuple(sorted(result.artifacts.items())) == artifacts, + "RESULT_CHANGED", + ) + return result + + +def health_coverage_kernels(qualified, *, receiving_version, after, require_current): + nodes = health_coverage_nodes( + qualified, receiving_version=receiving_version, after=after + ) + require(callable(require_current), "HOST_CALLBACK") + return tuple( + _HealthKernel( + qualified, + nodes, + require_current, + create=ref == "us.survey_health.source@1", + ref=ref, + ) + for ref in dict.fromkeys(n.kernel for n in nodes) + ) diff --git a/packages/microcosm-build/tests/test_us_current_survey_health_coverage.py b/packages/microcosm-build/tests/test_us_current_survey_health_coverage.py new file mode 100644 index 000000000..829b7b039 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_health_coverage.py @@ -0,0 +1,635 @@ +"""Invented health source values, exact source joins and two-clone attachment.""" + +import hashlib +import io +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_health_coverage as health +from microcosm.build.us_runtime import current_survey_health_source as source + + +def _health_acs_person(original): + """Decorate only an invented fixture's source-row constructor, before pins.""" + + def person(*args, **kwargs): + row = original(*args, **kwargs) + row.update({c: "2" for c in health.ACS_VALUE_COLUMNS}) + row.update( + {c: "0" for c in (*health.ACS_ALLOCATION_COLUMNS, *health.ACS_EDIT_COLUMNS)} + ) + row["HINS1"] = "1" + row["HINS7"] = "" if int(row["SPORDER"]) == 2 else "2" + return row + + return person + + +def _health_asec_table(table): + """Add source fields without changing existing money or coverage literals.""" + table = table.copy(deep=True) + for c in health.ASEC_VALUE_COLUMNS: + if c not in table: + table[c] = "2" + for c in health.ASEC_ALLOCATION_COLUMNS: + if c not in table: + table[c] = "0" + return table + + +def _health_source_arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as preparation_fixture + from test_us_current_asec_demographics import _demographic_arguments + + from microcosm.build.us_runtime import asec_person_income_source as restoration + from microcosm.build.us_runtime import current_asec_demographics as demographic + + monkeypatch.setattr( + preparation_fixture, "_person", _health_acs_person(preparation_fixture._person) + ) + arguments = _demographic_arguments(tmp_path, monkeypatch, zero=False) + source_dir = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in source.asec._MEMBER_PINS: + path = source_dir / member + original = pd.read_csv(path, dtype=str, keep_default_na=False) + table = _health_asec_table(original) + pd.testing.assert_frame_equal(table.loc[:, original.columns], original) + table.to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in (source.asec, restoration, demographic.demographic): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "health-restored-money" + restoration.restore_asec_person_income_source( + source_dir / "parent.h5", + source_dir / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, + source_dir / "person-income-attachment.h5", + ) + return arguments + + +def test_actual_health_qualifier_uses_retained_original_source_issuers( + tmp_path, monkeypatch +): + from microcosm.build.us_runtime import graph_current_survey_health as graph + + arguments = _health_source_arguments(tmp_path, monkeypatch) + prepared = source.source.prepare_authenticated_survey_population(**arguments) + original = prepared.checked_view().frame + retained = {e: original.table(e).copy(deep=True) for e in original.entities} + qualified = graph.qualify_health_coverage(prepared) + assert qualified.source_frame is original + assert qualified.origins.index.tolist() == original.person.person_id.tolist() + assert set(qualified.raw.source) == {"asec", "acs"} + assert qualified.evidence["asec_coverage_observation_year"] == 2025 + assert qualified.evidence["acs_coverage_observation_year"] == 2024 + assert not qualified.evidence["source_admission_issued"] + assert not qualified.evidence["unallocated_observation_claim"] + assert not qualified.evidence["release_eligible"] + for field in health.FIELDS: + result = health.recode_field(qualified.raw, field.output) + asec = qualified.raw.source.eq("asec") + assert result.loc[asec, field.output + "__known"].all() + if field.acs is None: + assert result.loc[~asec, field.output].isna().all() + acs = qualified.raw.source.eq("acs") + assert qualified.raw.loc[acs, "NOW_CAID"].map(lambda v: v is None).all() + assert qualified.raw.loc[acs, "HINS7"].eq("").any() + for entity, expected in retained.items(): + pd.testing.assert_frame_equal( + expected, original.table(entity), check_exact=True + ) + seal = graph.health_coverage_seal(qualified) + fresh = graph.qualify_health_coverage(prepared) + assert graph.health_coverage_seal(fresh) == seal + qualified.raw.loc[qualified.raw.source.eq("asec"), "NOW_CAID"] = "1" + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + graph.health_coverage_seal(qualified) + assert graph.health_coverage_seal(fresh) == seal + + +def invented_raw(): + index = pd.Index([10, 20], name="person_id") + raw = pd.DataFrame( + { + "source": ["asec", "acs"], + "source_year": [2024, 2024], + "survey_year": [2025, 2024], + }, + index=index, + ) + for c in health.RAW_COLUMNS: + raw[c] = pd.Series([None] * len(index), index=index, dtype=object) + for c in health.ASEC_VALUE_COLUMNS: + raw.loc[10, c] = "2" + for c in health.ASEC_ALLOCATION_COLUMNS: + raw.loc[10, c] = "0" + for c in health.ACS_VALUE_COLUMNS: + raw.loc[20, c] = "2" + for c in (*health.ACS_ALLOCATION_COLUMNS, *health.ACS_EDIT_COLUMNS): + raw.loc[20, c] = "0" + raw.loc[10, "NOW_GRP"] = raw.loc[20, "HINS1"] = "1" + raw.loc[10, "NOW_MCAID"] = "1" # Broad aggregate yes, exact Medicaid no. + raw.loc[20, "HINS7"] = "" + return raw + + +def invented_origins(): + return pd.DataFrame( + {"source": ["asec", "acs"], "native_person_id": [2, 3]}, + index=pd.Index([10, 20], name="person_id"), + ) + + +def invented_receiving(): + p = health.provenance + return SimpleNamespace( + person=pd.DataFrame( + { + "person_id": np.array([400, 100, 300, 200], dtype="int64"), + p.support_source_id_column("person"): np.array( + [20, 10, 20, 10], dtype="int64" + ), + p.support_clone_index_column("person"): np.array( + [1, 0, 0, 1], dtype="int64" + ), + p.spine_source_id_column("person"): np.array( + [3, 2, 3, 2], dtype="int64" + ), + p.support_channel_column("person"): ["acs", "asec", "acs", "asec"], + "unemployment_compensation": np.array( + [np.nan, 0, np.nan, 0], dtype="float64" + ), + } + ) + ) + + +@pytest.mark.parametrize( + "survey,flags,labels", + [ + ( + "asec", + ["0", "1", "2", "3", "", "9"], + [ + "reported", + "hotdeck", + "logical", + "whole_unit", + "allocation_unknown", + "allocation_unknown", + ], + ), + ( + "acs", + ["0", "1", "", "2", "0", "1"], + [ + "not_allocated", + "allocated", + "allocation_unknown", + "allocation_unknown", + "not_allocated", + "allocated", + ], + ), + ], +) +def test_yes_no_and_allocation_are_separate(survey, flags, labels): + result = health.recode(["1", "2", "1", "2", "1", "2"], flags, survey=survey) + assert result.value.tolist() == [True, False, True, False, True, False] + assert result.known.all() + assert result.status.tolist() == [ + ("source_yes_" if i % 2 == 0 else "source_no_") + label + for i, label in enumerate(labels) + ] + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing_source_value"), + ("0", "unrecognized_source_value"), + ("9", "unrecognized_source_value"), + (" 1", "unrecognized_source_value"), + ("1.0", "unrecognized_source_value"), + ("-1", "unrecognized_source_value"), + ], +) +def test_unknown_codes_never_turn_into_false(token, status): + result = health.recode([token], ["0"], survey="asec") + assert pd.isna(result.value.iloc[0]) + assert not result.known.iloc[0] + assert result.status.iloc[0] == status + + +@pytest.mark.parametrize("token", [True, 1, None, np.int64(1)]) +def test_recode_does_not_coerce_nonliteral_inputs(token): + with pytest.raises(ValueError, match="LITERAL_CONTRACT"): + health.recode([token], ["0"], survey="acs") + + +def test_medicaid_specific_recode_and_closed_profile_roster(): + raw = invented_raw() + medicaid = health.recode_field(raw, "has_medicaid_health_coverage_at_interview") + assert medicaid.iloc[0, 0] == False # noqa: E712 + assert raw.loc[10, "NOW_MCAID"] == "1" + assert set(f.output for f in health.FIELDS) == set( + health.US_REPORTED_COVERAGE_PERSON_INPUTS + ) + with pytest.raises(ValueError, match="FIELD"): + health.recode_field(raw, "medicaid_eligible") + + +def test_acs_exact_values_and_all_semantic_gaps(): + raw = invented_raw() + for field in health.FIELDS: + result = health.recode_field(raw, field.output) + assert not pd.isna(result.loc[10, field.output]) + if field.acs is None: + assert pd.isna(result.loc[20, field.output]) + assert not result.loc[20, field.output + "__known"] + assert result.loc[20, field.output + "__source_status"].startswith( + "semantic_gap:" + ) + assert health.recode_field(raw, "has_esi").loc[20, "has_esi"] + ihs = health.recode_field(raw, health.FIELDS[-1].output) + assert pd.isna(ihs.iloc[1, 0]) + assert ihs.iloc[1, 2] == "missing_source_value" + literal = health.source_columns(raw) + assert pd.isna(literal.loc[10, "health_source_HINS1"]) + assert literal.loc[20, "health_source_HINS7"] == "" + assert literal.loc[20, "health_source_HINS4"] == "2" + assert literal.health_source_survey_year.tolist() == [2025, 2024] + + +def test_attachment_uses_original_identity_and_preserves_unknowns(): + receiving, origins = invented_receiving(), invented_origins() + before = receiving.person.copy(deep=True) + columns = health.recode_field(invented_raw(), health.FIELDS[-1].output) + result = health.attach_columns(origins, receiving, columns) + value = result["person", health.FIELDS[-1].output] + assert value.index.tolist() == [400, 100, 300, 200] + assert pd.isna(value.loc[400]) and pd.isna(value.loc[300]) + assert value.loc[100] == value.loc[200] == False # noqa: E712 + pd.testing.assert_frame_equal(receiving.person, before, check_exact=True) + + +@pytest.mark.parametrize( + "mutation,reason", + [ + ("duplicate_id", "CLONE_SOURCE_ROSTER"), + ("duplicate_clone", "CLONE_PAIR"), + ("foreign_origin", "CLONE_SOURCE_ROSTER"), + ("wrong_native", "CLONE_SOURCE_IDENTITY"), + ("wrong_channel", "CLONE_SOURCE_IDENTITY"), + ("collision", "OWNERSHIP_COLLISION"), + ], +) +def test_attachment_refuses_invalid_clone_identity_or_existing_output(mutation, reason): + receiving = invented_receiving() + p = health.provenance + name, value = { + "duplicate_id": ("person_id", 100), + "duplicate_clone": (p.support_clone_index_column("person"), 0), + "foreign_origin": (p.support_source_id_column("person"), 99), + "wrong_native": (p.spine_source_id_column("person"), 2), + "wrong_channel": (p.support_channel_column("person"), "asec"), + "collision": ("has_esi", False), + }[mutation] + receiving.person.loc[0, name] = value + with pytest.raises(ValueError, match=reason): + health.attach_columns( + invented_origins(), + receiving, + health.recode_field(invented_raw(), "has_esi"), + ) + + +def _csv(survey, *, second=None): + columns = source.ASEC_COLUMNS if survey == "asec" else source.ACS_COLUMNS + row = {c: "0" if c.startswith(("I_", "F")) else "2" for c in columns} + row.update( + {"PERIDNUM": "0123456789012345678901", "PH_SEQ": "7", "A_LINENO": "1"} + if survey == "asec" + else {"SERIALNO": "2024HU0000001", "SPORDER": "1"} + ) + rows = [row] + ([] if second is None else [{**row, **second}]) + body = ( + ",".join(columns) + + "\n" + + "\n".join(",".join(r[c] for c in columns) for r in rows) + + "\n" + ) + return body.encode(), row + + +@pytest.mark.parametrize("survey", ["asec", "acs"]) +def test_literal_scan_preserves_values_and_exhausts_tail(survey): + body, row = _csv(survey) + key = source._key(row, survey) + selected = {} + assert ( + source._scan( + io.BytesIO(body), survey=survey, wanted={key}, selected=selected, maximum=1 + ) + == 1 + ) + assert selected == {key: row} + with pytest.raises(ValueError, match="SOURCE_ROW_SHAPE"): + source._scan( + io.BytesIO(body + b"malformed\n"), + survey=survey, + wanted={key}, + selected={}, + maximum=2, + ) + body, _ = _csv(survey, second={}) + with pytest.raises(ValueError, match="DUPLICATE_SELECTED"): + source._scan( + io.BytesIO(body), survey=survey, wanted={key}, selected={}, maximum=2 + ) + + +def test_combine_refuses_missing_and_changed_source_keys(): + body, row = _csv("asec") + del body + key = source._key(row, "asec") + origins = invented_origins().iloc[:1] + origins = origins.assign(source_year=2024, survey_year=2025) + keys = {("asec", key): 10} + selected = {"asec": {key: row}, "acs": {}} + result = source._combine(origins, keys, selected) + assert result.loc[10, "NOW_CAID"] == "2" + with pytest.raises(ValueError, match="SELECTED_SOURCE_ROSTER"): + source._combine(origins, keys, {"asec": {}, "acs": {}}) + selected["asec"][key] = {**row, "PH_SEQ": "8"} + with pytest.raises(ValueError, match="SOURCE_COORDINATE_CHANGED"): + source._combine(origins, keys, selected) + + +def test_public_values_do_not_authorize_native_projection(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + source.qualify_current_survey_health(SimpleNamespace(payload=b"{}")) + + +def _frame(people): + from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + people = people.copy(deep=True) + tables = {"person": people} + for entity in US_SCHEMA.group_entities: + people[US_SCHEMA.membership_column(entity)] = people.person_id.to_numpy() + tables[entity] = pd.DataFrame( + {US_SCHEMA.entity_id_column(entity): np.sort(people.person_id.to_numpy())} + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.ones(len(people)), WeightKind.DESIGN)}, + ) + + +def _qualified(): + import hashlib + + from microcosm.build.us_runtime import graph_current_survey_health as graph + + raw = invented_raw() + origins = invented_origins() + frame = _frame(pd.DataFrame({"person_id": raw.index.to_numpy()})) + projection = raw.reset_index().to_json(orient="table", index=False).encode() + q = source.QualifiedSurveyHealthCoverage( + frame, + origins, + raw, + projection, + { + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "fixture": "invented; no source owner or data admission claimed", + }, + ) + graph.health_coverage_seal(q) + return q + + +def _graph_case(tmp_path): + from microcosm.build.us_runtime import graph_current_survey_health as graph + from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, + ) + + qualified = _qualified() + people = invented_receiving().person + people[health.provenance.support_channel_column("person")] = pd.array( + people[health.provenance.support_channel_column("person")], dtype="string" + ) + receiving = _frame(people) + attachment_type = ArtifactType("invented.amount_attachment", 1) + + class InventedSource(KernelBase): + ref = "invented.health_receiving@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return "a" * 64 # Test-only fixed producer for the tiny invented Frame. + + def run(self, context): + return KernelResult( + frame=receiving, artifacts={"attachment": b"invented amount input"} + ) + + create = Node( + "invented.amounts", + InventedSource.ref, + sources=(graph.SOURCE_NAME,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("person", c, graph._dtype(people[c])) + for c in people + if c != "person_id" + ), + artifact_outputs=(ArtifactOutput("attachment", attachment_type),), + ) + after = ArtifactInput("amount_attachment", create.id, "attachment", attachment_type) + nodes = graph.health_coverage_nodes( + qualified, receiving_version=create.id, after=after + ) + compiled = compile_graph( + Graph("us", (SourceRef(graph.SOURCE_NAME, "raw-bytes-v1"),), (create, *nodes)) + ) + expected_seal = graph.health_coverage_seal(qualified) + calls = [] + + def current(): + assert graph.health_coverage_seal(qualified) == expected_seal + calls.append(True) + + kernels = KernelRegistry() + kernels.register(InventedSource()) + for kernel in graph.health_coverage_kernels( + qualified, receiving_version=create.id, after=after, require_current=current + ): + kernels.register(kernel) + path = tmp_path / "invented-source.txt" + path.write_bytes(b"invented health fixture only") + store = ContentStore(tmp_path / "store") + args = {"sources": {graph.SOURCE_NAME: path}, "store": store, "kernels": kernels} + cold = run_graph(compiled, **args) + observed = {} + + def capture(_node_id, population): + observed[population.version] = population + + warm = run_graph(compiled, **args, resume="require", _population_observer=capture) + return SimpleNamespace( + graph=graph, + qualified=qualified, + receiving=receiving, + nodes=nodes, + compiled=compiled, + cold=cold, + warm=warm, + store=store, + calls=calls, + observed=observed, + ) + + +def test_actual_graph_cold_required_replay_preserves_amount_columns(tmp_path): + case = _graph_case(tmp_path) + assert len(case.compiled.order) == 13 + assert case.cold.key == case.warm.key + assert all(n.hit for n in case.warm.nodes.values()) + assert ( + len(case.calls) == 24 + ) # Before/after 12 cold fragment nodes; replay invokes none. + output = case.warm.population("invented.amounts") + pd.testing.assert_frame_equal( + output.person[case.receiving.person.columns], + case.receiving.person, + check_exact=True, + ) + assert output.person.has_esi.tolist() == [True, True, True, True] + assert output.person.has_medicaid_health_coverage_at_interview.isna().tolist() == [ + True, + False, + True, + False, + ] + + +def test_fragment_dependency_edges_and_explicit_receiving_version(): + from microcosm.build.us_runtime import graph_current_survey_health as graph + from microcosm.graph import ArtifactInput, ArtifactType + + after = ArtifactInput( + "amount_attachment", + "country.amounts", + "attachment", + ArtifactType("invented", 1), + ) + nodes = graph.health_coverage_nodes( + _qualified(), receiving_version="country.receiving", after=after + ) + assert nodes[-1].population == "country.receiving" + assert after in nodes[-1].artifact_inputs + assert len(nodes[-1].artifact_inputs) == 12 + assert sum(n.id.startswith(graph.RECODE_PREFIX) for n in nodes) == 9 + assert all(n.structural.value == "none" for n in nodes[1:]) + + +def test_qualified_source_and_raw_mutation_changes_full_seal(): + from microcosm.build.us_runtime import graph_current_survey_health as graph + + qualified = _qualified() + before = graph.health_coverage_seal(qualified) + qualified.origins.loc[10, "native_person_id"] = 99 + assert graph.health_coverage_seal(qualified) != before + qualified.raw.loc[10, "NOW_GRP"] = "2" + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + graph.health_coverage_seal(qualified) + + +def test_actual_graph_independent_complete_reconstruction(tmp_path): + from microcosm.graph import ArtifactValue, NumericScope + from microcosm.graph import population as populations + + case = _graph_case(tmp_path) + expected = { + "invented.amounts": populations.Population.from_frame( + case.receiving, "invented.amounts" + ) + } + for node_id in case.compiled.order: + if node_id == "invented.amounts": + continue + node = case.compiled.graph.node(node_id) + artifacts = {} + for edge in node.artifact_inputs: + producer = case.warm.node(edge.producer) + key = producer.opaque_artifacts[edge.artifact] + artifacts[edge.name] = ArtifactValue( + case.store.load_bytes(key), edge.type, key, producer.key, NumericScope() + ) + version = case.compiled.versions[node_id] + incoming = expected.get(version) + expected[version] = case.graph.expected_health_population( + node_id, incoming, qualified=case.qualified, node=node, artifacts=artifacts + ) + for version, population in expected.items(): + case.graph.physical.replay.same_replayed_population( + population, case.observed[version] + ) + + +def test_source_artifact_values_cannot_be_replaced(): + from microcosm.build.us_runtime import graph_current_survey_health as graph + from microcosm.graph import ArtifactInput, ArtifactType, ArtifactValue, NumericScope + + after = ArtifactInput( + "amount_attachment", "amounts", "attachment", ArtifactType("invented", 1) + ) + q = _qualified() + node = graph.health_coverage_nodes(q, receiving_version="receiving", after=after)[1] + artifact = ArtifactValue( + b"{}", graph.PROJECTION_TYPE, "0" * 64, "1" * 64, NumericScope() + ) + with pytest.raises(ValueError, match="ARTIFACT_PAYLOAD"): + graph._check_artifacts(node, {"health_projection": artifact}, q) From 091261c0e979d0ee191ddbaf08a069b2554d7165 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:05:51 -0400 Subject: [PATCH 150/255] Add checked survey amount and health enrichment host --- ...-survey-current-amount-enrichment.added.md | 1 + docs/us-current-survey-amount-successor.md | 196 ++++ .../current_asec_unemployment_source.py | 347 ++++++ .../us_runtime/current_survey_amounts.py | 459 ++++++++ .../us_runtime/graph_us_survey_enrichment.py | 1043 +++++++++++++++++ .../tests/test_us_current_survey_amounts.py | 316 +++++ .../test_us_graph_us_survey_enrichment.py | 373 ++++++ 7 files changed, 2735 insertions(+) create mode 100644 changelog.d/us-survey-current-amount-enrichment.added.md create mode 100644 docs/us-current-survey-amount-successor.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_amounts.py create mode 100644 packages/microcosm-build/tests/test_us_graph_us_survey_enrichment.py diff --git a/changelog.d/us-survey-current-amount-enrichment.added.md b/changelog.d/us-survey-current-amount-enrichment.added.md new file mode 100644 index 000000000..dab37b769 --- /dev/null +++ b/changelog.d/us-survey-current-amount-enrichment.added.md @@ -0,0 +1 @@ +Add a checked US survey enrichment host that preserves the PUF population while attaching source-qualified unemployment, medical costs and current health coverage. Keep reporting unknowns, original observations and source-to-clone provenance explicit. diff --git a/docs/us-current-survey-amount-successor.md b/docs/us-current-survey-amount-successor.md new file mode 100644 index 000000000..250bab1ad --- /dev/null +++ b/docs/us-current-survey-amount-successor.md @@ -0,0 +1,196 @@ +# Current survey amount successor + +Implementation plan and source inventory, 12 September 2026. This is a local +successor to the retained survey PUF55 run, not an amendment to the frozen d35 +native pilot. The branch starts at reviewed integration 99543c3. + +## Immediate scope + +| Native ASEC field | Canonical person input | Reporting evidence | +| --- | --- | --- | +| UC_VAL | unemployment_compensation | UC_YN receipt code; amount question universe UC_YN = 1 | +| PHIP_VAL | health_insurance_premiums_without_medicare_part_b | All persons | +| PMED_VAL | other_medical_expenses | All persons | +| POTC_VAL | over_the_counter_health_expenses | All persons | + +These four names are absent from both PUF tax-detail ownership rosters. They +already have qualified current-money domains and direct amount mappings in +`cps_carried_current.py`. The complete legacy carry helper is not reusable: +it also writes existing PUF leaves and a legacy Social Security decomposition. +The modern `survey_social_security` source owner remains the authority for +Social Security; no age-62 zero fallback or prior wages is introduced here. + +The current ASEC source is the authenticated original 2025 person member, +income year and price basis 2024. ACS observations use the already qualified +2024 source predictors, whose earnings refer to the rolling prior 12 months. +That temporal harmonization is a documented model choice, not equivalent +observation windows. No new donor dataset is needed. + +## Amount and reporting status are different + +The [2025 CPS ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 50, defines UC_YN codes 0 (NIU), 1 (yes), and 2 (no), with the receipt +question asked of persons aged 15 or older. UC_VAL is asked of recipients; +its zero code conflates none with NIU. The maintained domain metadata pins +this dictionary to SHA256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The three health amounts have all-persons universes (PDF page 83). + +A small bounded UC_YN projection reads the same exact authenticated original +member. Native person and household/line coordinates must join the original +current-money owner exactly; carried columns and detached JSON do not confer +source authority. The projection retains the original amount, amount validity, +source classification, zero_origin, literal receipt code and reporting status. +Today's upstream preparation requires complete qualified current money before +issuing its handle. Literal missing-amount controls test the join's unknown +semantics; they do not claim that a missing amount passes this unchanged parent +prerequisite. Reporting unknownness can still occur with numerically valid amounts. + +UC attachment and donor policy: + +- A valid positive amount with an in-universe yes code is known receipt. +- A zero amount with an in-universe no code is known nonreceipt. +- Yes with zero is an ambiguous amount, not known nonreceipt. It remains in + the source projection but is excluded from fit and canonical attachment. +- NIU, blank/unrecognized codes, missing amounts, and contradictory amount/ + receipt pairs remain unresolved and are excluded from fit and attachment. +- Out-of-universe age does not infer a zero. ACS persons outside the receipt + question universe also retain unknown UC rather than receiving an adult draw. + +The health fields retain qualified valid all-persons amounts, including genuine +zero-dollar codes. No noanswer cell is converted to zero. Allocation and source +status remain evidence; known source values are not claimed to be unallocated. +PHIP_VAL is deliberately retained as the declared premium source, rather than +silently substituting PHIP_VAL2, whose treatment of inconsistent reported zero +premiums differs. The [Census health insurance user note](https://www.census.gov/programs-surveys/cps/technical-documentation/user-notes/health-insurance-user-notes/User-Notes-Selected-CPS-ASEC.html) +describes that distinction. Person-level conditional fitting preserves modeled +cross-component dependence; it does not certify household or insurance-unit +correlation, which still needs empirical validation. + +## One family, explicit graph operations + +The family has one parameterized source projection and attachment mechanism. +Each selected field declares its source transformation, donor eligibility, +recipient matrix, QRF fit and draw as graph nodes/artifacts. UC has its own +reporting mask. The health group uses an ordered PHIP → PMED → POTC conditional +chain, retaining observed health-component dependence through prior-target +conditioning instead of fitting the three amounts independently. This avoids +filling unknown UC to make a joint matrix finite. Fits use the existing regime-gated QRF implementation and +current source predictor qualification; donor weights remain original design +weights before allocation or cloning. No PUF value becomes an ASEC donor. + +Draws are keyed to the original ACS source person, then deliberately shared by +its original and PUF support clones. This preserves source observations and +paired support interpretation; clone labels are checked during attachment. +Source amount values and reporting status are copied to both ASEC clones. +Every existing parent column, owner, axis, membership, atomic geography, +weight, design anchor and mass ledger must remain identical. + +The parent is the exact `SurveyPuf55Run` checked by +`check_survey_puf55_run`. The original handle is retained and checked before +consumption and after source/store/export I/O. A checked JSON payload or copied +dataclass cannot issue authority. The extension copies kernel/source-codec +registries privately, preserves all prefix declarations/keys/artifacts, and +verifies actual observed populations on cold and required replay. Failed checks +revoke the original handle. The checked-output branch supplies this additive +API; the frozen native packet is left untouched. + +## Verification + +First run cheap invented controls: all UC code/amount/universe combinations, +health zero and unknown behavior, literal bounds and source-coordinate joins, +clone pairing, missing/duplicate source IDs, invalid source/clone channels, +ownership collisions and raw/status/zero-origin mutation. + +Then an actual invented authenticated-source fixture runs the family through +real graph cold and required replay, QRF train/apply, coverage, complete Frame +store/export/readback, parent retention and unchanged prefix seals. Rejection +controls mutate source projection, parent handle/population, declared graph, +artifact bytes, fit/draw lineage and output attachment independently. Acceptance +requires closed receipts and independent review. No native run, engine outcome, +held-out quality, calibration or release certification is claimed by these tests. + +## Remaining input inventory + +The accepted invented PUF baseline has 161 required names, of which 100 are +absent. Its 61 present names are not necessarily complete: some PUF leaves are +populated only on PUF clones. Counts are inventory for that fixture, not a native +coverage measurement. The next stage reduces absent names while reporting +unresolved cells separately. + +| Input family | Existing source producer or route | Integration requirement | +| --- | --- | --- | +| UC and three health amounts | authenticated current-money owner; direct `cps_carried_current` maps | This stage | +| Social Security components | `current_social_security_source`, `survey_social_security` | Preserve source reporting basis and unknown components; never legacy age split | +| Health coverage | Authenticated `current_survey_health_source` and explicit `graph_current_survey_health` fragment | This stage; seven narrow ACS concepts remain unknown where the survey cannot distinguish them | +| Disability, student status, children, veteran status | `eligibility_inputs` | Qualify literals and reporting universes before reuse | +| Parent relationships | `eligibility_inputs._own_children_in_household` resolves PEPAR1/PEPAR2 then drops links | Issue 884: retain parent_1_id/parent_2_id with household/line and clone joins; unresolved and absent differ; consumer issue 9404 is separate | +| Marital roles | `relationship_inputs` | Current source observations and clone preservation | +| Hours, work history, occupation, wage/tips | `hours_worked`, `org_wages`, `weeks_unemployed`, `sipp_tips` | Source/QRF qualification and explicit ownership; prior wages excluded | +| Child support, disability income, workers compensation | respective maintained runtime modules | Current source amount/universe qualification, then existing fit contracts | +| Housing, energy, care, education | `housing`, `energy_subsidy`, `adultcare`, education producers | Household/SPM source and clone-keyed joins; reconcile PUF tuition ownership | +| Financial wealth, vehicles, loans | SCF/SIPP producer modules | Actual donor authentication, existing fit dependencies and held-out quality | +| Retirement distributions/contributions | maintained distribution/contribution modules | Reconcile already PUF-owned taxable IRA/pension and self-employed pension leaves | +| Pregnancy, SSI disability, voluntary filing | maintained producer modules | Resolve source authority and distinct behavioral assumptions | +| Take-up | `take_up_contract` | Unsourced rates remain unresolved; no convenience constant fills | +| Mortgage details, marketplace benchmark ratio, remaining retirement details | Not yet mapped in this bounded inventory | Identify actual producer/owner or make an explicit source/model decision | + +The inventory used tracked files and `rg --no-ignore` within explicit source +directories because ordinary searches can skip the tracked build package. +Existing function names establish implementation leads; they do not prove that +those producers are integrated into the current graph or current native file. + + +## Fixed country composition + +`run_us_survey_enrichment` retains the original checked `SurveyPuf55Run` and +assembles the amount family and independently source-qualified health fragment. +Both attach to `survey_puf55.receiving`; the health CREATE and final attachment +explicitly consume `survey_amounts.attach/attachment`. Health adds nine coverage +inputs and their literal/status/knownness evidence after all four monetary leaves. +The host preserves the complete parent Frame, owners, design anchors and mass ledger, +compares every unchanged prefix version against the original parent manifest, and +requalifies both original-source projections before issuing its output. There is +one private issued-handle registry and one public checked country output, not a +family-level authority or executor. The health fragment's descriptive values do +not grant access to native sources. + +The combined invented fixture includes the exact source ACS HINS/flags and ASEC +NOW/flags before authenticating source owners. It runs the original financial and +PUF stages, both enrichment fragments, a required replay, input-coverage diagnosis +and complete Frame store/readback. No native build, model engine execution, +calibration, or release approval is implied by these checks. + + +## Invented acceptance, 12 September 2026 + +The final production source passed the 271-node combined graph: 245 validated +prefix hits followed by 26 newly executed enrichment nodes, then a required +replay with all 271 hits. Complete parent columns, owners, design anchors, +geography and the mass ledger were preserved. The full resulting Frame was +stored and read back with exact physical comparison. The fixture has 18 people; +required-name coverage improved from 61 to 74 of the 161-input profile, leaving +87 missing names. Presence does not imply complete values or statistical quality. + +All 18 people have the three health-cost values and an ESI observation. UC has +10 known and eight unresolved values. The narrow Medicaid-at-interview input has +eight known and ten unresolved values, preserving the ACS semantic gap. These +are counts from invented records, not estimates or native-data diagnostics. + +Acceptance combines five passing ordered tests from the third full attempt and +one corrected, focused final parent-revocation test. The third attempt retained +its failed result because that last test named a nonexistent generic income +column; it stopped before mutating anything. The final test uses the actual +`employment_income_before_lsr` input and verifies permanent parent and child +revocation after mutation, including refusal after restoring the original value. +It rebuilds a real parent and one child, without repeating the already-passed +required replay or readback. The focused attempt passed with all 857 Python +source pins stable, 624.872 seconds wall time and 885,669,888 bytes peak RSS. + +The separate 32-case cheap suite covers UC reporting classifications, literal +missingness, source/clone identity, ownership collisions and source-origin +serialization when `person_id` names both an index and a column. The serialization +binds the original int64 axis and its name separately from ordered column values. +Independent review covers the frozen UC source, health fragment, combined host +and this serialization correction. Preserved failed attempts remain part of the +local acceptance record; no native execution or release certificate is implied. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py new file mode 100644 index 000000000..7eb0d827c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py @@ -0,0 +1,347 @@ +"""Receipt-universe projection from the retained current ASEC original. + +UC_VAL's numeric zero is not itself evidence of nonreceipt. This module keeps +the source amount separate from a canonical amount whose reporting basis is +known. Detached projection values grant no source authority: consuming hosts +requalify the retained preparation and compare these values before/after I/O. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import asec_current_money as money +from . import source_csv_builtin +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-unemployment-source.v1" +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "UC_VAL", "UC_YN") +DICTIONARY = { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 50, + "printed_page": "6C-29", + "receipt_field": "UC_YN", + "receipt_codes": {"0": "niu", "1": "yes", "2": "no"}, + "receipt_universe": "All Persons aged 15+", + "amount_field": "UC_VAL", + "amount_universe": "UC_YN = 1", + "zero_code": "none or niu", +} + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_UNEMPLOYMENT_" + reason) + + +def reporting_basis(amount, age, receipt_tokens): + """Classify invented or observed arrays without issuing source authority. + + Invalid/missing receipt literals are unresolved, rather than recoded to no. + Contradictions remain inspectable source observations, excluded from the + modeled donor and canonical amount. Age only gates the printed universe; + it never creates an income zero. + """ + amounts = np.asarray(amount) + ages = np.asarray(age) + tokens = tuple(receipt_tokens) + require( + amounts.dtype == np.dtype("float64") + and ages.dtype == np.dtype("float64") + and amounts.ndim == ages.ndim == 1 + and len(amounts) == len(ages) == len(tokens), + "ARRAY_CONTRACT", + ) + require( + np.isfinite(ages).all() + and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), + "AGE_DOMAIN", + ) + require( + all(type(t) is str and len(t) <= 64 for t in tokens) + and not np.isinf(amounts).any() + and ( + (amounts[np.isfinite(amounts)] >= 0) + & (amounts[np.isfinite(amounts)] <= 99999) + ).all(), + "AMOUNT_OR_TOKEN_DOMAIN", + ) + result = pd.DataFrame({"source_amount": amounts.copy(), "receipt_literal": tokens}) + result["source_reporting_universe"] = ages >= 15 + result["receipt_code_known"] = [t in ("0", "1", "2") for t in tokens] + result["receipt_code"] = pd.array( + [int(t) if t in ("0", "1", "2") else pd.NA for t in tokens], dtype="Int8" + ) + canonical = np.full(len(amounts), np.nan, dtype=np.float64) + labels = [] + for i, (value, years, token) in enumerate(zip(amounts, ages, tokens, strict=True)): + if years < 15: + label = ( + "outside_reporting_universe" + if token == "0" and (np.isnan(value) or value == 0) + else "contradictory_outside_reporting_universe" + ) + elif token == "": + label = "missing_receipt_literal" + elif token not in ("0", "1", "2"): + label = "unrecognized_receipt_literal" + elif np.isnan(value): + label = "missing_amount" + elif token == "0": + label = "niu" if value == 0 else "contradictory_niu_positive" + elif token == "1": + label = "known_receipt" if value > 0 else "ambiguous_recipient_zero" + if value > 0: + canonical[i] = value + else: + label = "known_nonreceipt" if value == 0 else "contradictory_no_positive" + if value == 0: + canonical[i] = value + labels.append(label) + result["reporting_status"] = labels + result["canonical_amount"] = canonical + result["canonical_amount_known"] = np.isfinite(canonical) + return result + + +def _read_capture(path, *, rows): + """Bounded literal reader; its path and row count do not establish authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, keys, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + positions = [header.index(c) for c in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + record = dict(zip(READ_COLUMNS, (row[i] for i in positions), strict=True)) + key = record["PERIDNUM"] + require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") + for name, width in (("PH_SEQ", 5), ("A_LINENO", 2), ("A_AGE", 2)): + require( + re.fullmatch(r"[0-9]{1," + str(width) + "}", record[name], re.ASCII) + is not None, + "COORDINATE:" + name, + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and key not in keys, + "DUPLICATE_OR_INVALID_COORDINATE", + ) + require( + record["UC_VAL"] == "" + or re.fullmatch(r"[0-9]{1,5}", record["UC_VAL"], re.ASCII) is not None, + "AMOUNT_TOKEN", + ) + require(len(record["UC_YN"]) <= 64, "RECEIPT_TOKEN_BOUND") + records.append(record) + keys.add(key) + coordinates.add(pair) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records).set_index("PERIDNUM", drop=False) + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _amount_observations(field, positions, literals): + """Join literal validity; missing backing storage never becomes a survey zero. + + Today's retained parent requires complete ready money before issuing a + preparation. This helper keeps that prerequisite separate from the literal + join, so a missing field is represented correctly if that scope later grows. + """ + require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") + positions = np.asarray(positions) + require( + positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" + ) + tokens = tuple(literals) + require( + len(tokens) == len(positions) + and all( + type(t) is str and (t == "" or re.fullmatch(r"[0-9]{1,5}", t, re.ASCII)) + for t in tokens + ), + "AMOUNT_TOKEN", + ) + valid = field.validity[positions] == 1 + require( + np.array_equal(valid, np.array([t != "" for t in tokens])), + "CURRENT_AMOUNT_SOURCE_VALIDITY", + ) + amounts = field.amounts[positions].copy() + literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + require( + np.array_equal(amounts[valid], literal_values[valid]), + "CURRENT_AMOUNT_SOURCE_IDENTITY", + ) + amounts[~valid] = np.nan + return amounts + + +@dataclass(frozen=True) +class CurrentAsecUnemploymentValues: + person: pd.DataFrame + evidence: dict + + +def qualify_current_asec_unemployment(preparation): + """Capture the exact current source member retained by the original owner.""" + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + pins = [p for p in coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-uc-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity + and _file_sha(captured) == digest, + "CAPTURE_CHANGED", + ) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + parent_people = parent.frame.person.iloc[positions] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent_people[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("UC_VAL") + amounts = _amount_observations(field, positions, ordered.UC_VAL) + basis = reporting_basis( + amounts, ordered.A_AGE.to_numpy(dtype="float64"), ordered.UC_YN + ) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + basis["amount_status"] = field.statuses[positions] + basis["amount_validity"] = field.validity[positions] + basis["zero_origin"] = field.zero_origin[positions] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + native_ids = selected[spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "money_header_sha256": hashlib.sha256(ready.header).hexdigest(), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "selected_rows": len(out), + "projection_sha256": hashlib.sha256( + out.to_json(orient="table").encode() + ).hexdigest(), + "policy": "known_yes_positive_or_no_zero_only; ambiguous_NIU_missing_contradictions_retained_unknown", + "unallocated_observation_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require( + hashlib.sha256(out.to_json(orient="table").encode()).hexdigest() + == evidence["projection_sha256"], + "FINAL_PROJECTION_CHANGED", + ) + return CurrentAsecUnemploymentValues(out, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py new file mode 100644 index 000000000..78dd8e4fa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py @@ -0,0 +1,459 @@ +"""Qualified current-money targets and source-keyed clone attachment values. + +The closed family has one UC route and one conditional health-cost route. +Values are derived from live retained owners; this module issues no authority. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import current_asec_unemployment_source as unemployment +from . import current_survey_predictors as predictors +from . import graph_full_puf_enrichment as physical +from . import graph_survey_puf55 as parent_host +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.current-survey-amount-family.v1" +SEED = 579 + + +@dataclass(frozen=True) +class AmountGroup: + key: str + fields: tuple[tuple[str, str], ...] + + @property + def targets(self): + return tuple("survey_amount_target_" + raw for raw, _ in self.fields) + + +GROUPS = ( + AmountGroup("unemployment", (("UC_VAL", "unemployment_compensation"),)), + AmountGroup( + "health_costs", + ( + ("PHIP_VAL", "health_insurance_premiums_without_medicare_part_b"), + ("PMED_VAL", "other_medical_expenses"), + ("POTC_VAL", "over_the_counter_health_expenses"), + ), + ), +) +UC_REPORT_COLUMNS = ( + "source_amount", + "receipt_literal", + "source_reporting_universe", + "receipt_code_known", + "receipt_code", + "reporting_status", + "canonical_amount_known", + "amount_status", + "amount_validity", + "zero_origin", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_AMOUNTS_" + reason) + + +def selected_groups(names): + require( + type(names) is tuple + and bool(names) + and len(set(names)) == len(names) + and all(type(n) is str for n in names), + "GROUP_NAMES", + ) + result = tuple(g for g in GROUPS if g.key in names) + require(tuple(g.key for g in result) == names, "GROUP_ROSTER_OR_ORDER") + return result + + +@dataclass(frozen=True) +class GroupValues: + spec: AmountGroup + donor_frame: object + donor_columns: pd.DataFrame + matrix: bytes + keep: np.ndarray + + +@dataclass(frozen=True) +class QualifiedSurveyAmounts: + projection: bytes + evidence: dict + source_frame: object + origins: pd.DataFrame + native: pd.DataFrame + reports: pd.DataFrame + features: tuple[str, ...] + groups: tuple[GroupValues, ...] + + +def seal(value): + require(type(value) is QualifiedSurveyAmounts, "QUALIFIED_TYPE") + return ( + value.projection, + codec.encode_json(value.evidence), + value.features, + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + value.source_frame, "survey_amounts.source" + ) + ), + *( + physical._table_stamp(t) + for t in (value.origins, value.native, value.reports) + ), + tuple( + ( + g.spec, + g.matrix, + g.keep.dtype.str, + g.keep.tobytes(), + physical._table_stamp(g.donor_columns), + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + g.donor_frame, "survey_amounts.donor" + ) + ), + ) + for g in value.groups + ), + ) + + +def origin_digest(origins): + """Bind the original source axis and columns even when both name person_id.""" + axis = pd.Index(predictors._ids(origins.index.to_series(index=origins.index))) + columns = ( + origins.reset_index(drop=True).to_json(orient="table", index=False).encode() + ) + return codec.sha( + codec.encode_json( + { + "index_name": origins.index.name, + "index_sha256": codec.sha(axis.to_numpy(dtype="= 15 + require(keep.any(), "NO_QUALIFIED_DONORS:" + spec.key) + require(recipient.any(), "NO_QUALIFIED_RECIPIENTS:" + spec.key) + donor_frame = base.source_frame.select(keep) + require( + donor_frame.resolve_weights("person").kind is WeightKind.DESIGN, + "DONOR_DESIGN_WEIGHTS", + ) + donor_columns = features.loc[ids[keep]].copy() + for target, output in zip(spec.targets, outputs, strict=True): + donor_columns[target] = native.loc[ids[keep], output].to_numpy() + require( + np.array_equal( + donor_frame.person.person_id.to_numpy(), ids[keep].to_numpy() + ), + "DONOR_AXIS", + ) + recipient_matrix = model_input.encode_recipient_matrix( + features.loc[ids[recipient]], + entity="person", + entity_ids=ids[recipient].to_numpy(dtype="= 0).all(), + "DRAW_AXIS_OR_VALUES", + ) + require( + qualified.origins.loc[draw.index, "source"].eq("acs").all(), "DRAW_SOURCE" + ) + for target, (_, output) in zip( + group.spec.targets, group.spec.fields, strict=True + ): + completed.loc[draw.index, output] = draw[target].to_numpy() + all_columns = pd.concat([completed, qualified.reports], axis=1) + require(not set(all_columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + index = pd.Index(ids, name="person_id") + return { + ("person", name): pd.Series( + all_columns[name].reindex(original).array.copy(), index=index, name=name + ) + for name in all_columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py new file mode 100644 index 000000000..c9c3ae590 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py @@ -0,0 +1,1043 @@ +"""Fixed US post-PUF host for source-qualified survey enrichment fragments. + +The owning boundary retains a real checked PUF run and authenticated source +projection. Models use separate original-design donor branches; attachment +adds the declared amount and coverage families to the existing receiving frame. +""" + +from __future__ import annotations + +import sys +import weakref +from dataclasses import dataclass, replace +from types import FunctionType, SimpleNamespace + +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + StructuralDelta, + codecs, + compile_graph, + run_graph, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import current_survey_amounts as values +from . import graph_current_survey_health as health_graph +from . import graph_current_survey_predictors as predictor_graph + +parent = values.parent_host +physical = values.physical +require = values.require +PROJECTION_NODE = "survey_amounts.source_projection" +ATTACH_NODE = "survey_amounts.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_amount_projection", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_amount_attachment", 1) +_ISSUED = {} + + +def _live(): + result = [] + for module in ( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + health_graph.health, + health_graph.source, + ): + for name, item in vars(module).items(): + if type(item) is FunctionType: + result.append( + ( + module.__name__, + name, + values.predictors.source._function_seal(item), + ) + ) + elif isinstance(item, type) and item.__module__ == module.__name__: + result.append((module.__name__, name, item)) + for member, function in vars(item).items(): + if isinstance(function, (classmethod, staticmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + result.append( + ( + module.__name__, + name, + member, + values.predictors.source._function_seal(function), + ) + ) + result.append( + ( + "amount_configuration", + values.SEED, + tuple((g.key, g.fields, g.targets) for g in values.GROUPS), + values.UC_REPORT_COLUMNS, + values.unemployment.PROTOCOL, + values.unemployment.READ_COLUMNS, + codec.encode_json(values.unemployment.DICTIONARY), + ) + ) + result.append( + ( + "health_configuration", + health_graph.health.PROTOCOL, + tuple( + (f.output, f.asec, f.acs, f.acs_gap) for f in health_graph.health.FIELDS + ), + health_graph.health.RAW_COLUMNS, + health_graph.health.SOURCE_PREFIX, + health_graph.source.ASEC_COLUMNS, + health_graph.source.ACS_COLUMNS, + ) + ) + return tuple(result) + + +def _amount_edge(): + return ArtifactInput( + "amount_attachment", ATTACH_NODE, "attachment", ATTACHMENT_TYPE + ) + + +def _upstream_edge(): + return ArtifactInput( + "puf_finalization", + parent.attach.ATTACH_NODE, + "finalization", + parent.attach.FINALIZATION_TYPE, + ) + + +def _projection_edge(): + return ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE) + + +def _ids(group): + base = "survey_amounts." + group.spec.key + return base + ".donor", base + ".columns", base + ".fit", base + ".apply" + + +def amount_nodes(qualified, receiving, *, parent_digest, n_estimators): + require( + type(n_estimators) is int and n_estimators > 0 and codec._hash(parent_digest), + "NODE_PARAMETERS", + ) + params = { + "projection_sha256": codec.sha(qualified.projection), + "parent_sha256": parent_digest, + "n_estimators": n_estimators, + "groups": tuple(g.spec.key for g in qualified.groups), + "protocol": values.PROTOCOL, + } + nodes = [ + Node( + PROJECTION_NODE, + CurrentSurveyAmountProjectionKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + artifact_inputs=(_upstream_edge(),), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + *( + ArtifactOutput( + g.spec.key + "_matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + for g in qualified.groups + ), + ), + description="Qualify original current survey amount and reporting-universe evidence; preserve source unknowns.", + ) + ] + attach_edges = [_projection_edge(), _upstream_edge()] + for group in qualified.groups: + donor, columns, fit_prefix, apply_prefix = _ids(group) + group_params = {**params, "group": group.spec.key} + matrix_edge = ArtifactInput( + group.spec.key + "_matrix", + PROJECTION_NODE, + group.spec.key + "_matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ) + nodes.extend( + ( + Node( + donor, + CurrentSurveyAmountDonorKernel.ref, + base=values.predictors.host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="free", + inputs=predictor_graph._inputs(qualified.source_frame), + params=group_params, + artifact_inputs=(_projection_edge(),), + description="Select source ASEC persons with jointly known targets; retain original design weights before allocation and clones.", + ), + Node( + columns, + CurrentSurveyAmountColumnsKernel.ref, + population=donor, + inputs=predictor_graph._inputs(group.donor_frame), + params=group_params, + outputs=tuple( + Owned("person", c, "float64") + for c in (*qualified.features, *group.spec.targets) + ), + artifact_inputs=(_projection_edge(),), + description="Map qualified current amounts and source predictors without numeric or reporting-unknown fills.", + ), + ) + ) + fits = legacy_qrf_train_nodes( + fit_prefix, + population=donor, + entity="person", + predictors=qualified.features, + targets=group.spec.targets, + seed=values.SEED, + n_estimators=n_estimators, + zero_atol=0, + phase="current_survey_amounts." + group.spec.key, + ) + applies = legacy_qrf_apply_matrix_nodes( + apply_prefix, + population=parent.attach.FILTER_NODE, + fit_nodes=fits, + matrix_producer=PROJECTION_NODE, + matrix_artifact=group.spec.key + "_matrix", + seed=values.SEED, + phase="current_survey_amounts." + group.spec.key, + ) + nodes.extend((*fits, *applies)) + attach_edges.append(matrix_edge) + for i, node in enumerate(applies): + attach_edges.extend( + ( + ArtifactInput( + f"{group.spec.key}_raw_{i}", + node.id, + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"{group.spec.key}_state_{i}", + node.id, + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + columns = pd.concat([qualified.native, qualified.reports], axis=1) + nodes.append( + Node( + ATTACH_NODE, + CurrentSurveyAmountAttachKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + outputs=tuple(Owned("person", c, str(columns[c].dtype)) for c in columns), + artifact_inputs=tuple(attach_edges), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Join source observations and ACS conditional draws to both support clones; retain all PUF and source columns, geography, weights and ledger.", + ) + ) + return tuple(nodes) + + +class Boundary: + """Internal retained-value seam; a detached projection cannot construct it.""" + + def __init__(self, run, *, groups, n_estimators): + self.run = run + self.parent_view = parent.check_survey_puf55_run(run) + self.parent_entry = parent._run_entry(run) + self.qualified = values.qualify_current_survey_amounts(run, groups=groups) + self.qualified_stamp = values.seal(self.qualified) + self.parent_stamp = physical._population_stamp(run.population) + self.preparation = run.financial_run.prefix.preparation + self.health = health_graph.qualify_health_coverage(self.preparation) + self.health_stamp = health_graph.health_coverage_seal(self.health) + self.n_estimators = n_estimators + self.amount_nodes = amount_nodes( + self.qualified, + run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=n_estimators, + ) + self.health_nodes = health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + self.nodes = (*self.amount_nodes, *self.health_nodes) + self.declaration = tuple(self.nodes) + self.live = _live() + self.compiled = self.kernels = self.store = None + self.paths = self.source_keys = self.keys = self.implementations = None + self.parent_objects = ( + run.population, + run.compiled, + run.manifest, + run.store, + run.kernels, + run.sources, + ) + + def pure(self): + parent._pure_run(self.run, self.parent_entry) + require( + parent._run_entry(self.run) is self.parent_entry + and all( + a is b + for a, b in zip( + self.parent_objects, + ( + self.run.population, + self.run.compiled, + self.run.manifest, + self.run.store, + self.run.kernels, + self.run.sources, + ), + strict=True, + ) + ) + and physical._population_stamp(self.run.population) == self.parent_stamp + and values.seal(self.qualified) == self.qualified_stamp + and self.run.financial_run.prefix.preparation is self.preparation + and health_graph.health_coverage_seal(self.health) == self.health_stamp + and self.nodes == self.declaration + and _live() == self.live, + "BOUNDARY_CHANGED", + ) + require( + self.amount_nodes + == amount_nodes( + self.qualified, + self.run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=self.n_estimators, + ) + and self.health_nodes + == health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + and self.nodes == (*self.amount_nodes, *self.health_nodes), + "BOUNDARY_DECLARATIONS", + ) + if self.compiled is not None: + require( + all( + a is b + for a, b in zip( + self.bound_objects, + (self.compiled, self.store, self.kernels, self.store.codecs), + strict=True, + ) + ) + and tuple(self.kernels.as_mapping().items()) == self.kernel_items + and tuple(self.store.codecs.as_mapping().items()) == self.codec_items + and tuple(self.store.codecs.as_bytes_mapping().items()) + == self.bytes_codec_items, + "BOUND_REGISTRY_CHANGED", + ) + require( + self.compiled == compile_graph(self.compiled.graph) + and graph_to_json(self.compiled.graph) == self.graph_json, + "COMPILED_CHANGED", + ) + + def borrow(self): + require( + parent.check_survey_puf55_run(self.run).payload == self.parent_view.payload, + "PARENT_IDENTITY", + ) + if self.compiled is not None: + paths, source_keys = _source_paths_and_keys( + self.compiled, dict(self.paths), self.store + ) + keys, implementations = _all_node_keys( + self.compiled, self.kernels, source_keys + ) + require( + tuple(sorted(paths.items())) == self.paths + and tuple(sorted(source_keys.items())) == self.source_keys + and tuple(sorted(keys.items())) == self.keys + and tuple(sorted(implementations.items())) == self.implementations, + "SOURCE_OR_IMPLEMENTATION_CHANGED", + ) + self.pure() + + def context(self, context): + # Entry/final host fences check the complete PUF owner. These ordinary + # fragment calls consume retained, source-qualified values and check + # their pure seals; they do not repeatedly reread the full PUF pipeline. + self.pure() + require( + not context.sources + and context.node in self.nodes + and set(context.artifacts) + == {e.name for e in context.node.artifact_inputs}, + "CONTEXT_DECLARATION", + ) + for edge in context.node.artifact_inputs: + value = values.predictors.host.shared.artifact( + context, edge.name, edge.type + ) + require( + value.producer_key == dict(self.keys)[edge.producer], + "ARTIFACT_PRODUCER_KEY", + ) + if edge.name == "projection": + require(value.payload == self.qualified.projection, "PROJECTION_BYTES") + if edge == _upstream_edge(): + record = self.run.manifest.node(edge.producer) + payload = self.run.store.load_bytes( + record.opaque_artifacts[edge.artifact] + ) + require( + value.producer_key == record.key + and value.key == record.opaque_artifacts[edge.artifact] + and value.payload == payload, + "PARENT_ARTIFACT", + ) + self.pure() + return self.qualified + + def requalify(self): + """Reconstruct owned source transformations at the host's final I/O fence.""" + fresh = values.qualify_current_survey_amounts( + self.run, groups=tuple(g.spec.key for g in self.qualified.groups) + ) + require( + values.seal(fresh) == self.qualified_stamp, "SOURCE_REQUALIFICATION_CHANGED" + ) + fresh_health = health_graph.qualify_health_coverage(self.preparation) + require( + health_graph.health_coverage_seal(fresh_health) == self.health_stamp, + "HEALTH_SOURCE_REQUALIFICATION_CHANGED", + ) + self.pure() + + +class _Kernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=predictor_graph._Kernel.capabilities.dependencies, + ) + + def __init__(self, boundary): + self.boundary = boundary + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + parent, + physical, + predictor_graph, + values.predictors, + values.unemployment.coverage, + values.unemployment.source_csv_builtin, + population_ops, + model_input, + qrf, + qrf_target, + dependencies=self.capabilities.dependencies, + ) + + +class CurrentSurveyAmountProjectionKernel(_Kernel): + ref = "us.survey_amounts.source_projection@1" + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + return KernelResult( + artifacts={ + "projection": qualified.projection, + **{g.spec.key + "_matrix": g.matrix for g in qualified.groups}, + }, + receipt=qualified.evidence, + ) + + +class CurrentSurveyAmountDonorKernel(_Kernel): + ref = "us.survey_amounts.source_donor@1" + capabilities = replace(_Kernel.capabilities, structural=StructuralDelta.FILTER) + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame(context, qualified.source_frame) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + return KernelResult( + keep=pd.Series( + group.keep.copy(), + index=pd.Index( + qualified.source_frame.person.person_id.to_numpy(), name="person_id" + ), + ), + receipt={ + "group": group.spec.key, + "donor_persons": int(group.keep.sum()), + "selection": "joint_known_current_ASEC_person_targets", + "weight_kind": "design", + }, + ) + + +class CurrentSurveyAmountColumnsKernel(_Kernel): + ref = "us.survey_amounts.source_columns@1" + + def run(self, context): + qualified = self.boundary.context(context) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + values.predictors.host._current_context_frame(context, group.donor_frame) + return KernelResult( + columns={ + ("person", c): group.donor_columns[c] for c in group.donor_columns + }, + receipt={ + "group": group.spec.key, + "projection_sha256": codec.sha(qualified.projection), + }, + ) + + +def read_draws(qualified, artifacts): + """Check target history, exact matrix producer and immutable draw bytes.""" + draws = {} + for group in qualified.groups: + name = group.spec.key + matrix_value = artifacts[name + "_matrix"] + require(matrix_value.payload == group.matrix, "MATRIX_BYTES") + matrix = model_input.decode_recipient_matrix(group.matrix) + require(tuple(matrix.features) == qualified.features, "MATRIX_FEATURES") + result = pd.DataFrame(index=matrix.features.index) + models, raw_history = [], [] + for i, target in enumerate(group.spec.targets): + raw = artifacts[f"{name}_raw_{i}"].payload + state_value = artifacts[f"{name}_state_{i}"] + require( + state_value.producer_key == artifacts[f"{name}_raw_{i}"].producer_key, + "RAW_STATE_SIBLINGS", + ) + packet = decode_matrix_apply_state(state_value.payload) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + raw_history.append({"target": target, "sha256": codec.sha(raw)}) + require( + packet["matrix_sha256"] == codec.sha(group.matrix) + and packet["matrix_producer_key"] == matrix_value.producer_key + and chain.entity == "person" + and tuple(chain.predictors) == qualified.features + and tuple(chain.targets) == group.spec.targets + and tuple(chain.completed_targets) == group.spec.targets[: i + 1] + and chain.recipient_index == qrf._index_identity(matrix.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] == raw_history + and application["models"][:i] == models + and len(application["models"]) == i + 1, + "DRAW_CHAIN", + ) + models = application["models"] + result[target] = codec.read_raw_target( + raw, target=target, index=matrix.features.index + ) + draws[name] = result + return draws + + +def _attachment_result(boundary, artifacts): + qualified = boundary.qualified + require( + artifacts["projection"].payload == qualified.projection, "ATTACHMENT_PROJECTION" + ) + draws = read_draws(qualified, artifacts) + columns = values.attach_columns(qualified, boundary.run.population.frame, draws) + receipt = { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "projection_sha256": codec.sha(qualified.projection), + "draw_sha256": { + n: codec.sha(v.payload) for n, v in artifacts.items() if "_raw_" in n + }, + "source_unknowns_preserved": True, + "weights_changed": False, + "prior_wages_consumed": False, + "release_eligible": False, + } + return KernelResult( + columns=columns, + artifacts={"attachment": codec.encode_json(receipt)}, + receipt=receipt, + ) + + +class CurrentSurveyAmountAttachKernel(_Kernel): + ref = "us.survey_amounts.attach@1" + + def run(self, context): + self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + result = _attachment_result(self.boundary, context.artifacts) + self.boundary.pure() + return result + + +def _construct(run, *, groups, n_estimators): + boundary = Boundary(run, groups=groups, n_estimators=n_estimators) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *boundary.nodes)) + ) + kernels = KernelRegistry() + for kernel in run.kernels.as_mapping().values(): + kernels.register(kernel) + for cls in ( + CurrentSurveyAmountProjectionKernel, + CurrentSurveyAmountDonorKernel, + CurrentSurveyAmountColumnsKernel, + CurrentSurveyAmountAttachKernel, + ): + require(cls.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(cls(boundary)) + for kernel in health_graph.health_coverage_kernels( + boundary.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + require_current=boundary.pure, + ): + require(kernel.ref not in kernels.refs(), "HEALTH_KERNEL_COLLISION") + kernels.register(kernel) + registry = parent._registry(run.store.codecs, codecs.SourceCodecRegistry()) + store = ContentStore(run.store.root, codecs=registry) + paths, source_keys = _source_paths_and_keys(compiled, dict(run.sources), store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + require( + all( + keys[n] == run.manifest.node(n).key + and implementations[n] == run.manifest.node(n).kernel_impl_hash + and compiled.graph.node(n) == run.compiled.graph.node(n) + for n in run.compiled.order + ), + "PREFIX_DECLARATION_OR_KEY", + ) + boundary.compiled, boundary.kernels, boundary.store = compiled, kernels, store + boundary.paths = tuple(sorted(paths.items())) + boundary.source_keys = tuple(sorted(source_keys.items())) + boundary.keys = tuple(sorted(keys.items())) + boundary.implementations = tuple(sorted(implementations.items())) + boundary.graph_json = graph_to_json(compiled.graph) + boundary.bound_objects = (compiled, store, kernels, store.codecs) + boundary.kernel_items = tuple(kernels.as_mapping().items()) + boundary.codec_items = tuple(store.codecs.as_mapping().items()) + boundary.bytes_codec_items = tuple(store.codecs.as_bytes_mapping().items()) + boundary.borrow() + return boundary + + +def _verify_models(boundary, loaded, donors): + """Verify persisted models against actual donor values after artifact admission.""" + for group in boundary.qualified.groups: + donor_id, _, fit_prefix, apply_prefix = _ids(group) + donor = donors[donor_id] + first = boundary.compiled.graph.node(fit_prefix + ".000") + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": donor.frame.resolve_weights("person")}), + first.inputs[0], + donor.frame.person, + ) + model = qrf.RegimeGatedQRF( + seed=values.SEED, + n_estimators=boundary.n_estimators, + zero_atol=0, + max_samples_leaf=None, + ) + before = qrf_target.LegacyQRFTrainingState.from_chain( + model.start_chain( + model_frame, + list(boundary.qualified.features), + list(group.spec.targets), + weights="design", + ) + ) + history = [] + for i, target in enumerate(group.spec.targets): + node = f"{fit_prefix}.{i:03d}" + payload = loaded[node, "model"] + packet, after = codec.read_training(loaded[node, "training_state"]) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=codec.sha(payload) + ) + require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, + (*boundary.qualified.features, *group.spec.targets[:i], target), + ), + "TRAINING_DONOR", + ) + history.append( + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + } + ) + require( + packet["models"] == history + and decode_matrix_apply_state( + loaded[f"{apply_prefix}.{i:03d}", "apply_state"] + )["application"]["models"] + == history, + "TRAINING_APPLY_HISTORY", + ) + before = after + + +@dataclass(frozen=True) +class SurveyEnrichmentRun: + parent_run: parent.SurveyPuf55Run + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple + receipt: bytes + + def checked_view(self): + return check_survey_enrichment_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyEnrichmentRun: + """Descriptive values; authority remains in the original retained host.""" + + payload: bytes + digest: str + population: population_ops.Population + + +def _run_seal(run): + return ( + run.receipt, + run.manifest.to_json(), + graph_to_json(run.compiled.graph), + physical._population_stamp(run.population), + parent._heterogeneous_manifest_seals(run.manifest, run.compiled), + tuple(sorted(run.manifest.populations)), + tuple(sorted(run.manifest.mass_ledgers)), + ) + + +def check_survey_enrichment_run(run): + entry = _ISSUED.get(id(run)) + require( + type(run) is SurveyEnrichmentRun and entry is not None and entry[0]() is run, + "UNISSUED_RUN", + ) + _, boundary, stamp, objects, artifacts = entry + try: + boundary.borrow() + require( + all( + a is b + for a, b in zip( + objects, + ( + run.parent_run, + run.population, + run.manifest, + run.compiled, + run.store, + run.kernels, + run.sources, + ), + strict=True, + ) + ), + "RUN_OBJECTS", + ) + loaded = parent.financial._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == artifacts, + "RUN_ARTIFACTS", + ) + boundary.borrow() + require(_run_seal(run) == stamp, "RUN_CHANGED") + boundary.pure() + except BaseException: + if _ISSUED.get(id(run)) is entry: + del _ISSUED[id(run)] + raise + return CheckedSurveyEnrichmentRun( + run.receipt, codec.sha(run.receipt), run.population + ) + + +def run_us_survey_enrichment( + run, *, groups=("unemployment", "health_costs"), n_estimators=100, resume="auto" +): + """Execute one extension and verify its complete observed parent and output.""" + require(resume in ("auto", "require"), "RESUME") + boundary = _construct(run, groups=groups, n_estimators=n_estimators) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require(all(record.hit for record in manifest.nodes.values()), "REQUIRED_HITS") + boundary.borrow() + loaded = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + for node in run.compiled.graph.nodes: + old = run.manifest.node(node.id) + for name, key in old.opaque_artifacts.items(): + require( + loaded[node.id, name] == run.store.load_bytes(key), "PREFIX_ARTIFACT" + ) + physical.replay.same_replayed_population( + run.population, observed[parent.attach.ATTACH_NODE] + ) + require( + loaded[PROJECTION_NODE, "projection"] == boundary.qualified.projection, + "SOURCE_PROJECTION", + ) + attach = boundary.compiled.graph.node(ATTACH_NODE) + artifacts = parent._loaded_values(boundary, manifest, loaded, attach) + result = _attachment_result(boundary, artifacts) + require( + loaded[ATTACH_NODE, "attachment"] == result.artifacts["attachment"], + "ATTACHMENT_ARTIFACT", + ) + # Compare every unchanged prefix terminal Frame and ledger against the + # retained parent manifest. Full execution owners/design anchors on the + # receiving population are checked separately above against the original. + prefix_terminal = {} + for node_id in run.compiled.order: + prefix_terminal[run.compiled.versions[node_id]] = observed[node_id] + for version, population in prefix_terminal.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + run.manifest.population(version), + version, + mass_ledger=run.manifest.mass_ledger(version), + ), + ) + current, donors = {}, {} + health_ids = {n.id for n in boundary.health_nodes} + group_nodes = {_ids(g)[0]: g for g in boundary.qualified.groups} + column_nodes = {_ids(g)[1]: g for g in boundary.qualified.groups} + original = population_ops.Population.from_frame( + boundary.qualified.source_frame, values.predictors.host.survey_graph.CREATE_NODE + ) + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id in health_ids: + health_artifacts = parent._loaded_values(boundary, manifest, loaded, node) + expected = health_graph.expected_health_population( + node_id, + current.get(version), + qualified=boundary.health, + node=node, + artifacts=health_artifacts, + ) + # Bind every persisted source/recode/attachment artifact to its + # independent domain result, including attachment metadata. + expected_result = health_graph._result( + boundary.health, + node, + None if current.get(version) is None else current[version].frame.person, + ) + require( + all( + loaded[node_id, name] == payload + for name, payload in expected_result.artifacts.items() + ), + "HEALTH_RESULT_ARTIFACT", + ) + elif node_id in group_nodes: + expected = population_ops.patch( + original, node, KernelResult(frame=group_nodes[node_id].donor_frame) + ) + elif node_id in column_nodes: + group = column_nodes[node_id] + expected = population_ops.patch( + current[version], + node, + KernelResult( + columns={ + ("person", c): group.donor_columns[c] + for c in group.donor_columns + } + ), + ) + donors[_ids(group)[0]] = expected + elif node_id == ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + _verify_models(boundary, loaded, donors) + for version, population in current.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ), + ) + receipt = codec.encode_json( + { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "manifest_key": manifest.key, + "node_count": len(boundary.compiled.order), + "groups": list(groups), + "complete_population_compared": True, + "projection_sha256": codec.sha(boundary.qualified.projection), + "attachment_sha256": codec.sha(result.artifacts["attachment"]), + "health_projection_sha256": codec.sha(boundary.health.projection), + "health_attachment_sha256": codec.sha( + loaded[health_graph.ATTACH_NODE, "attachment"] + ), + "health_fields": [f.output for f in health_graph.health.FIELDS], + "release_eligible": False, + } + ) + output = SurveyEnrichmentRun( + run, + observed[health_graph.ATTACH_NODE], + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + objects = ( + output.parent_run, + output.population, + output.manifest, + output.compiled, + output.store, + output.kernels, + output.sources, + ) + stamp = _run_seal(output) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + boundary.requalify() + boundary.borrow() + fresh = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes + and _run_seal(output) == stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT", + ) + boundary.pure() + require(id(output) not in _ISSUED, "RUN_ALREADY_ISSUED") + ident = id(output) + + def forget(ref): + old = _ISSUED.get(ident) + if old is not None and old[0] is ref: + del _ISSUED[ident] + + entry = (weakref.ref(output, forget), boundary, stamp, objects, artifact_hashes) + _ISSUED[ident] = entry + return output diff --git a/packages/microcosm-build/tests/test_us_current_survey_amounts.py b/packages/microcosm-build/tests/test_us_current_survey_amounts.py new file mode 100644 index 000000000..89f92aecb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_amounts.py @@ -0,0 +1,316 @@ +"""Invented reporting bases and clone joins; no native data or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_unemployment_source as uc +from microcosm.build.us_runtime import current_survey_amounts as values +from microcosm.fit import model_input +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +@pytest.mark.parametrize( + "amount,age,token,label,canonical", + [ + (100, 40, "1", "known_receipt", 100), + (0, 40, "2", "known_nonreceipt", 0), + (0, 40, "1", "ambiguous_recipient_zero", None), + (0, 40, "0", "niu", None), + (0, 14, "0", "outside_reporting_universe", None), + (0, 14, "2", "contradictory_outside_reporting_universe", None), + (100, 14, "1", "contradictory_outside_reporting_universe", None), + (100, 40, "2", "contradictory_no_positive", None), + (100, 40, "0", "contradictory_niu_positive", None), + (0, 40, "", "missing_receipt_literal", None), + (100, 40, "9", "unrecognized_receipt_literal", None), + (100, 40, "yes", "unrecognized_receipt_literal", None), + (100, 40, "01", "unrecognized_receipt_literal", None), + (np.nan, 40, "1", "missing_amount", None), + ], +) +def test_uc_raw_amount_and_reporting_knownness_are_separate( + amount, age, token, label, canonical +): + basis = uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), [token] + ) + row = basis.iloc[0] + assert row.reporting_status == label + assert row.receipt_literal == token + assert bool(row.source_reporting_universe) == (age >= 15) + assert bool(row.canonical_amount_known) == (canonical is not None) + if canonical is None: + assert np.isnan(row.canonical_amount) + else: + assert row.canonical_amount == canonical + assert ( + np.isnan(row.source_amount) if np.isnan(amount) else row.source_amount == amount + ) + + +@pytest.mark.parametrize( + "amount,age", [(-1, 40), (100000, 40), (np.inf, 40), (0, -1), (0, 40.5)] +) +def test_uc_refuses_out_of_dictionary_numeric_domain(amount, age): + with pytest.raises(ValueError): + uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), ["2"] + ) + + +def test_uc_bounded_literal_reader_and_duplicate_join_refusal(tmp_path): + path = tmp_path / "invented.csv" + header = ",".join(uc.READ_COLUMNS) + "\n" + row = "0000000000000000000001,1,1,40,0,2\n" + path.write_text(header + row) + result = uc._read_capture(path, rows=1) + assert result.iloc[0].UC_YN == "2" + path.write_text(header + row + row) + with pytest.raises(ValueError, match="DUPLICATE"): + uc._read_capture(path, rows=2) + path.write_text(header + row.replace(",0,2", ",0," + "9" * 65)) + with pytest.raises(ValueError, match="TOKEN_BOUND"): + uc._read_capture(path, rows=1) + + +def test_money_literal_join_retains_selected_and_unselected_missingness(): + field = uc.money.MoneyField( + "UC_VAL", + np.array([0.0, 0.0, 40.0], dtype=" 0 else "2" + for age, amount in zip(raw.A_AGE, raw.UC_VAL, strict=True) + ] + if year == 2024: + # These remain source observations, not known model donors. + raw.loc[raw.PERIDNUM.eq(str(7).zfill(22)), "UC_YN"] = "1" + raw.loc[raw.PERIDNUM.eq(str(8).zfill(22)), "UC_YN"] = "0" + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for owner in (uc.coverage, restoration, demographics.demographic): + patch.setattr(owner, "_MEMBER_PINS", tuple(pins)) + output = root / "amount-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, source / "person-income-attachment.h5" + ) + return arguments + + +def test_actual_current_uc_projection_preserves_ambiguous_and_contradictory_sources( + tmp_path, monkeypatch +): + arguments = enrichment_source_arguments(tmp_path, monkeypatch) + prepared = uc.source.prepare_authenticated_survey_population(**arguments) + qualified = uc.qualify_current_asec_unemployment(prepared) + person = qualified.person.set_index("native_person_id") + assert person.loc[105, "canonical_amount"] == 120.0 + assert np.isnan(person.loc[106, "canonical_amount"]) + assert person.loc[106, "reporting_status"] == "outside_reporting_universe" + assert person.loc[107, "reporting_status"] == "ambiguous_recipient_zero" + assert ( + person.loc[108, "reporting_status"] + == "contradictory_outside_reporting_universe" + ) + assert person.loc[[107, 108], "canonical_amount"].isna().all() + assert person.loc[108, "source_amount"] == 500.0 + assert person.amount_validity.eq(1).all() + assert ( + hashlib.sha256(qualified.person.to_json(orient="table").encode()).hexdigest() + == qualified.evidence["projection_sha256"] + ) + health = graph.health_graph.qualify_health_coverage(prepared) + assert set(health.raw.source) == {"asec", "acs"} + graph.health_graph.health_coverage_seal(health) + prepared.checked_view() + + +@pytest.fixture(scope="module") +def enriched(tmp_path_factory, request): + root = tmp_path_factory.mktemp("survey-enrichment") + with pytest.MonkeyPatch.context() as patch: + arguments = enrichment_source_arguments(root, patch) + # Development retries copy only an explicitly supplied invented cache. + # Every hit still passes the real graph source/key/artifact validators; + # no previous process's source handle or receipt grants authority. + cache = os.environ.get("MICROCOSM_TEST_INVENTED_ENRICHMENT_CACHE") + if cache: + shutil.copytree(cache, root / "store") + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = graph.parent.financial.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + print("INVENTED financial cold start", flush=True) + financial = graph.parent.financial.run_atomic_survey_financial( + **arguments, + geography_config=config, + store_root=root / "store", + demographic_conditioning=True, + n_estimators=2, + return_values=True, + ) + print("INVENTED financial closed; PUF source start", flush=True) + definition, paths, _ = _original_sources( + root / "original-puf", full_finalization_support=True + ) + print("INVENTED PUF cold start", flush=True) + parent = graph.parent.run_survey_puf55( + financial, + donor_sources=paths, + fixture_definition=definition, + seed=578, + n_estimators=2, + zero_atol=0, + ) + print( + "INVENTED PUF closed; cached nodes", + sum(n.hit for n in parent.manifest.nodes.values()), + "enrichment cold start", + flush=True, + ) + cold = graph.run_us_survey_enrichment(parent, n_estimators=2) + final_control_only = all( + item.name == "test_final_parent_revocation_invalidates_the_retained_child" + for item in request.session.items + ) + warm = None + if not final_control_only: + print("INVENTED enrichment cold closed; required start", flush=True) + warm = graph.run_us_survey_enrichment( + parent, n_estimators=2, resume="require" + ) + print("INVENTED required closed; owner and readback controls", flush=True) + else: + print( + "INVENTED child ready; focused final parent revocation control", + flush=True, + ) + case = SimpleNamespace( + parent=parent, + financial=financial, + cold=cold, + warm=warm, + root=root, + parent_revoked=False, + ) + yield case + if not case.parent_revoked: + graph.parent.check_survey_puf55_run(parent) + + +def test_actual_enrichment_cold_required_and_complete_parent_preservation(enriched): + cold, warm = enriched.cold, enriched.warm + assert cold.manifest.key == warm.manifest.key + assert all(r.hit for r in warm.manifest.nodes.values()) + assert len(cold.compiled.order) == 271 + graph.physical.replay.same_replayed_population(cold.population, warm.population) + parent = enriched.parent.population + for entity in parent.frame.entities: + pd.testing.assert_frame_equal( + parent.frame.table(entity), + cold.population.frame.table(entity).loc[ + :, parent.frame.table(entity).columns + ], + check_exact=True, + ) + for owner, value in parent.owners.items(): + assert cold.population.owners[owner] == value + assert cold.population.mass_ledger == parent.mass_ledger + people = cold.population.frame.person + for _, rows in people.groupby( + graph.values.provenance.support_source_id_column("person") + ): + assert len(rows) == 2 + outputs = [out for g in graph.values.GROUPS for _, out in g.fields] + np.testing.assert_array_equal( + rows[outputs].iloc[0].to_numpy(), rows[outputs].iloc[1].to_numpy() + ) + assert people.unemployment_compensation.isna().any() + assert people.survey_uc_reporting_status.eq("ambiguous_recipient_zero").any() + assert people.survey_uc_reporting_status.eq( + "contradictory_outside_reporting_universe" + ).any() + assert set(f.output for f in graph.health_graph.health.FIELDS) <= set(people) + assert people.has_medicaid_health_coverage_at_interview.isna().any() + assert people.has_esi.notna().all() + for run in (cold, warm): + assert run.checked_view().population is run.population + table = warm.population.frame.person + column = "unemployment_compensation" + original = float(table.loc[table.index[0], column]) + table.loc[table.index[0], column] = 777.0 + with pytest.raises(ValueError, match="RUN_CHANGED"): + warm.checked_view() + table.loc[table.index[0], column] = original + with pytest.raises(ValueError, match="UNISSUED_RUN"): + warm.checked_view() + + +def test_checked_enrichment_refuses_copied_handle(enriched): + with pytest.raises(ValueError, match="UNISSUED_RUN"): + graph.check_survey_enrichment_run(replace(enriched.cold)) + + +def test_retained_projection_reports_and_artifact_lineage_controls(enriched): + run = enriched.cold + boundary = graph._ISSUED[id(run)][1] + for table, column in ( + (boundary.qualified.native, "unemployment_compensation"), + (boundary.qualified.reports, "survey_current_UC_VAL_origin"), + ): + original = table[column].iloc[0] + table.loc[table.index[0], column] = ( + 987.0 if column == "unemployment_compensation" else "invented-mutation" + ) + with pytest.raises(ValueError, match="BOUNDARY_CHANGED"): + boundary.pure() + table.loc[table.index[0], column] = original + boundary.pure() + original = boundary.health.raw.iloc[0]["HINS1"] + boundary.health.raw.loc[boundary.health.raw.index[0], "HINS1"] = "invented-mutation" + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + boundary.pure() + boundary.health.raw.loc[boundary.health.raw.index[0], "HINS1"] = original + boundary.pure() + loaded = graph.parent.financial._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + node = run.compiled.graph.node(graph.ATTACH_NODE) + artifacts = graph.parent._loaded_values(boundary, run.manifest, loaded, node) + edge = artifacts["projection"] + changed = { + **artifacts, + "projection": replace( + edge, producer_key="f" * 64, key=opaque_artifact_key("f" * 64, "projection") + ), + } + with pytest.raises(ValueError, match="ARTIFACT_PRODUCER_KEY"): + boundary.context(SimpleNamespace(node=node, sources={}, artifacts=changed)) + boundary.context(SimpleNamespace(node=node, sources={}, artifacts=artifacts)) + run.checked_view() + + +def test_complete_frame_store_readback_and_input_coverage(enriched): + from microcosm.build.us_runtime.population_input_coverage import ( + diagnose_us_input_coverage, + ) + from microcosm.build.us_runtime.survey_population_replay import same_replayed_frame + + run = enriched.cold + run.checked_view() + before = diagnose_us_input_coverage( + enriched.parent.population, + compiled=enriched.parent.compiled, + manifest=enriched.parent.manifest, + ) + after = diagnose_us_input_coverage( + run.population, compiled=run.compiled, manifest=run.manifest + ) + outputs = {out for g in graph.values.GROUPS for _, out in g.fields} | { + f.output for f in graph.health_graph.health.FIELDS + } + assert outputs <= set(before.missing_inputs) + assert not outputs & set(after.missing_inputs) + assert len(after.inputs) == 161 + assert not after.ambiguous_grains and not after.block_storage_issues + key = hashlib.sha256(b"invented-survey-enrichment-export" + run.receipt).hexdigest() + run.store.put_frame(key, run.population.frame) + restored = run.store.load_frame(key, node_key=key) + same_replayed_frame(run.population.frame, restored) + (enriched.root / "coverage.json").write_bytes(after.to_bytes()) + (enriched.root / "acceptance.json").write_bytes( + graph.codec.encode_json( + { + "protocol": "microcosm.us.invented-enrichment-test.v1", + "nodes": len(run.compiled.order), + "input_names": len(after.inputs), + "missing_inputs": len(after.missing_inputs), + "frame_store_key": key, + "full_frame_readback": True, + "financial_setup_hits": sum( + n.hit for n in enriched.financial.manifest.nodes.values() + ), + "puf_setup_hits": sum( + n.hit for n in enriched.parent.manifest.nodes.values() + ), + "cold_hits": sum(n.hit for n in run.manifest.nodes.values()), + "required_hits": sum( + n.hit for n in enriched.warm.manifest.nodes.values() + ), + "native_data_used": False, + "release_eligible": False, + } + ) + ) + run.checked_view() + + +def test_final_parent_revocation_invalidates_the_retained_child(enriched): + # Last destructive control after readback: never reconstruct or restore an issuer. + run = enriched.cold + people = enriched.parent.population.frame.person + column = "employment_income_before_lsr" + position = np.flatnonzero(np.isfinite(people[column].to_numpy()))[0] + row = people.index[position] + original = float(people.loc[row, column]) + people.loc[row, column] = original + 321.0 + with pytest.raises(ValueError): + run.checked_view() + enriched.parent_revoked = True + people.loc[row, column] = original + with pytest.raises(ValueError, match="UNISSUED_PUF55_RUN"): + graph.parent.check_survey_puf55_run(enriched.parent) + with pytest.raises(ValueError, match="UNISSUED_RUN"): + run.checked_view() From 2f62bb8448010bcc9b9e1419ea24aafcb85917d1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:11:27 -0400 Subject: [PATCH 151/255] Add the grouped x snapshot cross-product controls Adds the checklist's minimal cross product to the three existing grouped test files rather than a new one, so every case reuses the fixtures already there (the repo runs pytest with --import-mode=importlib and only the build tests directory on pythonpath, so a new calibrate test file could not import them). tools/ci_test_groups.py --verify stays green and no file changes lane. test_grouped_fixed_support.py (24 -> 42 cases): - Snapshots off/on over dense x CSR, positive x fixed-zero support, and plain / warm-started / L2 runs: identical returned weights, trajectory, options, closing loss, stored frame weights, ordered IDs and fixed-zero bytes, and an identical sequence of private post-projection proof payloads. Exactly 20 _apply_constraint and 3 problem.estimates calls either way, and identical torch and numpy RNG state afterwards. - Every grouped snapshot reports best_retained unavailable and the closing_state selection labels; in-loop rows are float32 current, the closing row is float64 selected at epoch == epochs. - Each in-loop row carries the exact float32 tensor that epoch's loss was computed from, proven against a wrapped _apply_constraint, and differs from a recomputation on the accepted vector the private observer reports next. - The closing row equals the reused float64 final diagnostics estimates. - A sink that ruins every delivered payload cannot change the weights or the next payload; a sink that mutates live frame IDs at the closing emission still trips the ordered-ID admission guard, which therefore still runs last. - No record-level content in any grouped payload. test_group_bounds.py (79 -> 82 cases): - Zero epochs through the internal grouped seam emits nothing and evaluates nothing. - Caps of 81 against an initial per-group total of 80 make the projection correct on every epoch; corrections, accepted bytes and group totals are identical with the observer attached. test_solve_us_launch_integration.py (8 -> 12 cases): - The deliberately oscillating example, observed. Ordinary Adam's closing snapshot names the earlier epoch it returned and admits a retained best; the grouped run's names the closing epoch and reports none. A reused emitter that inferred retain-best from an empty receipt would pass the second assertion by accident, so the grouped selection label is asserted explicitly. Full microcosm-calibrate shard: 443 passed. Co-Authored-By: Claude Opus 5 --- .../tests/test_group_bounds.py | 118 +++++- .../tests/test_grouped_fixed_support.py | 358 +++++++++++++++++- .../tests/test_solve_us_launch_integration.py | 120 ++++++ 3 files changed, 594 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-calibrate/tests/test_group_bounds.py b/packages/microcosm-calibrate/tests/test_group_bounds.py index 403dbde2e..ec47fe5b1 100644 --- a/packages/microcosm-calibrate/tests/test_group_bounds.py +++ b/packages/microcosm-calibrate/tests/test_group_bounds.py @@ -14,8 +14,21 @@ import pytest import torch -from microcosm.calibrate import GroupedUpperBounds, Target, TargetSet, calibrate, solve +from microcosm.calibrate import ( + GroupedUpperBounds, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + calibrate, + solve, +) from microcosm.calibrate.initialization import GateInitialization +from microcosm.calibrate.target_snapshots import ( + EVERY_EPOCH, + ITERATE_CURRENT, + ITERATE_SELECTED, +) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights @@ -601,3 +614,106 @@ def test_grouped_mode_constraints_survive_the_reordering(monkeypatch, kwargs, ex with pytest.raises(ValueError, match=expected): _grouped_run(epochs=2, **kwargs) assert not calls + + +def _snapshot_observer(seen, **kwargs): + return TargetSnapshotObserver( + sink=seen.append, + run_id="grouped-bounds", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + **kwargs, + ) + + +def test_zero_epochs_through_the_internal_grouped_path_adds_no_evaluation(monkeypatch): + """A bound observer on a no-update run neither emits nor evaluates. + + ``calibrate`` refuses zero epochs publicly, so the internal seam is the + only way to reach this state; it must stay a pure admission path. + """ + applications = [] + apply_constraint = solve._apply_constraint + + def counted(matrix, weights): + applications.append(matrix.layout) + return apply_constraint(matrix, weights) + + monkeypatch.setattr(solve, "_apply_constraint", counted) + initial = np.array([100.0, 120.0, 80.0, 40.0]) + seen = [] + bound = _snapshot_observer(seen).bind( + names=("population@2024",), targets=np.array([200.0]) + ) + final, trajectory = _internal( + initial, _groups((220.0, 120.0)), epochs=0, snapshots=bound + ) + assert not len(trajectory) + assert final.tobytes() == initial.tobytes() + assert applications == [] + assert seen == [] + + +@pytest.mark.parametrize("sparse", [False, True], ids=["dense", "csr"]) +def test_a_binding_bound_that_actually_corrects_is_unchanged_by_snapshots( + monkeypatch, sparse +): + """Projection still corrects, identically, with the observer attached. + + Caps of 81 sit just above the initial per-group total of 80, so the very + first Adam step overshoots them and the projection has to pull each group + back on every epoch — the case where an observer could plausibly disturb + the accepted state. + """ + if sparse: + monkeypatch.setattr(solve, "_SPARSE_MIN_CELLS", 1) + monkeypatch.setattr(solve, "_SPARSE_DENSITY_CUTOFF", 1.0) + + def run(**extra): + observations = [] + result = _grouped_run( + grouped_upper_bounds=_groups((81.0, 81.0)), + _post_projection_observer=observations.append, + **extra, + ) + return result, observations + + off, observed_off = run() + seen = [] + on, observed_on = run(target_snapshots=_snapshot_observer(seen)) + + # The bound really binds on this fixture, or the test proves nothing. + assert sum(item["corrected_group_count"] for item in observed_off) > 0 + + assert on.weights.tobytes() == off.weights.tobytes() + assert on.loss_trajectory.tobytes() == off.loss_trajectory.tobytes() + assert on.options == off.options + assert on.options["grouped_upper_bounds"] == off.options["grouped_upper_bounds"] + for payload_on, payload_off in zip(observed_on, observed_off, strict=True): + assert payload_on["weights"].tobytes() == payload_off["weights"].tobytes() + assert ( + payload_on["group_totals"].tobytes() + == payload_off["group_totals"].tobytes() + ) + assert ( + payload_on["corrected_group_count"] == payload_off["corrected_group_count"] + ) + + in_loop = [item for item in seen if item["iterate"] != ITERATE_SELECTED] + assert [item["epoch"] for item in in_loop] == list(range(1, 7)) + assert all(item["iterate"] == ITERATE_CURRENT for item in in_loop) + assert all(item["precision"] == "float32" for item in in_loop) + assert all( + item["selection"] + == { + "rule": "closing_state", + "constraint_mode": "grouped_upper_bounds", + "grouped_preserve_zeros": False, + } + for item in seen + ) + selected = [item for item in seen if item["iterate"] == ITERATE_SELECTED] + assert len(selected) == 1 + assert selected[0]["epoch"] == 6 and selected[0]["precision"] == "float64" + assert [row["estimate"] for row in selected[0]["targets"]] == [ + diagnostic.final_estimate for diagnostic in on.diagnostics + ] diff --git a/packages/microcosm-calibrate/tests/test_grouped_fixed_support.py b/packages/microcosm-calibrate/tests/test_grouped_fixed_support.py index 9be61a66f..f9a0c7313 100644 --- a/packages/microcosm-calibrate/tests/test_grouped_fixed_support.py +++ b/packages/microcosm-calibrate/tests/test_grouped_fixed_support.py @@ -9,7 +9,21 @@ import pytest import torch -from microcosm.calibrate import GroupedUpperBounds, Target, TargetSet, calibrate, solve +from microcosm.calibrate import ( + GroupedUpperBounds, + Target, + TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, + calibrate, + solve, +) +from microcosm.calibrate.matrix import CalibrationProblem +from microcosm.calibrate.target_snapshots import ( + EVERY_EPOCH, + ITERATE_CURRENT, + ITERATE_SELECTED, +) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights @@ -59,6 +73,79 @@ def _fit(**kwargs): return calibrate(_frame(), _targets(), **options) +def _observer(seen, **kwargs): + """The recording sink every grouped snapshot case below shares.""" + return TargetSnapshotObserver( + sink=seen.append, + run_id="grouped-run", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + **kwargs, + ) + + +def _count_evaluations(monkeypatch): + """Count the real forward and closing evaluations without changing them. + + Returns the counter and the list of float32 estimate vectors the loss was + actually computed from, in epoch order, so a test can prove a snapshot + carries that exact tensor rather than a recomputation. + """ + counts = {"apply": 0, "estimates": 0} + observed = [] + apply_constraint = solve._apply_constraint + problem_estimates = CalibrationProblem.estimates + + def apply(matrix, weights): + counts["apply"] += 1 + value = apply_constraint(matrix, weights) + observed.append(value.detach().cpu().numpy().copy()) + return value + + def estimates(self, weights): + counts["estimates"] += 1 + return problem_estimates(self, weights) + + monkeypatch.setattr(solve, "_apply_constraint", apply) + monkeypatch.setattr(CalibrationProblem, "estimates", estimates) + return counts, observed + + +def _rng_state(): + return ( + torch.random.get_rng_state().clone(), + np.random.get_state()[1].copy(), + ) + + +def _seeded(): + torch.manual_seed(0) + np.random.seed(0) + + +def _positive_groups(): + """The same ordered map, with a bound group zero can meet when positive. + + ``_groups`` caps the origin group at zero, which is exactly what makes it + a fixed-zero fixture; the default grouped contract needs every row strictly + positive, so the positive arm of the cross product raises that one bound and + leaves the map, the ordering and the other two bounds alone. + """ + return GroupedUpperBounds( + tuple(range(6)), np.array([0, 0, 1, 1, 2, 2]), np.array([40.0, 100.0, 30.0]) + ) + + +#: Feasible under _groups(): group 0 stays at its zero bound, group 1 totals 80 +#: against 100 and group 2 totals 25 against 30, with the fixed zeros intact. +_WARM = np.array([0.0, 0.0, 30.0, 50.0, 0.0, 25.0]) + +_VARIANTS = [ + pytest.param({}, id="plain"), + pytest.param({"warm_start_weights": _WARM}, id="warm-start"), + pytest.param({"l2_lambda": 0.01}, id="l2"), +] + + @pytest.mark.parametrize("sparse", [False, True]) def test_full_population_survives_positive_only_log_parameters(monkeypatch, sparse): if sparse: @@ -290,3 +377,272 @@ def corrupt(self, weights, last_corrected_count=0): monkeypatch.setattr(GroupedUpperBounds, "diagnostics", corrupt) with pytest.raises(ValueError, match="ordered IDs"): _fit() + + +@pytest.mark.parametrize("sparse", [False, True], ids=["dense", "csr"]) +@pytest.mark.parametrize("preserve_zeros", [False, True], ids=["positive", "fixed"]) +@pytest.mark.parametrize("extra", _VARIANTS) +def test_snapshots_leave_the_grouped_run_bit_identical( + monkeypatch, sparse, preserve_zeros, extra +): + """Enabling the observer changes nothing a grouped run produces or draws. + + The cross product is dense/CSR times positive/fixed-zero support times a + plain, a warm-started and an L2 run, over the shared mixed and zero-only + groups with group two's bound binding on the initial vector. + """ + if sparse: + monkeypatch.setattr(solve, "_SPARSE_MIN_CELLS", 1) + monkeypatch.setattr(solve, "_SPARSE_DENSITY_CUTOFF", 1.0) + options = dict(extra) + if preserve_zeros: + weights = (0.0, 0.0, 40.0, 40.0, 0.0, 20.0) + groups = _groups() + else: + # The default grouped contract is strictly positive, so lift the zeros. + weights = (10.0, 10.0, 40.0, 40.0, 5.0, 20.0) + groups = _positive_groups() + if "warm_start_weights" in options: + options["warm_start_weights"] = np.array([5.0, 5.0, 30.0, 50.0, 5.0, 20.0]) + + def run(observer): + _seeded() + with monkeypatch.context() as scoped: + counts, observed = _count_evaluations(scoped) + proof = [] + result = calibrate( + _frame(weights), + _targets(), + epochs=20, + learning_rate=0.08, + grouped_upper_bounds=groups, + grouped_preserve_zeros=preserve_zeros, + _post_projection_observer=proof.append, + **({} if observer is None else {"target_snapshots": observer}), + **options, + ) + return result, counts, observed, proof, _rng_state() + + off, counts_off, observed_off, proof_off, rng_off = run(None) + seen = [] + on, counts_on, observed_on, proof_on, rng_on = run(_observer(seen)) + + assert on.weights.tobytes() == off.weights.tobytes() + assert on.loss_trajectory.tobytes() == off.loss_trajectory.tobytes() + assert on.options == off.options + assert float(on.closing_loss) == float(off.closing_loss) + stored_on = on.frame.resolve_weights("household").values + assert ( + stored_on.tobytes() == off.frame.resolve_weights("household").values.tobytes() + ) + assert ( + tuple(on.frame.table("household")["household_id"]) + == tuple(off.frame.table("household")["household_id"]) + == tuple(range(6)) + ) + if preserve_zeros: + zeros = np.asarray(weights) == 0 + assert stored_on[zeros].tobytes() == np.zeros(int(zeros.sum())).tobytes() + + # The private proof seam is byte-for-byte the same sequence of payloads. + assert len(proof_on) == len(proof_off) == 21 + for payload_on, payload_off in zip(proof_on, proof_off, strict=True): + assert payload_on.keys() == payload_off.keys() + assert payload_on["weights"].tobytes() == payload_off["weights"].tobytes() + assert ( + payload_on["group_totals"].tobytes() + == payload_off["group_totals"].tobytes() + ) + assert ( + payload_on["corrected_group_count"] == payload_off["corrected_group_count"] + ) + assert payload_on["household_ids"] == payload_off["household_ids"] + + # Cadence buys observation, not evaluation, and advances no RNG stream. + assert counts_on == counts_off == {"apply": 20, "estimates": 3} + assert len(observed_on) == len(observed_off) == 20 + assert torch.equal(rng_on[0], rng_off[0]) + assert (rng_on[1] == rng_off[1]).all() + assert len(seen) == 21 + + +@pytest.mark.parametrize("preserve_zeros", [False, True], ids=["positive", "fixed"]) +def test_grouped_snapshots_report_a_closing_state_run_with_no_retained_best( + monkeypatch, preserve_zeros +): + """Grouped labels say "this solver retains nothing", not "best is unset".""" + if preserve_zeros: + weights = (0.0, 0.0, 40.0, 40.0, 0.0, 20.0) + groups = _groups() + else: + weights = (10.0, 10.0, 40.0, 40.0, 5.0, 20.0) + groups = _positive_groups() + seen = [] + result = calibrate( + _frame(weights), + _targets(), + epochs=20, + learning_rate=0.08, + grouped_upper_bounds=groups, + grouped_preserve_zeros=preserve_zeros, + target_snapshots=_observer(seen), + ) + in_loop = [item for item in seen if item["iterate"] != ITERATE_SELECTED] + selected = [item for item in seen if item["iterate"] == ITERATE_SELECTED] + assert len(selected) == 1 + assert [item["epoch"] for item in in_loop] == list(range(1, 21)) + labels = { + "rule": "closing_state", + "constraint_mode": "grouped_upper_bounds", + "grouped_preserve_zeros": preserve_zeros, + } + for item in seen: + assert item["best_retained"] == { + "available": False, + "epoch": None, + "loss": None, + } + assert item["selection"] == labels + assert item["epochs"] == 20 + for item in in_loop: + assert item["iterate"] == ITERATE_CURRENT + assert item["precision"] == "float32" + assert selected[0]["precision"] == "float64" + assert selected[0]["epoch"] == 20 + # The label is the option the result itself records, not a second opinion. + assert result.options["iterate_selection"] == "closing_state" + assert result.options["iterate_selection_receipt"] == {} + + +def test_grouped_in_loop_estimates_are_the_observed_loss_tensor(monkeypatch): + """Each row is the epoch's own float32 forward pass, not a recomputation. + + The contrast is the post-update accepted vector the private observer + reports at the same epoch number: producing target totals for that vector + would need another matrix evaluation, and the values differ. + """ + counts, observed = _count_evaluations(monkeypatch) + seen = [] + proof = [] + result = calibrate( + _frame(), + _targets(), + epochs=20, + learning_rate=0.08, + grouped_upper_bounds=_groups(), + grouped_preserve_zeros=True, + _post_projection_observer=proof.append, + target_snapshots=_observer(seen), + ) + in_loop = [item for item in seen if item["iterate"] != ITERATE_SELECTED] + assert len(in_loop) == len(observed) == 20 + differed = 0 + for epoch, item in enumerate(in_loop): + emitted = [row["estimate"] for row in item["targets"]] + assert emitted == list(observed[epoch].astype(np.float64)) + assert item["loss"] == float(result.loss_trajectory[epoch]) + # The accepted vector of the projection that follows this evaluation. + accepted = result.problem.estimates(proof[epoch + 1]["weights"]) + if not np.array_equal(np.asarray(emitted), accepted): + differed += 1 + assert differed == 20 + + selected = [item for item in seen if item["iterate"] == ITERATE_SELECTED][0] + finals = [diagnostic.final_estimate for diagnostic in result.diagnostics] + assert [row["estimate"] for row in selected["targets"]] == finals + assert selected["loss"] == float(result.closing_loss) + # The counted closing evaluations are the two diagnostics ones plus the + # single reused final estimate; the selected snapshot adds none. + assert counts["estimates"] == 3 + 20 # 20 recomputations this test made + + +def test_a_sink_mutating_delivered_payloads_cannot_reach_the_solver(monkeypatch): + """Delivered metadata and target rows are the sink's to ruin, alone.""" + baseline = _fit() + seen = [] + delivered = [] + + def hostile(payload): + # Read the payload as delivered, then ruin every mutable part of it. + delivered.append( + ( + payload["epoch"], + payload["iterate"], + dict(payload["selection"]), + [row["name"] for row in payload["targets"]], + ) + ) + seen.append(payload) + payload["selection"]["rule"] = "tampered" + payload["best_retained"]["available"] = True + payload["epoch"] = -1 + for row in payload["targets"]: + row["estimate"] = 0.0 + row["name"] = "tampered" + + result = _fit( + target_snapshots=TargetSnapshotObserver( + sink=hostile, + run_id="grouped-run", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + ) + ) + assert result.weights.tobytes() == baseline.weights.tobytes() + assert result.loss_trajectory.tobytes() == baseline.loss_trajectory.tobytes() + # Every later payload arrived freshly built rather than carrying the damage. + assert len(delivered) == 21 + assert [item[0] for item in delivered] == list(range(1, 21)) + [20] + assert [item[1] for item in delivered[:-1]] == [ITERATE_CURRENT] * 20 + assert delivered[-1][1] == ITERATE_SELECTED + for _, _, selection, names in delivered: + assert selection["rule"] == "closing_state" + assert names == ["native@0", "detail@0"] + # And the sink's damage really did land on the objects it was handed. + assert seen[-1]["targets"][0]["name"] == "tampered" + + +def test_a_sink_mutating_live_frame_ids_still_trips_the_final_admission(monkeypatch): + """The last thing to run is still the identity guard, not the sink.""" + frame = _frame() + + def saboteur(payload): + if payload["iterate"] == ITERATE_SELECTED: + frame.table("household").loc[[0, 2], "household_id"] = [2, 0] + + with pytest.raises(ValueError, match="ordered IDs"): + calibrate( + frame, + _targets(), + epochs=20, + learning_rate=0.08, + grouped_upper_bounds=_groups(), + grouped_preserve_zeros=True, + target_snapshots=TargetSnapshotObserver( + sink=saboteur, + run_id="grouped-run", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + ), + ) + + +def test_grouped_snapshots_carry_no_record_level_content(): + """Aggregate target rows only: no weights, IDs, group map or bounds.""" + seen = [] + _fit(target_snapshots=_observer(seen)) + forbidden = {"household_id", "household_ids", "weights", "group_indices"} + + def scan(value): + if isinstance(value, dict): + assert not (set(map(str, value)) & forbidden), sorted(value) + for item in value.values(): + scan(item) + elif isinstance(value, (list, tuple)): + # Two compiled targets; nothing of the six-record length. + assert len(value) != 6 + for item in value: + scan(item) + + for payload in seen: + scan(payload) + assert payload["n_targets"] == 2 + assert len(payload["targets"]) == 2 diff --git a/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py b/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py index 308263840..3c7893a64 100644 --- a/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py +++ b/packages/microcosm-calibrate/tests/test_solve_us_launch_integration.py @@ -18,10 +18,13 @@ GroupedUpperBounds, Target, TargetSet, + TargetSnapshotCadence, + TargetSnapshotObserver, calibrate, diagnostics_payload, ) from microcosm.calibrate import solve as solve_module +from microcosm.calibrate.target_snapshots import EVERY_EPOCH, ITERATE_SELECTED from microcosm.frame import EntitySchema, Frame, WeightKind, Weights @@ -209,3 +212,120 @@ def forbidden_compile(*args, **kwargs): monkeypatch.setattr(solve_module, "build_constraint_matrix", forbidden_compile) with pytest.raises(ValueError, match="original aligned constraints"): rebuild(grouped, grouped_frame) + + +@pytest.mark.parametrize("fixed_zeros", [False, True], ids=["positive", "fixed-zeros"]) +@pytest.mark.parametrize( + "target,warm,epochs,selected_epoch", + [ + pytest.param(2.01, 2.0, 2, 0, id="warm-start-overshoot"), + pytest.param(1.5, None, 3, 2, id="intermediate-overshoot"), + ], +) +def test_snapshots_describe_each_solver_s_own_selection_on_the_same_oscillation( + fixed_zeros, target, warm, epochs, selected_epoch +) -> None: + """The same oscillating example, now observed by both solvers. + + Ordinary Adam's closing snapshot must name the earlier epoch it actually + returned and admit a retained best; the grouped run's must name the closing + epoch and say no best iterate was retained at all. A single reused emitter + that inferred retain-best from an empty receipt would look right here only + by accident, which is why the grouped label is asserted too. + """ + targets = TargetSet( + ( + Target( + name="population", + entity="household", + value=target, + measure="household_count", + ), + ) + ) + common = {"epochs": epochs, "learning_rate": 0.2, "mass": "free", "seed": 0} + + def observer(seen): + return TargetSnapshotObserver( + sink=seen.append, + run_id="oscillation", + cadence=TargetSnapshotCadence(every=EVERY_EPOCH), + ) + + ordinary_seen = [] + ordinary = calibrate( + _count_frame((1.0,)), + targets, + warm_start_weights=None if warm is None else np.array([warm]), + target_snapshots=observer(ordinary_seen), + **common, + ) + initial = np.array([0.0, 1.0, 0.0] if fixed_zeros else [1.0]) + groups = GroupedUpperBounds( + tuple(range(len(initial))), + np.array([0, 1, 1] if fixed_zeros else [0]), + np.array([0.0, 3.0] if fixed_zeros else [3.0]), + ) + grouped_seen = [] + grouped = calibrate( + _count_frame(tuple(initial)), + targets, + warm_start_weights=None if warm is None else initial * warm, + grouped_upper_bounds=groups, + grouped_preserve_zeros=fixed_zeros, + target_snapshots=observer(grouped_seen), + **common, + ) + + # The example is still an oscillation the two rules resolve differently. + assert grouped.final_loss > ordinary.final_loss + 0.01 + + ordinary_selected = [ + item for item in ordinary_seen if item["iterate"] == ITERATE_SELECTED + ] + grouped_selected = [ + item for item in grouped_seen if item["iterate"] == ITERATE_SELECTED + ] + assert len(ordinary_selected) == len(grouped_selected) == 1 + ordinary_selected = ordinary_selected[0] + grouped_selected = grouped_selected[0] + + assert ordinary_selected["epoch"] == selected_epoch + assert ordinary_selected["epoch"] < epochs + assert ordinary_selected["best_retained"]["available"] is True + assert ordinary_selected["selection"]["rule"] == "best_feasible_loss" + assert ordinary_selected["selection"]["selected_epoch"] == selected_epoch + + assert grouped_selected["epoch"] == epochs + assert grouped_selected["best_retained"] == { + "available": False, + "epoch": None, + "loss": None, + } + assert grouped_selected["selection"] == { + "rule": "closing_state", + "constraint_mode": "grouped_upper_bounds", + "grouped_preserve_zeros": fixed_zeros, + } + + # Both closing snapshots are the float64 estimates their result returned. + for selected, result in ( + (ordinary_selected, ordinary), + (grouped_selected, grouped), + ): + assert selected["precision"] == "float64" + assert selected["epochs"] == epochs + assert [row["estimate"] for row in selected["targets"]] == [ + diagnostic.final_estimate for diagnostic in result.diagnostics + ] + assert selected["loss"] == float(result.closing_loss) + + # Every in-loop grouped snapshot reports the closing-state rule; the + # ordinary run's carry its own best-iterate bookkeeping instead. + for item in grouped_seen: + assert item["best_retained"]["available"] is False + assert any( + item["best_retained"]["available"] is True + for item in ordinary_seen + if item["iterate"] != ITERATE_SELECTED + ) From c71ea055b3e99167505d9aed2ec33cc76cee92d5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:12:10 -0400 Subject: [PATCH 152/255] Stop completing amounts the printed dictionary leaves ambiguous Seven corrections from adversarial review, each removing a claim the source does not support: A recipient zero now resolves only where the printed entry says its zero is valid dollars, read from the pinned domains artifact. That is ANN_VAL alone, whose NIU is the separate -1 code. RNT_VAL and FRSE_VAL print the same "0 = none or niu" label the gross entries print, so receipt_with_net_zero stays a distinct status but no longer completes a known zero. Off-route distribution dollars or an answered off-route recipiency now contradict the row instead of leaving an affirmative known zero, and an applicable slot declaring an account whose amount is a "none or niu" zero leaves the composition and the regular-IRA share unresolved. Other-income routing follows the printed receipt universe, so an out-of-universe row no longer reports a category or reads as reported alimony. Allocation origins no longer assert publisher-confirmed non-allocation: each flag prints a conditional universe that is not evaluated here, so an all-zero reading is reported as exactly that. Receipt code labels are per entry, so OI_YN reports its printed "none or niu" while the other eight report "niu"/"Niu" as printed. The literal transport gains its own digest inside the final identity check, and the public projection validates its routing token arrays. Co-Authored-By: Claude Opus 5 --- docs/us-current-asec-income-routing-source.md | 46 ++- .../current_asec_income_routing_source.py | 144 ++++++-- .../test_us_current_asec_income_routing.py | 317 ++++++++++++++++-- 3 files changed, 439 insertions(+), 68 deletions(-) diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index f9df50b0c..bf364c66b 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -15,6 +15,11 @@ graph does not yet supply: Thirteen published allocation flags are read alongside them. +Both returned frames carry a digest: `evidence["projection_sha256"]` over +`person` and `evidence["literals_sha256"]` over `asec_literals`. Both are +re-checked at the end of the qualifier, so a host comparing the documented +digests detects a corrupted transport on either frame. + `qualify_current_asec_income_routing(preparation)` takes a live `AuthenticatedSurveyPopulationPreparation`, requalifies it, captures the pinned 2024 member once, joins it to the money parent by exact native keys, and returns @@ -58,7 +63,14 @@ around its own I/O, exactly as the accepted unemployment qualifier requires. `net_property_component_split_known` is constant `False`, so the total is not independently labelled rental and no subdivision is separately modelled. - `FRSE_VAL` is farm self-employment, not nonfarm `SEMP_VAL`; - `farm_is_nonfarm_self_employment` is constant `False`. + `farm_is_nonfarm_self_employment` is constant `False`. Its printed label + includes the composite clause naming `ERN_VAL` (when `ERN_SRCE=3`) and + `FRM_VAL`, so the total already contains those components. +- `retirement_distribution_regular_ira_amount` is published only when every + applicable slot is fully resolved on both axes. An unreadable account code + could itself be a regular IRA, and a declared account whose amount is a + "none or niu" zero does not observe a zero dollar distribution, so both leave + the regular-IRA share unknown rather than at zero. - `OI_OFF` code 20 is the reported alimony category. Nothing maps any other code — including 19, `anything else` — onto alimony or onto a miscellaneous residual; `other_income_residual_rule_applied` is constant `False`. @@ -74,23 +86,42 @@ around its own I/O, exactly as the accepted unemployment qualifier requires. ## Reporting status vocabulary `receipt_status` is the single classifier. Only `known_receipt`, -`receipt_with_net_zero` and `known_nonreceipt` establish a dollar reading +`known_recipient_zero` and `known_nonreceipt` establish a dollar reading (`KNOWN_AMOUNT_STATUSES`); every other status leaves the canonical amount unknown rather than completing it with zero. | Status | Meaning | | --- | --- | | `known_receipt` | in universe, receipt yes, amount non-zero (a signed loss included) | -| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`) | +| `known_recipient_zero` | in universe, receipt yes, amount zero on an entry whose printed zero is valid dollars — `ANN_VAL` alone, whose NIU is the separate `-1` code | +| `receipt_with_net_zero` | in universe, receipt yes, amount zero on a signed net measure (`RNT_VAL`, `FRSE_VAL`). Distinct from a gross entry's recipient zero, but **not** a known amount: both print `0 = none or niu` | | `ambiguous_recipient_zero` | in universe, receipt yes, amount zero on a gross entry whose printed zero reads "none or niu" | | `known_nonreceipt` | in universe, receipt no, amount zero | | `niu` | in universe, receipt 0, amount zero or a declared non-money code | | `missing_amount` / `missing_receipt_literal` | one side of the pair is absent | | `unrecognized_receipt_literal` | the receipt literal is malformed or outside the printed range | | `contradictory_no_nonzero`, `contradictory_niu_nonzero`, `contradictory_declared_niu_amount`, `contradictory_outside_reporting_universe` | retained source contradictions, excluded from every canonical amount | +| `contradictory_offroute_evidence` | distributions only: the route that does not apply carries dollars or an answered recipiency | +| `unresolved_slot_composition` | distributions only: an applicable slot declares an account whose amount is a "none or niu" zero | | `outside_reporting_universe` | the printed universe excludes the row and the literals agree | | `unresolved_reporting_universe` | the printed universe cannot be resolved from the retained literals | +Which recipient zeros resolve is read from the pinned domains artifact, not +decided here: `_zero_is_dollars` compares each entry's `zero_semantics` against +`valid_zero_dollars`. Eight of the nine entries record +`none_or_niu_not_distinguishable_from_amount_alone`, including both signed net +measures, so a signed zero is separated by label but never completed. + +The receipt code labels are per entry, not shared: `receipt_codes(field)` reads +the printed zero label from `RECEIPT_ENTRIES`, so `OI_YN` reports `none or niu` +where `PEN_YN` reports `niu` and `FRSE_YN` reports `Niu` exactly as printed. + +Other-income routing (`other_income_routing_status`) follows the same universe: +`OI_OFF` is printed for `OI_YN = 1` and `OI_YN` for persons aged 15+, so a row +outside or unresolved on that universe reports +`outside_reporting_universe_routing` or `unresolved_reporting_universe_routing` +and never a reported category. + Universes are per family and never reduced to age alone. `PEN_YN`, `ANN_YN`, `RNT_YN` and `OI_YN` print `All Persons aged 15+`. `FRSE_YN` prints `ERN_YN=1 or FRMOTR=1`, so the farm universe is `True` when either literal is @@ -149,9 +180,12 @@ the pooled 2022/2023 cohorts would require handling that restatement first. `*_allocation_origin` is derived from published flags only: - `publisher_allocated` — some applicable flag is non-zero. -- `publisher_no_allocation` — every applicable flag is populated and zero, and - every field in the family carries a published flag. -- `publisher_no_allocation_on_published_flags_only` — the same, but the family +- `published_flags_all_zero` — every applicable flag is populated and reads + zero, and every field in the family carries a published flag. This is **not** + an assertion of non-allocation: each flag prints a conditional universe (for + example `I_RNTVAL` is printed for `RNT_VAL > 0`) and those universes are not + evaluated here, so a non-recipient row sits outside its own flag's universe. +- `published_flags_all_zero_with_unflagged_fields` — the same, but the family also holds a field the dictionary does not flag. - `allocation_flag_not_populated` — a flag literal is absent. Every flag here prints a conditional universe, so this is not a defect and not "no diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 56502ea36..afb51e24b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -146,7 +146,18 @@ def printed_amount_entries(): "FRMOTR": (1, 389, 43, "6C-22", "ERN_OTR = 1", "niu"), "OI_YN": (1, 555, 47, "6C-26", "All Persons aged 15+", "none or niu"), } -RECEIPT_CODES = {0: "niu_or_none", 1: "yes", 2: "no"} +RECEIPT_CODE_DOMAIN = (0, 1, 2) + + +def receipt_codes(field): + """Printed yes/no labels for one entry; the zero label is not shared. + + OI_YN prints "none or niu" where the other entries print "niu"/"Niu", so a + single shared label would attribute a "respondent reported none" reading to + eight fields whose dictionary entry does not support it. + """ + return {0: RECEIPT_ENTRIES[field][5], 1: "yes", 2: "no"} + # Retirement account identity. Code 4 names a regular IRA; it does not observe # any taxable fraction of the distribution. @@ -377,20 +388,28 @@ def amount_state(value, status): # Statuses whose dollar reading is established by the source itself. Every other # status leaves the canonical amount unknown rather than completing it with zero. +# A recipient zero qualifies only where the printed entry says the zero is valid +# dollars; every entry here except ANN_VAL prints "0 = none or niu", which the +# pinned domains artifact records as not distinguishable from an amount alone. +DOLLAR_ZERO_SEMANTICS = "valid_zero_dollars" KNOWN_AMOUNT_STATUSES = ( "known_receipt", - "receipt_with_net_zero", + "known_recipient_zero", "known_nonreceipt", ) -def receipt_status(universe, receipt, amount_kind, *, net_measure): +def receipt_status(universe, receipt, amount_kind, *, net_measure, zero_is_dollars): """Classify one family row without inventing receipt, absence or dollars. `universe` is True, False, or None when the printed universe itself is not - resolved by the retained literals. A zero amount under a yes answer is a - genuine net zero only for a signed net measure; for a gross entry the - printed zero reads "none or niu" and stays ambiguous. + resolved by the retained literals. + + A zero amount under a yes answer establishes a dollar reading only when the + printed entry says its zero is valid dollars. `net_measure` does not make a + zero known: it only separates a signed net entry's recipient zero, which a + consumer may want to treat differently, from a gross entry's. Both stay + unknown, because both print "0 = none or niu". """ code, status = receipt if universe is None: @@ -415,11 +434,13 @@ def receipt_status(universe, receipt, amount_kind, *, net_measure): if code == 1: if amount_kind == "nonzero": return "known_receipt" + if zero_is_dollars: + return "known_recipient_zero" return "receipt_with_net_zero" if net_measure else "ambiguous_recipient_zero" return "known_nonreceipt" if amount_kind == "zero" else "contradictory_no_nonzero" -def _classify(amounts, statuses, universes, receipts, *, net_measure): +def _classify(amounts, statuses, universes, receipts, *, net_measure, zero_is_dollars): """Row-wise family classification returning labels and canonical amounts.""" labels, canonical = [], np.full(len(amounts), np.nan, dtype=np.float64) kinds, sources = [], np.full(len(amounts), np.nan, dtype=np.float64) @@ -427,13 +448,24 @@ def _classify(amounts, statuses, universes, receipts, *, net_measure): kind, dollars = amount_state(value, statuses[i]) kinds.append(kind) sources[i] = dollars - label = receipt_status(universes[i], receipts[i], kind, net_measure=net_measure) + label = receipt_status( + universes[i], + receipts[i], + kind, + net_measure=net_measure, + zero_is_dollars=zero_is_dollars, + ) labels.append(label) if label in KNOWN_AMOUNT_STATUSES: canonical[i] = dollars if label == "known_receipt" else 0.0 return labels, canonical, kinds, sources +def _zero_is_dollars(field): + """Whether the printed entry says its zero is valid dollars, not none/niu.""" + return printed_amount_entries()[field].zero_semantics == DOLLAR_ZERO_SEMANTICS + + def _codes_frame(prefix, tokens, allowed, labels=None, *, width=1): """Typed code/label/status columns for one printed routing literal.""" pairs = [literal_code(t, allowed, width=width) for t in tokens] @@ -455,10 +487,12 @@ def _age_universe(ages): def _allocation_origin(flags, unflagged): """Family allocation provenance from published flags only. - Every published flag here prints a conditional universe, so an unpopulated - literal is not a defect and is not "no allocation" either. A family that - also publishes unflagged fields can never report a clean "no allocation": - the dictionary does not publish that evidence for those fields. + Every published flag here prints a conditional universe (for example + I_RNTVAL is printed for RNT_VAL > 0), and those universes are not evaluated + here. An unpopulated literal is therefore not a defect, and an all-zero + reading is not an assertion of non-allocation: on a non-recipient row the + flag is outside its own printed universe. The zero readings are reported as + exactly that, never as publisher-confirmed absence of allocation. """ origins = [] count = len(next(iter(flags.values()))[0]) @@ -472,9 +506,9 @@ def _allocation_origin(flags, unflagged): elif any(s == "missing" for s in statuses): origins.append("allocation_flag_not_populated") elif unflagged: - origins.append("publisher_no_allocation_on_published_flags_only") + origins.append("published_flags_all_zero_with_unflagged_fields") else: - origins.append("publisher_no_allocation") + origins.append("published_flags_all_zero") return pd.array(origins, dtype="string") @@ -486,11 +520,12 @@ def _pension_annuity(raw, ages): ("pension", "PNSN_VAL", "PEN_YN"), ("annuity", "ANN_VAL", "ANN_YN"), ): + printed = receipt_codes(receipt_field) frame, codes, statuses = _codes_frame( "pension_annuity_" + prefix + "_receipt", raw[receipt_field], - RECEIPT_CODES, - RECEIPT_CODES, + printed, + printed, ) labels, canonical, kinds, sources = _classify( raw["amounts"][amount_field], @@ -498,6 +533,7 @@ def _pension_annuity(raw, ages): universe, list(zip(codes, statuses, strict=True)), net_measure=False, + zero_is_dollars=_zero_is_dollars(amount_field), ) out = pd.concat([out, frame], axis=1) out["pension_annuity_" + prefix + "_source_total"] = sources @@ -542,7 +578,7 @@ def _retirement_distribution(raw, ages): ("slot1_young", "DST_SC1_YNG", "DST_VAL1_YNG", "under_age58"), ("slot2_young", "DST_SC2_YNG", "DST_VAL2_YNG", "under_age58"), ) - slot_codes, slot_amounts, slot_kinds = {}, {}, {} + slot_codes, slot_amounts, slot_kinds, slot_status = {}, {}, {}, {} for name, code_field, amount_field, slot_route in slots: prefix = "retirement_distribution_" + name + "_account" frame, codes, statuses = _codes_frame( @@ -584,6 +620,7 @@ def _retirement_distribution(raw, ages): slot_codes[name] = codes slot_amounts[name] = values slot_kinds[name] = kinds + slot_status[name] = statuses_out applicable = { "age58_and_over": ("slot1", "slot2"), "under_age58": ("slot1_young", "slot2_young"), @@ -606,10 +643,10 @@ def _retirement_distribution(raw, ages): for n in names ) ) - if all( - slot_codes[n][i] is not None and slot_kinds[n][i] != "missing" - for n in names - ): + # Both axes must be resolved: an unreadable account code could itself be + # a regular IRA, and a declared account whose amount is a "none or niu" + # zero does not observe a zero dollar distribution from that account. + if all(slot_status[n][i] in ("niu_slot", "known_slot") for n in names): matched = [n for n in names if slot_codes[n][i] == REGULAR_IRA_CODE] ira[i] = float(sum(slot_amounts[n][i] for n in matched)) ira_slots.append(len(matched)) @@ -623,8 +660,8 @@ def _retirement_distribution(raw, ages): frame, codes, statuses = _codes_frame( "retirement_distribution_receipt_" + suffix, raw[field], - RECEIPT_CODES, - RECEIPT_CODES, + receipt_codes(field), + receipt_codes(field), ) out = pd.concat([out, frame], axis=1) route_codes[field] = (codes, statuses) @@ -645,7 +682,10 @@ def _retirement_distribution(raw, ages): for i in range(rows) ] frame, codes, statuses = _codes_frame( - "retirement_distribution_receipt", receipt_tokens, RECEIPT_CODES, RECEIPT_CODES + "retirement_distribution_receipt", + receipt_tokens, + receipt_codes("DST_YN"), + receipt_codes("DST_YN"), ) out = pd.concat([out, frame], axis=1) # The two printed DST universes name only the age-58 split. Unlike the four @@ -660,7 +700,19 @@ def _retirement_distribution(raw, ages): universes, list(zip(codes, statuses, strict=True)), net_measure=False, + zero_is_dollars=_zero_is_dollars("DST_VAL1"), ) + # An answered off-route recipiency or off-route dollars contradict the route + # this row is on, and an applicable slot whose declared account carries a + # "none or niu" zero leaves the composition unresolved. Neither may end in a + # known amount, however the route's own receipt literal reads. + for i in range(rows): + if offroute[i] or offroute_receipt[i]: + labels[i] = "contradictory_offroute_evidence" + canonical[i] = np.nan + elif ambiguity[i]: + labels[i] = "unresolved_slot_composition" + canonical[i] = np.nan out["retirement_distribution_source_total"] = totals out["retirement_distribution_reporting_status"] = pd.array(labels, dtype="string") out["retirement_distribution_known_amount"] = canonical @@ -702,7 +754,10 @@ def _net_property(raw, ages): rows = len(ages) universe = _age_universe(ages) frame, codes, statuses = _codes_frame( - "net_property_receipt", raw["RNT_YN"], RECEIPT_CODES, RECEIPT_CODES + "net_property_receipt", + raw["RNT_YN"], + receipt_codes("RNT_YN"), + receipt_codes("RNT_YN"), ) labels, canonical, kinds, sources = _classify( raw["amounts"]["RNT_VAL"], @@ -710,6 +765,7 @@ def _net_property(raw, ages): universe, list(zip(codes, statuses, strict=True)), net_measure=True, + zero_is_dollars=_zero_is_dollars("RNT_VAL"), ) out = frame out["net_property_source_total"] = sources @@ -742,7 +798,10 @@ def _farm(raw, ages): universe_codes = {} for field in ("ERN_YN", "FRMOTR"): frame, codes, _ = _codes_frame( - "farm_" + field.lower(), raw[field], RECEIPT_CODES, RECEIPT_CODES + "farm_" + field.lower(), + raw[field], + receipt_codes(field), + receipt_codes(field), ) out = pd.concat([out, frame], axis=1) universe_codes[field] = codes @@ -756,7 +815,10 @@ def _farm(raw, ages): else: universe.append(None) frame, codes, statuses = _codes_frame( - "farm_receipt", raw["FRSE_YN"], RECEIPT_CODES, RECEIPT_CODES + "farm_receipt", + raw["FRSE_YN"], + receipt_codes("FRSE_YN"), + receipt_codes("FRSE_YN"), ) out = pd.concat([out, frame], axis=1) labels, canonical, kinds, sources = _classify( @@ -765,6 +827,7 @@ def _farm(raw, ages): universe, list(zip(codes, statuses, strict=True)), net_measure=True, + zero_is_dollars=_zero_is_dollars("FRSE_VAL"), ) out["farm_source_total"] = sources out["farm_amount_kind"] = pd.array(kinds, dtype="string") @@ -788,7 +851,10 @@ def _other_income(raw, ages): rows = len(ages) universe = _age_universe(ages) frame, codes, statuses = _codes_frame( - "other_income_receipt", raw["OI_YN"], RECEIPT_CODES, RECEIPT_CODES + "other_income_receipt", + raw["OI_YN"], + receipt_codes("OI_YN"), + receipt_codes("OI_YN"), ) category, category_codes, category_statuses = _codes_frame( "other_income_category", @@ -803,6 +869,7 @@ def _other_income(raw, ages): universe, list(zip(codes, statuses, strict=True)), net_measure=False, + zero_is_dollars=_zero_is_dollars("OI_VAL"), ) out = pd.concat([frame, category], axis=1) out["other_income_source_total"] = sources @@ -813,7 +880,13 @@ def _other_income(raw, ages): routing = [] for i in range(rows): code, status = category_codes[i], category_statuses[i] - if status == "missing": + # OI_OFF is printed for OI_YN = 1, and OI_YN for persons aged 15+, so a + # row outside that universe carries no in-universe category either. + if universe[i] is None: + routing.append("unresolved_reporting_universe_routing") + elif universe[i] is False: + routing.append("outside_reporting_universe_routing") + elif status == "missing": routing.append("missing_category_literal") elif code is None: routing.append("unrecognized_category_literal") @@ -1015,6 +1088,14 @@ def project_income_routing(raw, ages): and np.isin(statuses, [int(c) for c in money.CodebookStatus]).all(), "AMOUNT_ARRAY_CONTRACT:" + name, ) + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES, "OI_OFF"): + tokens = raw.get(name) + require( + type(tokens) is list + and len(tokens) == len(ages) + and all(type(t) is str and len(t) <= TOKEN_MAX_CHARS for t in tokens), + "ROUTING_TOKEN_CONTRACT:" + name, + ) parts = [ _pension_annuity(raw, ages), _retirement_distribution(raw, ages), @@ -1255,6 +1336,7 @@ def qualify_current_asec_income_routing(preparation): "source_admission_issued": False, "release_eligible": False, "projection_sha256": _sha(out.to_json(orient="table").encode()), + "literals_sha256": _sha(literals.to_json(orient="table").encode()), } require( preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, @@ -1269,7 +1351,9 @@ def qualify_current_asec_income_routing(preparation): "FINAL_OWNER", ) require( - _sha(out.to_json(orient="table").encode()) == evidence["projection_sha256"], + _sha(out.to_json(orient="table").encode()) == evidence["projection_sha256"] + and _sha(literals.to_json(orient="table").encode()) + == evidence["literals_sha256"], "FINAL_PROJECTION_CHANGED", ) return CurrentAsecIncomeRoutingValues(out, literals, evidence) diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index 21fb7702b..0cce205a2 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -227,11 +227,16 @@ def test_pension_and_annuity_totals_stay_separate_and_unsplit(tmp_path, monkeypa assert np.isnan(person.loc[105, "pension_annuity_annuity_source_total"]) assert person.loc[105, "pension_annuity_annuity_amount_kind"] == "declared_niu" assert person.loc[105, "pension_annuity_annuity_reporting_status"] == "niu" - # A yes answer with a zero gross amount is ambiguous, not a known zero. + # ANN_VAL is the one entry here whose printed zero is valid dollars (its NIU + # is the separate -1 code), so a yes answer with a zero amount resolves. assert person.loc[107, "pension_annuity_annuity_reporting_status"] == ( - "ambiguous_recipient_zero" + "known_recipient_zero" ) - assert np.isnan(person.loc[107, "pension_annuity_annuity_known_amount"]) + assert person.loc[107, "pension_annuity_annuity_known_amount"] == 0.0 + # PNSN_VAL prints "0 = none or niu", so the same pattern would not resolve + # there; the module reads that distinction from the pinned domains artifact. + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("PNSN_VAL") # A no answer against a positive total is a retained contradiction. assert person.loc[107, "pension_annuity_pension_reporting_status"] == ( "contradictory_no_nonzero" @@ -304,9 +309,13 @@ def test_signed_property_and_farm_totals_keep_losses_and_net_zero_receipt( ) assert person.loc[107, "net_property_source_total"] == 500.0 assert np.isnan(person.loc[107, "net_property_known_amount"]) - # Farm receipt with a net zero is distinct from absence, NIU and unknown. + # Farm receipt with a net zero stays distinct from absence, NIU and missing, + # but FRSE_VAL prints "0 = none or niu" just as the gross entries do, so the + # amount is not completed to a known zero. assert person.loc[105, "farm_reporting_status"] == "receipt_with_net_zero" - assert person.loc[105, "farm_known_amount"] == 0.0 + assert np.isnan(person.loc[105, "farm_known_amount"]) + assert person.loc[105, "farm_source_total"] == 0.0 + assert "receipt_with_net_zero" not in owner.KNOWN_AMOUNT_STATUSES assert person.loc[107, "farm_source_total"] == -9000.0 assert person.loc[107, "farm_reporting_status"] == "known_receipt" assert bool(person.loc[107, "farm_is_net_loss"]) @@ -533,39 +542,83 @@ def _write_member(path, rows): @pytest.mark.parametrize( - "universe,token,kind,net,label", + "universe,token,kind,net,dollars,label", [ - (True, "1", "nonzero", False, "known_receipt"), - (True, "1", "zero", False, "ambiguous_recipient_zero"), - (True, "1", "zero", True, "receipt_with_net_zero"), - (True, "1", "nonzero", True, "known_receipt"), - (True, "2", "zero", False, "known_nonreceipt"), - (True, "2", "nonzero", False, "contradictory_no_nonzero"), - (True, "0", "zero", False, "niu"), - (True, "0", "nonzero", False, "contradictory_niu_nonzero"), - (True, "0", "declared_niu", False, "niu"), - (True, "1", "declared_niu", False, "contradictory_declared_niu_amount"), - (True, "1", "missing", False, "missing_amount"), - (True, "", "zero", False, "missing_receipt_literal"), - (True, "9", "zero", False, "unrecognized_receipt_literal"), - (True, "01", "zero", False, "unrecognized_receipt_literal"), - (False, "0", "zero", False, "outside_reporting_universe"), - (False, "0", "missing", False, "outside_reporting_universe"), - (False, "1", "nonzero", False, "contradictory_outside_reporting_universe"), - (False, "2", "zero", False, "contradictory_outside_reporting_universe"), - (None, "1", "nonzero", False, "unresolved_reporting_universe"), - (None, "0", "zero", True, "unresolved_reporting_universe"), + (True, "1", "nonzero", False, False, "known_receipt"), + (True, "1", "zero", False, False, "ambiguous_recipient_zero"), + # A signed net measure's recipient zero stays distinct, and stays + # unknown: RNT_VAL and FRSE_VAL print the same "0 = none or niu" label + # the gross entries print. + (True, "1", "zero", True, False, "receipt_with_net_zero"), + # Only an entry whose printed zero is valid dollars (ANN_VAL) resolves. + (True, "1", "zero", False, True, "known_recipient_zero"), + (True, "1", "nonzero", True, False, "known_receipt"), + (True, "2", "zero", False, False, "known_nonreceipt"), + (True, "2", "nonzero", False, False, "contradictory_no_nonzero"), + (True, "0", "zero", False, False, "niu"), + (True, "0", "nonzero", False, False, "contradictory_niu_nonzero"), + (True, "0", "declared_niu", False, True, "niu"), + (True, "1", "declared_niu", False, True, "contradictory_declared_niu_amount"), + (True, "1", "missing", False, False, "missing_amount"), + (True, "", "zero", False, False, "missing_receipt_literal"), + (True, "9", "zero", False, False, "unrecognized_receipt_literal"), + (True, "01", "zero", False, False, "unrecognized_receipt_literal"), + (False, "0", "zero", False, False, "outside_reporting_universe"), + (False, "0", "missing", False, False, "outside_reporting_universe"), + ( + False, + "1", + "nonzero", + False, + False, + "contradictory_outside_reporting_universe", + ), + ( + False, + "2", + "zero", + False, + False, + "contradictory_outside_reporting_universe", + ), + (None, "1", "nonzero", False, False, "unresolved_reporting_universe"), + (None, "0", "zero", True, False, "unresolved_reporting_universe"), ], ) -def test_receipt_and_amount_knownness_stay_separate(universe, token, kind, net, label): - receipt = owner.literal_code(token, owner.RECEIPT_CODES, width=1) - assert owner.receipt_status(universe, receipt, kind, net_measure=net) == label +def test_receipt_and_amount_knownness_stay_separate( + universe, token, kind, net, dollars, label +): + receipt = owner.literal_code(token, owner.receipt_codes("PEN_YN"), width=1) + assert ( + owner.receipt_status( + universe, receipt, kind, net_measure=net, zero_is_dollars=dollars + ) + == label + ) # Only three statuses establish a dollar reading; nothing else completes. assert (label in owner.KNOWN_AMOUNT_STATUSES) == ( - label in ("known_receipt", "receipt_with_net_zero", "known_nonreceipt") + label in ("known_receipt", "known_recipient_zero", "known_nonreceipt") ) +def test_only_ann_val_prints_a_zero_that_reads_as_valid_dollars(): + entries = owner.printed_amount_entries() + dollars = { + name + for name, entry in entries.items() + if entry.zero_semantics == owner.DOLLAR_ZERO_SEMANTICS + } + assert dollars == {"ANN_VAL"} + for name in set(owner.AMOUNT_FIELDS) - dollars: + assert ( + entries[name].zero_semantics + == "none_or_niu_not_distinguishable_from_amount_alone" + ) + assert owner._zero_is_dollars("ANN_VAL") + assert not owner._zero_is_dollars("RNT_VAL") + assert not owner._zero_is_dollars("FRSE_VAL") + + @pytest.mark.parametrize( "status,value,kind,dollars", [ @@ -796,8 +849,13 @@ def test_both_distribution_recipiency_literals_are_retained_with_the_route(): assert out.retirement_distribution_receipt_58_literal.tolist() == ["0", "1"] assert out.retirement_distribution_receipt_young_literal.tolist() == ["1", "0"] assert out.retirement_distribution_offroute_receipt.tolist() == [True, True] - # The applicable literal still drives the reporting status. - assert out.retirement_distribution_reporting_status.tolist() == ["niu", "niu"] + # An answered off-route recipiency contradicts the route, so the applicable + # literal cannot resolve the row however it reads. + assert out.retirement_distribution_reporting_status.tolist() == [ + "contradictory_offroute_evidence", + "contradictory_offroute_evidence", + ] + assert out.retirement_distribution_known_amount.isna().all() # A clean pair reports no off-route answer. Here each person answers yes on # their own route with every applicable slot at the printed "none or niu" # zero, which stays ambiguous rather than becoming a known zero. @@ -896,6 +954,201 @@ def test_printed_flag_and_scope_claims_match_the_dictionary_as_printed(): assert owner.ACCOUNT_ENTRIES["DST_SC1_YNG"][4] == ("DST_YN_YNG = 1 and a_age < 58") assert owner.ACCOUNT_ENTRIES["DST_SC2_YNG"][4] == ("DST_VAL_YNG > 0 and a_age < 58") assert owner.RECEIPT_ENTRIES["OI_YN"][5] == "none or niu" + assert owner.receipt_codes("OI_YN")[0] == "none or niu" + for name in ("PEN_YN", "ANN_YN", "DST_YN", "DST_YN_YNG", "RNT_YN", "ERN_YN"): + assert owner.receipt_codes(name)[0] == "niu" + assert owner.receipt_codes("FRSE_YN")[0] == "Niu" assert owner.RECEIPT_ENTRIES["FRSE_YN"][4] == "ERN_YN=1 or FRMOTR=1" for name in ("PEN_YN", "ANN_YN", "RNT_YN", "OI_YN"): assert owner.RECEIPT_ENTRIES[name][4] == "All Persons aged 15+" + + +def _pure_rows(n, ages, **literals): + codebook = owner.money.CodebookStatus + amounts = {name: np.zeros(n) for name in owner.AMOUNT_FIELDS} + statuses = { + name: np.full(n, int(codebook.ZERO_NONE_OR_NIU), "u1") + for name in owner.AMOUNT_FIELDS + } + statuses["ANN_VAL"] = np.full(n, int(codebook.ZERO_DOLLARS_AS_CODED), "u1") + raw = { + "amounts": amounts, + "statuses": statuses, + "allocations": { + name: ([0] * n, ["in_printed_range"] * n) + for name in owner.ALLOCATION_ENTRIES + }, + } + for name in (*owner.RECEIPT_ENTRIES, *owner.ACCOUNT_ENTRIES, "OI_OFF"): + raw[name] = ["0"] * n + raw.update(literals) + return raw, np.asarray(ages, dtype="float64") + + +def _set_amount(raw, field, values): + codebook = owner.money.CodebookStatus + raw["amounts"][field] = np.asarray(values, dtype="float64") + raw["statuses"][field] = np.array( + [ + int(codebook.AMOUNT_NONZERO) + if v + else int( + codebook.ZERO_DOLLARS_AS_CODED + if field == "ANN_VAL" + else codebook.ZERO_NONE_OR_NIU + ) + for v in values + ], + dtype="u1", + ) + + +def test_offroute_distribution_dollars_block_a_known_absence(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["2"], DST_SC1_YNG=["4"]) + _set_amount(raw, "DST_VAL1_YNG", [8000.0]) + out = owner.project_income_routing(raw, ages) + assert out.retirement_distribution_route.iloc[0] == "age58_and_over" + assert bool(out.retirement_distribution_offroute_nonzero.iloc[0]) + # The applicable literals alone would read as a known zero; the retained + # off-route dollars contradict that, so nothing is resolved. + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "contradictory_offroute_evidence" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert out.retirement_distribution_slot1_young_amount.iloc[0] == 8000.0 + + +def test_an_ambiguous_slot_zero_leaves_the_total_and_the_ira_share_unknown(): + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + out = owner.project_income_routing(raw, ages) + # Slot 2 declares a 401k account whose amount is a "none or niu" zero. + assert out.retirement_distribution_slot2_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert bool(out.retirement_distribution_slot_zero_ambiguity.iloc[0]) + assert out.retirement_distribution_source_total.iloc[0] == 5000.0 + assert out.retirement_distribution_reporting_status.iloc[0] == ( + "unresolved_slot_composition" + ) + assert np.isnan(out.retirement_distribution_known_amount.iloc[0]) + assert np.isnan(out.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(out.retirement_distribution_regular_ira_slots.iloc[0]) + # A regular IRA slot whose own amount is that ambiguous zero is not a known + # zero distribution from that account either. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["1"]) + _set_amount(raw, "DST_VAL2", [5000.0]) + ambiguous = owner.project_income_routing(raw, ages) + assert ambiguous.retirement_distribution_slot1_slot_status.iloc[0] == ( + "ambiguous_slot_zero" + ) + assert np.isnan(ambiguous.retirement_distribution_regular_ira_amount.iloc[0]) + # A contradictory NIU slot carrying dollars is likewise unresolved. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + contradictory = owner.project_income_routing(raw, ages) + assert contradictory.retirement_distribution_slot1_slot_status.iloc[0] == ( + "contradictory_niu_slot_amount" + ) + assert np.isnan(contradictory.retirement_distribution_regular_ira_amount.iloc[0]) + assert pd.isna(contradictory.retirement_distribution_regular_ira_slots.iloc[0]) + # Two fully resolved slots do give a known composition. + raw, ages = _pure_rows(1, [70.0], DST_YN=["1"], DST_SC1=["4"], DST_SC2=["0"]) + _set_amount(raw, "DST_VAL1", [5000.0]) + resolved = owner.project_income_routing(raw, ages) + assert resolved.retirement_distribution_reporting_status.iloc[0] == "known_receipt" + assert resolved.retirement_distribution_known_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_amount.iloc[0] == 5000.0 + assert resolved.retirement_distribution_regular_ira_slots.iloc[0] == 1 + + +@pytest.mark.parametrize( + "age,ern,frmotr,routing", + [ + (14, "1", "0", "outside_reporting_universe_routing"), + (40, "", "", "reported_category"), + ], +) +def test_other_income_routing_follows_the_printed_receipt_universe( + age, ern, frmotr, routing +): + raw, ages = _pure_rows( + 1, + [float(age)], + OI_YN=["1"], + OI_OFF=["20"], + ERN_YN=[ern], + FRMOTR=[frmotr], + ) + _set_amount(raw, "OI_VAL", [3000.0]) + out = owner.project_income_routing(raw, ages) + assert out.other_income_routing_status.iloc[0] == routing + assert bool(out.other_income_is_reported_alimony.iloc[0]) == ( + routing == "reported_category" + ) + # The farm universe on the same rows comes from ERN_YN/FRMOTR, not from age. + assert pd.isna(out.farm_source_reporting_universe.iloc[0]) == (ern == "") + + +def test_allocation_origins_do_not_assert_publisher_confirmed_non_allocation(): + raw, ages = _pure_rows(1, [40.0]) + zeroed = owner.project_income_routing(raw, ages) + # Every flag here prints a conditional universe that is not evaluated, so an + # all-zero reading is reported as exactly that, never as "no allocation". + assert zeroed.net_property_allocation_origin.iloc[0] == "published_flags_all_zero" + assert zeroed.pension_annuity_allocation_origin.iloc[0] == ( + "published_flags_all_zero_with_unflagged_fields" + ) + for column in [c for c in zeroed.columns if c.endswith("_allocation_origin")]: + assert "no_allocation" not in zeroed[column].iloc[0] + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTVAL"] = ([4], ["in_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "publisher_allocated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["missing"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "allocation_flag_not_populated" + ) + raw, ages = _pure_rows(1, [40.0]) + raw["allocations"]["I_RNTYN"] = ([None], ["outside_printed_range"]) + assert ( + owner.project_income_routing(raw, ages).net_property_allocation_origin.iloc[0] + == "unresolved_allocation_provenance" + ) + + +@pytest.mark.parametrize("field", ("OI_YN", "DST_SC1", "OI_OFF")) +def test_the_public_projection_validates_its_routing_token_arrays(field): + raw, ages = _pure_rows(2, [40.0, 40.0]) + raw[field] = ["0"] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + raw[field] = ["0", 0] + with pytest.raises(ValueError, match="ROUTING_TOKEN_CONTRACT:" + field): + owner.project_income_routing(raw, ages) + + +def test_the_literal_transport_carries_its_own_digest(tmp_path, monkeypatch): + _, qualified = _qualified(tmp_path, monkeypatch) + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + == qualified.evidence["literals_sha256"] + ) + assert ( + qualified.evidence["literals_sha256"] != qualified.evidence["projection_sha256"] + ) + # A host comparing the documented digests detects a corrupted literal frame. + index = qualified.asec_literals.index[0] + qualified.asec_literals.loc[index, "PNSN_VAL"] = "999999" + assert ( + hashlib.sha256( + qualified.asec_literals.to_json(orient="table").encode() + ).hexdigest() + != qualified.evidence["literals_sha256"] + ) From 84aed94bb3a3666aca377c02ad9fdb1725fcb74c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:14:52 -0400 Subject: [PATCH 153/255] Record integrated survey enrichment and current release boundaries --- docs/us-uk-release-path.md | 61 +++++++++++-- ...-survey-enrichment-integration-20260912.md | 90 +++++++++++++++++++ 2 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 experiments/us-survey-enrichment-integration-20260912.md diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 163ea01f8..9fc4fe14b 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -42,7 +42,7 @@ common source population alone does not guarantee identical estimates. | Stage | US demonstrated result | UK demonstrated result | | --- | --- | --- | -| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF host passes invented cold/replay controls, with native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF and subsequent amount/health hosts pass scoped invented controls; native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | | Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | | Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | | Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | @@ -68,8 +68,11 @@ native-input, owned-file and thread-control checks remained unchanged, with no unexpected access refusals. See the [replay record](../experiments/us-financial19-required-replay-20260912.md). Meanwhile the maintained graph's observer-mutation defect has been fixed and -independently reviewed, and the shared graph changes are being consolidated -against the amended main interface. Those software changes still need their +independently reviewed. The shared graph consolidation merged in +[PR #913](https://github.com/PolicyEngine/microcosm/pull/913) at +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`; the release branch has incorporated +that main revision without changing its existing graph implementation. Those +software changes still need their own native candidate verification; this replay certifies only its frozen financial stage, not the newer source or a release. @@ -82,6 +85,18 @@ runtime or quality of the full native donor. A separate current-source native packet uses a fresh writable store and the actual retained financial run. See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). +That native packet's second attempt closed with an assertion failure after +about 82 minutes, during its financial-stage acceptance and before any PUF +execution. The source and resource pins remained unchanged and no unexpected +access refusal was reported. Source review identified an invalid harness +assumption: the atomic host first materializes CREATE and ALLOCATION, so those +two nodes legitimately hit when its expanded graph runs in the same store. +Requiring every prefix node to miss rejects that valid execution. The refusal +record does not identify its precise assertion frame, so this diagnosis does +not establish completed native financial acceptance. A narrowly corrected v3 +harness and invented assertion controls are in preparation; no retry pass or +native PUF result is claimed. + The maintained financial graph now retains an eighth field: the tax-exempt interest remainder of its existing observed or modeled total. The split remains a modeling assumption. Twenty-five financial/source controls and three full @@ -123,12 +138,29 @@ source/store corruption and late output mutation. Independent review approved the source and mechanical test colocation; 45 cases share one expensive fixture. See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). -An earlier minimal invented fixture reported 61 present input names out of 161, -a diagnostic inventory with unresolved original-channel values. Native coverage -will be measured on the new candidate. Parallel successor work is qualifying -unemployment receipt/amount, three health-cost amounts and current interview -health coverage from the actual survey sources. Reporting universes and missing -values remain explicit; no prior-wage or age-only zero fallback is introduced. +The fixed post-PUF country host now assembles source-qualified unemployment, +three health-cost amounts and nine current health-coverage inputs. It retains +the checked PUF owner and attaches both fragments to the same receiving +population, preserving existing columns, geography, design anchors and mass +ledger. ASEC observations and unresolved values are retained on both clones; +modeled ACS draws are deliberately shared within each original person's clone +pair. Reporting universes remain explicit; no prior-wage or age-only zero +fallback is introduced. The narrower ACS coverage concepts remain nullable +where the source does not establish equivalence. + +The accepted invented fixture reports 74 present input names out of 161, up +from 61, leaving 87 missing. It executed 26 new enrichment nodes over a validated +245-node prefix, then required all 271 replay hits and complete Frame readback. +Its six logical controls passed across five tests in the main attempt and a +corrected focused parent-revocation follow-up, not one all-green full run. +Independent review approved that scoped evidence. After integration with the +eighth interest field and merged shared graph, 80 cheap source, attachment and +health-fragment controls pass. Declaration checks still give 271 nodes: the +eighth interest field adds an output to the existing financial attachment, +not a new fit or graph node. This integration did not repeat the full combined +host or measure fresh native coverage. See +[the combined integration record](../experiments/us-survey-enrichment-integration-20260912.md) +and [source/model decisions](us-current-survey-amount-successor.md). Actual graph tracing also confirms that the legacy full ASEC carry and ACS gap-fill are not executed by this host. Remaining pension, retirement-account, @@ -140,6 +172,17 @@ bridges and overlapping income categories must be resolved explicitly before claiming reconciliation. The broader ACS retirement total is not merely pension plus regular-IRA income. +Two reviewed building blocks for that completion are adopted in the release +branch. The [ACS anchor qualifier](../experiments/us-current-acs-income-anchors-20260912.md) +preserves original INTP/RETP amounts, adjustment factors, original ages and +allocation flags through the retained source owner; its 35 invented-source +controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) +projects joint component draws onto a caller-qualified total while allowing +declared signed components and preserving loss offsets; its 45 numeric controls +pass. These components are not yet invoked by the financial or PUF host. +The donor measurement bridge, scales, named graph reconciliation operation and +dependent model replay remain to be implemented and verified. + The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale and release verification. The existing age-development runner is a separate diff --git a/experiments/us-survey-enrichment-integration-20260912.md b/experiments/us-survey-enrichment-integration-20260912.md new file mode 100644 index 000000000..3fef6cc29 --- /dev/null +++ b/experiments/us-survey-enrichment-integration-20260912.md @@ -0,0 +1,90 @@ +# Survey amount and health integration + +The release branch now includes the source-qualified health fragment from +`7cb1384be82ae4bd559577c5ebe142bf708e8570` and the fixed post-PUF enrichment host +from `99c35b0865e6f6cfe86ede15a10da91360a5e66b`. Their local integration commits +are `f92418a859e576ab7fd40919d3e9d75f41bd1ee4` and +`091261c0e979d0ee191ddbaf08a069b2554d7165`. Both cherry-picks were conflict-free. + +The preceding main merge incorporated shared graph PR #913, main revision +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`, without changing the release branch's +existing source tree. The checked PUF API, eighth financial output, source-string +hashing optimization, ACS anchor qualification and signed reconciliation helper +are preserved. None of those building blocks is interpreted as native acceptance +of this integrated candidate. + +## Accepted combined stage + +The host retains the original checked PUF run and original survey source owners. +It adds four amounts and nine coverage fields to `survey_puf55.receiving`, with +health explicitly dependent on the amount attachment. Source reporting-universe, +knownness and allocation information remain distinct from numeric values. +The [source/model specification](../docs/us-current-survey-amount-successor.md) +records UC classification, PHIP_VAL rather than PHIP_VAL2, the conditional +health-cost chain, deliberate ACS clone-sharing and narrower coverage gaps. + +On the accepted prior implementation, the 18-person invented fixture executed +26 new nodes over a validated 245-node prefix. Required replay hit all 271 nodes; +complete Frame readback preserved parent columns, owners, geography, design +anchors and mass ledger. The diagnostic profile contained 74 of 161 input names, +leaving 87 missing. UC had ten known and eight unresolved values; all 18 had the +three medical-cost amounts and ESI, while narrower Medicaid coverage had eight +known and ten unresolved values. These are invented counts, not population +estimates, native coverage or evidence of statistical quality. + +Acceptance combines the first five passing tests from the main v3 attempt and +the corrected final parent-revocation test from a focused v4 attempt. The v3 +result remains failed: its last test named a nonexistent income column and +stopped before mutation. The corrected test uses the actual +`employment_income_before_lsr` column and proves permanent parent and child +revocation, including after restoring the original value. It passed without +repeating the already-passed required replay and Frame readback. There is no +claim of one all-green full v3 run. Independent source and evidence review +approved this six-control coverage at unchanged production bytes. + +| Preserved local record | SHA256 | +| --- | --- | +| Combined v3 receipt | `19495b1cf3e6aad2f60771c8dce557ccbdc6a78b98c1a1ad453229276ac7e4a4` | +| Combined v3 aggregate acceptance | `07ac957851b19546a5ea1d2123975b9fa616dc8fe94a75e171c93e2379a094da` | +| Focused v4 parent-revocation receipt | `42bbeeae05b01a737d3c50cf3b5569be37cab43d71378fd8e78872965697170c` | + +The v3 acceptance took 1,388.313 seconds with 917,979,136 bytes peak RSS; the +focused v4 control took 624.872 seconds with 885,669,888 bytes peak RSS. Both +retained 857 Python source pins and Torch thread settings of 1/1. These wrappers +are not the strict native pilot audit guard. The source-reviewed execution path +used invented fixtures; receipt flags alone do not prove native admission or +independent network/resource restrictions. Failed attempts remain preserved. + +## Checks after integration + +At integrated source `091261c0e979d0ee191ddbaf08a069b2554d7165`, 80 tests passed +with zero failures, errors or skips in 27.149 seconds wall time; peak RSS was +503,201,792 bytes. The selection includes all 32 amount controls, all 31 health +source/fragment controls, the actual invented UC source-qualification test and +the cheap financial/source controls, including the conserving interest +attachment. The health fragment executes cold and required replay. The expensive +financial/PUF/combined fixture was not selected again. + +All 477 Python files under the shard source directories retained their hashes; +Torch threads remained 1/1. CPU and wall limits were applied. This is bounded +invented integration evidence, not the native admission guard. + +| Integrated record | SHA256 | +| --- | --- | +| Receipt | `b2a764144afe952bba7a0f274cbf4f71ad6f0e46174743610127df7d42e30ba9` | +| JUnit result | `201fff9ac69b9259a9a7c11a291904dac97f082a92364521b622fc80d6805074` | + +Actual declaration functions, invoked with invented descriptive values, confirm +three financial model targets, eight financial output fields and ten financial +extension nodes. Each PUF route retains 55 fits and 55 applications. The amount +fragment has 14 nodes and health has 12; together they compile with the explicit +parent and attachment dependencies. The complete host therefore still declares +271 nodes. The added tax-exempt-interest output changes the existing attachment +and its source identity, not the target chain or node count. No issued parent, +native source or full graph execution is established by this declaration probe. + +Native PUF execution, original-channel tax-detail completion, remaining inputs, +full-candidate coverage, model evaluation, calibration and release verification +remain separate work. The ACS anchor qualifier and signed projection helper are +adopted source/numeric building blocks; the current enrichment host does not yet +invoke them to reconcile property or retirement components. From 52805a4cf21e67369b86f177c14dfadd847c80a9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:17:02 -0400 Subject: [PATCH 154/255] Record corrected native pilot launch after harness verification --- docs/us-uk-release-path.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 9fc4fe14b..a9a5a51a9 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -93,9 +93,12 @@ assumption: the atomic host first materializes CREATE and ALLOCATION, so those two nodes legitimately hit when its expanded graph runs in the same store. Requiring every prefix node to miss rejects that valid execution. The refusal record does not identify its precise assertion frame, so this diagnosis does -not establish completed native financial acceptance. A narrowly corrected v3 -harness and invented assertion controls are in preparation; no retry pass or -native PUF result is claimed. +not establish completed native financial acceptance. The corrected v3 harness +passes two actual invented financial cold/replay controls and thirteen bounded +source and diagnostic controls. After independent review of its exact packet, +one fresh native run started with unchanged calculation source, native inputs +and resource limits. Its result is pending; no retry pass or native PUF result +is claimed. The maintained financial graph now retains an eighth field: the tax-exempt interest remainder of its existing observed or modeled total. The split remains From a585d70eddb4292ede667158c43c787150202e2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:30:22 -0400 Subject: [PATCH 155/255] Register the qualifier in the fail-closed US runtime rosters test_us_spine_blindness globs every us_runtime module and fails closed on an unregistered one, so the new qualifier is classified beside the Social Security qualifier it mirrors and added to the reviewed source-spine provenance owners: it selects the ASEC channel and binds original identities, and routes no PUF-detail imputation. Three of that file's tests remain red for four modules that were already unclassified at the reviewed base d5cbe60b2 - graph_fiscal_dense_calibration, graph_fiscal_measurement, graph_puf55_route_attachment and graph_survey_puf55 - which this lane does not own and has not touched. Also names the printed entry tuples so use sites and the emitted receipt index them by field rather than position, compares the account code domain against the canonical definition instead of a re-typed literal, and corrects the contract note: TAXABLE_PENSION_FRACTION is live in the prepared-ASEC stage, not archived, so a second split attached here would double-count. Co-Authored-By: Claude Opus 5 --- docs/us-current-asec-income-routing-source.md | 11 +- .../current_asec_income_routing_source.py | 130 +++++++++++++----- .../test_us_current_asec_income_routing.py | 31 +++-- .../tests/test_us_spine_blindness.py | 9 ++ 4 files changed, 135 insertions(+), 46 deletions(-) diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index bf364c66b..ed6c59408 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -216,6 +216,11 @@ page, range, universe and values per vintage. The routing code systems `ALLOCATION_ENTRIES`) are not in that artifact and are defined here from the pinned dictionary. +The printed entry tables (`RECEIPT_ENTRIES`, `ACCOUNT_ENTRIES`, +`ALLOCATION_ENTRIES`, `OTHER_INCOME_CATEGORY_ENTRY`) are `NamedTuple`s, so every +use site and the emitted receipt reach their fields by name rather than by +position. + Two adjacent definitions exist in tree and are deliberately **not** imported, so this qualifier keeps the accepted source-only import neighbourhood (`asec_coverage_authentication`, `asec_current_money`, `source_csv_builtin`, @@ -224,7 +229,11 @@ modelled stage machinery: - `retirement_distributions.US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS` — the same eight `DST_SC*`/`DST_VAL*` column names. -- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` — the same category code 20. +- `alimony._ASEC_ALIMONY_OTHER_INCOME_CODE` and + `alimony._ASEC_STRIKE_BENEFITS_OTHER_INCOME_CODE` — the same category codes 20 + and 12. +- `retirement_distributions._VALID_ACCOUNT_CODES` — the same 0-7 account code + domain that `ACCOUNT_CODES` gives printed labels for. `test_routing_code_systems_agree_with_the_existing_domain_constants` asserts both agreements, per `docs/shared-constants.md` rule 5. If a later change makes diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index afb51e24b..9c679a3db 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -27,6 +27,7 @@ from functools import lru_cache from importlib import resources from pathlib import Path +from typing import NamedTuple import numpy as np import pandas as pd @@ -127,11 +128,19 @@ def printed_amount_entries(): # Printed yes/no entries. The zero label is not shared: OI_YN prints # "none or niu" where the others print "niu", so a zero receipt literal does not # carry the same meaning across families. -# name: (length, position, pdf_page, printed_page, universe, zero_label) +class ReceiptEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + zero_label_as_printed: str + + RECEIPT_ENTRIES = { - "PEN_YN": (1, 570, 47, "6C-26", "All Persons aged 15+", "niu"), - "ANN_YN": (1, 444, 44, "6C-23", "All Persons aged 15+", "niu"), - "DST_YN": ( + "PEN_YN": ReceiptEntry(1, 570, 47, "6C-26", "All Persons aged 15+", "niu"), + "ANN_YN": ReceiptEntry(1, 444, 44, "6C-23", "All Persons aged 15+", "niu"), + "DST_YN": ReceiptEntry( 1, 519, 46, @@ -139,12 +148,14 @@ def printed_amount_entries(): "Persons aged 58 and over (a_age \u2265 58)", "niu", ), - "DST_YN_YNG": (1, 520, 46, "6C-25", "Persons under age 58 (a_age < 58)", "niu"), - "RNT_YN": (1, 627, 49, "6C-28", "All Persons aged 15+", "niu"), - "FRSE_YN": (1, 397, 43, "6C-22", "ERN_YN=1 or FRMOTR=1", "Niu"), - "ERN_YN": (1, 381, 43, "6C-22", "WORKYN=1 OR WTEMP=1", "niu"), - "FRMOTR": (1, 389, 43, "6C-22", "ERN_OTR = 1", "niu"), - "OI_YN": (1, 555, 47, "6C-26", "All Persons aged 15+", "none or niu"), + "DST_YN_YNG": ReceiptEntry( + 1, 520, 46, "6C-25", "Persons under age 58 (a_age < 58)", "niu" + ), + "RNT_YN": ReceiptEntry(1, 627, 49, "6C-28", "All Persons aged 15+", "niu"), + "FRSE_YN": ReceiptEntry(1, 397, 43, "6C-22", "ERN_YN=1 or FRMOTR=1", "Niu"), + "ERN_YN": ReceiptEntry(1, 381, 43, "6C-22", "WORKYN=1 OR WTEMP=1", "niu"), + "FRMOTR": ReceiptEntry(1, 389, 43, "6C-22", "ERN_OTR = 1", "niu"), + "OI_YN": ReceiptEntry(1, 555, 47, "6C-26", "All Persons aged 15+", "none or niu"), } RECEIPT_CODE_DOMAIN = (0, 1, 2) @@ -156,7 +167,7 @@ def receipt_codes(field): single shared label would attribute a "respondent reported none" reading to eight fields whose dictionary entry does not support it. """ - return {0: RECEIPT_ENTRIES[field][5], 1: "yes", 2: "no"} + return {0: RECEIPT_ENTRIES[field].zero_label_as_printed, 1: "yes", 2: "no"} # Retirement account identity. Code 4 names a regular IRA; it does not observe @@ -172,12 +183,21 @@ def receipt_codes(field): 7: "Other type of retirement account", } REGULAR_IRA_CODE = 4 -# name: (length, position, pdf_page, printed_page, universe) + + +class AccountEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + + ACCOUNT_ENTRIES = { - "DST_SC1": (1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age \u2265 58"), - "DST_SC1_YNG": (1, 492, 45, "6C-24", "DST_YN_YNG = 1 and a_age < 58"), - "DST_SC2": (1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age \u2265 58"), - "DST_SC2_YNG": (1, 494, 45, "6C-24", "DST_VAL_YNG > 0 and a_age < 58"), + "DST_SC1": AccountEntry(1, 491, 45, "6C-24", "DST_VAL1 > 0 and a_age \u2265 58"), + "DST_SC1_YNG": AccountEntry(1, 492, 45, "6C-24", "DST_YN_YNG = 1 and a_age < 58"), + "DST_SC2": AccountEntry(1, 493, 45, "6C-24", "DST_VAL2 > 0 and a_age \u2265 58"), + "DST_SC2_YNG": AccountEntry(1, 494, 45, "6C-24", "DST_VAL_YNG > 0 and a_age < 58"), } # OI_OFF, verbatim. Code 20 is the reported alimony category. No residual rule @@ -206,7 +226,7 @@ def receipt_codes(field): 20: "alimony", } ALIMONY_CATEGORY_CODE = 20 -OTHER_INCOME_CATEGORY_ENTRY = (2, 547, 47, "6C-26", "OI_YN = 1") +OTHER_INCOME_CATEGORY_ENTRY = AccountEntry(2, 547, 47, "6C-26", "OI_YN = 1") # Published allocation flags. Values 0-9 follow I_ANNVAL; the DST composites # follow I_INTYN (0, 10, 11); I_DSTSC prints its own 0/1/9 set. I_FRMYN prints @@ -218,12 +238,28 @@ def receipt_codes(field): # I_FRMYN's printed Values block is empty; its (0:9) range header is all that is # published, so no code meaning is claimed for it. ALLOCATION_PRINTED_RANGE_CODES = tuple(range(10)) -# name: (length, position, pdf_page, printed_page, universe, codes) + + +class AllocationEntry(NamedTuple): + printed_length: int + printed_position: int + pdf_page_1based: int + printed_page: str + universe_as_printed: str + codes: tuple + + ALLOCATION_ENTRIES = { - "I_ANNVAL": (1, 802, 53, "6C-32", "ANN_YN =1", ALLOCATION_ANNVAL_CODES), - "I_ANNYN": (1, 803, 53, "6C-32", "ANN_YN > 0", ALLOCATION_ANNVAL_CODES), - "I_DSTSC": (1, 821, 55, "6C-34", "DST_YN =1", ALLOCATION_DSTSC_CODES), - "I_DSTSCCOMP": ( + "I_ANNVAL": AllocationEntry( + 1, 802, 53, "6C-32", "ANN_YN =1", ALLOCATION_ANNVAL_CODES + ), + "I_ANNYN": AllocationEntry( + 1, 803, 53, "6C-32", "ANN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_DSTSC": AllocationEntry( + 1, 821, 55, "6C-34", "DST_YN =1", ALLOCATION_DSTSC_CODES + ), + "I_DSTSCCOMP": AllocationEntry( 1, 822, 55, @@ -231,15 +267,33 @@ def receipt_codes(field): "DST_YN = 1 or DST_YNG_YN = 1", ALLOCATION_ANNVAL_CODES, ), - "I_DSTVAL1COMP": (2, 823, 55, "6C-34", "", ALLOCATION_COMPOSITE_CODES), - "I_DSTVAL2COMP": (2, 825, 55, "6C-34", "DST_VAL2> 0", ALLOCATION_COMPOSITE_CODES), - "I_DSTYNCOMP": (2, 827, 55, "6C-34", "DST_YN > 0", ALLOCATION_COMPOSITE_CODES), - "I_ERNYN": (1, 833, 55, "6C-34", "ERN_YN > 0", ALLOCATION_ANNVAL_CODES), - "I_FRMYN": (1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_PRINTED_RANGE_CODES), - "I_OIVAL": (1, 843, 56, "6C-35", "OI_VAL > 0", ALLOCATION_ANNVAL_CODES), - "I_PENYN": (1, 854, 57, "6C-36", "PEN_YN > 0", ALLOCATION_ANNVAL_CODES), - "I_RNTVAL": (1, 861, 57, "6C-36", "RNT_VAL > 0", ALLOCATION_ANNVAL_CODES), - "I_RNTYN": (1, 862, 57, "6C-36", "RNT_YN > 0", ALLOCATION_ANNVAL_CODES), + "I_DSTVAL1COMP": AllocationEntry( + 2, 823, 55, "6C-34", "", ALLOCATION_COMPOSITE_CODES + ), + "I_DSTVAL2COMP": AllocationEntry( + 2, 825, 55, "6C-34", "DST_VAL2> 0", ALLOCATION_COMPOSITE_CODES + ), + "I_DSTYNCOMP": AllocationEntry( + 2, 827, 55, "6C-34", "DST_YN > 0", ALLOCATION_COMPOSITE_CODES + ), + "I_ERNYN": AllocationEntry( + 1, 833, 55, "6C-34", "ERN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_FRMYN": AllocationEntry( + 1, 837, 55, "6C-34", "FRMOTR > 0", ALLOCATION_PRINTED_RANGE_CODES + ), + "I_OIVAL": AllocationEntry( + 1, 843, 56, "6C-35", "OI_VAL > 0", ALLOCATION_ANNVAL_CODES + ), + "I_PENYN": AllocationEntry( + 1, 854, 57, "6C-36", "PEN_YN > 0", ALLOCATION_ANNVAL_CODES + ), + "I_RNTVAL": AllocationEntry( + 1, 861, 57, "6C-36", "RNT_VAL > 0", ALLOCATION_ANNVAL_CODES + ), + "I_RNTYN": AllocationEntry( + 1, 862, 57, "6C-36", "RNT_YN > 0", ALLOCATION_ANNVAL_CODES + ), } # Which printed field each published flag names, and the fields for which the @@ -861,7 +915,7 @@ def _other_income(raw, ages): raw["OI_OFF"], OTHER_INCOME_CATEGORIES, OTHER_INCOME_CATEGORIES, - width=OTHER_INCOME_CATEGORY_ENTRY[0], + width=OTHER_INCOME_CATEGORY_ENTRY.printed_length, ) labels, canonical, kinds, sources = _classify( raw["amounts"]["OI_VAL"], @@ -1113,7 +1167,8 @@ def _raw_arrays(ordered, ready, positions): allocations = {} for name, entry in ALLOCATION_ENTRIES.items(): pairs = [ - literal_code(t, entry[5], width=entry[0]) for t in ordered[name].tolist() + literal_code(t, entry.codes, width=entry.printed_length) + for t in ordered[name].tolist() ] allocations[name] = ( [c for c, _ in pairs], @@ -1263,11 +1318,12 @@ def qualify_current_asec_income_routing(preparation): } for name, e in printed_amount_entries().items() }, - "receipt_entries": {k: list(v) for k, v in RECEIPT_ENTRIES.items()}, - "account_entries": {k: list(v) for k, v in ACCOUNT_ENTRIES.items()}, - "other_income_category_entry": list(OTHER_INCOME_CATEGORY_ENTRY), + "receipt_entries": {k: v._asdict() for k, v in RECEIPT_ENTRIES.items()}, + "account_entries": {k: v._asdict() for k, v in ACCOUNT_ENTRIES.items()}, + "other_income_category_entry": OTHER_INCOME_CATEGORY_ENTRY._asdict(), "allocation_entries": { - k: [*v[:5], list(v[5])] for k, v in ALLOCATION_ENTRIES.items() + k: {**v._asdict(), "codes": list(v.codes)} + for k, v in ALLOCATION_ENTRIES.items() }, "fields_without_published_allocation_flag": list(UNFLAGGED_FIELDS), "published_allocation_flag_by_field": dict( diff --git a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py index 0cce205a2..73b68e66e 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_income_routing.py +++ b/packages/microcosm-build/tests/test_us_current_asec_income_routing.py @@ -505,6 +505,7 @@ def test_the_qualifier_imports_only_the_accepted_source_neighbourhood(): "functools", "importlib", "pathlib", + "typing", "numpy", "pandas", } @@ -809,8 +810,16 @@ def test_routing_code_systems_agree_with_the_existing_domain_constants(): assert projected == slots assert owner.ALIMONY_CATEGORY_CODE == alimony._ASEC_ALIMONY_OTHER_INCOME_CODE assert owner.OTHER_INCOME_CATEGORIES[owner.ALIMONY_CATEGORY_CODE] == "alimony" - assert set(owner.ACCOUNT_CODES) == set(range(8)) + # Compare against the canonical code domain, not a re-typed literal. + assert set(owner.ACCOUNT_CODES) == set(distributions._VALID_ACCOUNT_CODES) + assert set(distributions._EXPECTED_OUTPUT_BY_ACCOUNT_CODE) <= set( + owner.ACCOUNT_CODES + ) assert owner.ACCOUNT_CODES[owner.REGULAR_IRA_CODE] == "Regular IRA" + assert ( + owner.OTHER_INCOME_CATEGORIES[alimony._ASEC_STRIKE_BENEFITS_OTHER_INCOME_CODE] + == "strike benefits" + ) # Published flags and unflagged fields are disjoint and jointly complete. named = set(owner.PUBLISHED_ALLOCATION_FLAG_BY_FIELD) assert named.isdisjoint(owner.UNFLAGGED_FIELDS) @@ -926,13 +935,15 @@ def test_other_income_routing_never_invents_a_reported_category( def test_printed_flag_and_scope_claims_match_the_dictionary_as_printed(): # I_FRMYN prints an empty Values block, so only its (0:9) range header is # published; no code meaning is claimed for it. - assert owner.ALLOCATION_ENTRIES["I_FRMYN"][5] == tuple(range(10)) - assert owner.ALLOCATION_ENTRIES["I_FRMYN"][5] is not owner.ALLOCATION_ANNVAL_CODES - assert owner.ALLOCATION_ENTRIES["I_DSTSC"][5] == (0, 1, 9) + assert owner.ALLOCATION_ENTRIES["I_FRMYN"].codes == tuple(range(10)) + assert ( + owner.ALLOCATION_ENTRIES["I_FRMYN"].codes is not owner.ALLOCATION_ANNVAL_CODES + ) + assert owner.ALLOCATION_ENTRIES["I_DSTSC"].codes == (0, 1, 9) for name in ("I_DSTVAL1COMP", "I_DSTVAL2COMP", "I_DSTYNCOMP"): - assert owner.ALLOCATION_ENTRIES[name][5] == (0, 10, 11) + assert owner.ALLOCATION_ENTRIES[name].codes == (0, 10, 11) # I_DSTVAL1COMP's printed universe line is empty in the dictionary. - assert owner.ALLOCATION_ENTRIES["I_DSTVAL1COMP"][4] == "" + assert owner.ALLOCATION_ENTRIES["I_DSTVAL1COMP"].universe_as_printed == "" # The DST_SC(2) notation and the empty Values block stay recorded as # printed ambiguities rather than resolved silently. assert set(owner.AMBIGUOUS_FLAG_COVERAGE) == { @@ -958,9 +969,13 @@ def test_printed_flag_and_scope_claims_match_the_dictionary_as_printed(): for name in ("PEN_YN", "ANN_YN", "DST_YN", "DST_YN_YNG", "RNT_YN", "ERN_YN"): assert owner.receipt_codes(name)[0] == "niu" assert owner.receipt_codes("FRSE_YN")[0] == "Niu" - assert owner.RECEIPT_ENTRIES["FRSE_YN"][4] == "ERN_YN=1 or FRMOTR=1" + assert owner.RECEIPT_ENTRIES["FRSE_YN"].universe_as_printed == ( + "ERN_YN=1 or FRMOTR=1" + ) for name in ("PEN_YN", "ANN_YN", "RNT_YN", "OI_YN"): - assert owner.RECEIPT_ENTRIES[name][4] == "All Persons aged 15+" + assert owner.RECEIPT_ENTRIES[name].universe_as_printed == ( + "All Persons aged 15+" + ) def _pure_rows(n, ages, **literals): diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 3c7efd00e..06a52e9c8 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -128,6 +128,12 @@ # select or validate origin rows and to bind original identities; # none routes a PUF-detail imputation by spine. "current_asec_demographics.py", # ASEC rows -> sex/household state. + # Source-owned pension/annuity, retirement distribution, net property, + # farm and other-income reporting and routing per row. Selects the ASEC + # channel and binds original identities exactly as the Social Security + # qualifier beside it does; it routes no PUF-detail imputation and + # models nothing. + "current_asec_income_routing_source.py", "current_social_security_source.py", # Source-owned SS totals per row. "current_survey_geography.py", # Origin roster and draw keys. "current_survey_predictors.py", # Source-qualified predictor prep. @@ -438,6 +444,9 @@ "cps_carried_current.py", # Current survey source projection for ASEC sex and household state "current_asec_demographics.py", + # Current ASEC pension/annuity, retirement distribution, net property, + # farm and other-income reporting and routing projection + "current_asec_income_routing_source.py", # Current survey Social Security totals, reason literals and unknown shares "current_social_security_source.py", # Qualified observed geography and stable household draw keys for both From 4d4b9cac1b8e2369d8f3539d953b9bb512e8830d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:33:04 -0400 Subject: [PATCH 156/255] Record the adversarial review round in the lane journal Co-Authored-By: Claude Opus 5 --- PROGRESS-current-asec-income-routing.md | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/PROGRESS-current-asec-income-routing.md b/PROGRESS-current-asec-income-routing.md index 4ea5f5494..9f76d23d9 100644 --- a/PROGRESS-current-asec-income-routing.md +++ b/PROGRESS-current-asec-income-routing.md @@ -49,6 +49,48 @@ issuer, no engine execution, no native cell writes, no release claim. resolving an unknown slot account into a known non-IRA zero each turn the suite red; reverting each returns it green. +## Second review round: corrections after adversarial review + +An independent five-lens adversarial review of the finished lane (source +fidelity, projection semantics, owner pattern, tests, repository fit) found a +further set of defects, every one of which is fixed on this branch: + +1. **Fail-closed roster (CI-breaking).** `test_us_spine_blindness.py` globs + every `us_runtime` module and fails closed on an unregistered one. The + qualifier is now classified there and listed among the reviewed source-spine + provenance owners. Three of that file's tests remain red for four modules + already unclassified at the reviewed base `d5cbe60b2` — + `graph_fiscal_dense_calibration.py`, `graph_fiscal_measurement.py`, + `graph_puf55_route_attachment.py`, `graph_survey_puf55.py` — which this lane + neither owns nor touched. **Root should route those to their owners.** +2. **Recipient zeros were completed on an unprinted rationale.** `RNT_VAL` and + `FRSE_VAL` print the same `0 = none or niu` label the gross entries print, so + a signed net measure's recipient zero is no longer a known amount. Only + `ANN_VAL`, whose printed zero is `valid_zero_dollars`, resolves; that test is + read from the pinned domains artifact rather than decided in code. +3. **Off-route distribution evidence yielded an affirmative zero.** Off-route + dollars or an answered off-route recipiency now contradict the row, and an + applicable slot declaring an account whose amount is a "none or niu" zero + leaves both the total and the regular-IRA share unresolved. +4. **Other-income routing ignored the printed receipt universe**, so an + out-of-universe row read as reported alimony. +5. **Allocation origins asserted publisher-confirmed non-allocation** from flags + whose printed universes are conditional and unevaluated. +6. **The printed zero receipt label was collapsed** to one shared `niu_or_none` + across nine entries that print three different labels. +7. **`asec_literals` escaped with no digest**, outside the final identity check. +8. **`I_FRMYN` prints an empty `Values:` block**; its codes were described as + following `I_ANNVAL`. **`FARM_AMOUNT_SCOPE` dropped its composite clause.** + The claim that every other family prints a 15+ floor was false — the farm + family prints none. +9. **The doc called `TAXABLE_PENSION_FRACTION` archived**; it is live in + `cps_carried_current.py` under the prepared-ASEC stage, so a second split + attached here would double-count. +10. Printed entry tables are `NamedTuple`s now, the account code domain is + compared against `retirement_distributions._VALID_ACCOUNT_CODES` rather than + a re-typed literal, and `project_income_routing` validates its routing token + arrays. + ## Corrections made after independent review of the first draft 1. The money owner normalizes `ANN_VAL`'s printed `-1` to a stored zero and From 4b9ea6176a7bd02efcde0add9eb9e32787239dd5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:23:13 -0400 Subject: [PATCH 157/255] Record the lane's scope, evidence and residual risks Adds the local experiment for this integration and brings the lane's PROGRESS section up to the commits. The prior journals on both merged branches are left untouched as history. The record states plainly what this branch is not: it is the shared solver seam only. us_runtime/graph_fiscal_dense_calibration.py still calls calibrate without an observer, so nothing here makes a real US calibration emit a snapshot, and no observer registry, Node.params callback or replay-reruns-the-optimizer claim was invented to pretend otherwise. Residual risks recorded: cadence cost on a real US run is unmeasured here (the only overhead evidence in the tree is #908's synthetic benchmark); the grouped `selection` labels on in-loop rows are an integration judgement inside what checklist section 2 sanctions rather than a checklist requirement; host wiring is the gap between this and any dashboard; the recomputed identities are only valid for the locked dependency versions they were taken under; and the engine-gated identity proofs were exercised with the engines present, so a lane without the US extra skips rather than checks them. Co-Authored-By: Claude Opus 5 --- PROGRESS.md | 42 ++++-- ...-target-snapshot-integration-20260912.json | 90 +++++++++++++ ...ed-target-snapshot-integration-20260912.md | 124 ++++++++++++++++++ 3 files changed, 247 insertions(+), 9 deletions(-) create mode 100644 experiments/grouped-target-snapshot-integration-20260912.json create mode 100644 experiments/grouped-target-snapshot-integration-20260912.md diff --git a/PROGRESS.md b/PROGRESS.md index 5c1a7e372..b85581777 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -14,9 +14,17 @@ written; see "Root journals are history, not state" in `CLAUDE.md`. ## State -Merge resolved. Neither solver path was replaced. Grouped instrumentation, -identity recomputation and the cross-product tests follow in later commits on -this branch. +Complete and local. Four commits: the merge, the grouped instrumentation, the +identity recomputation, and the cross-product controls. Nothing pushed, no PR, +no release action. Root reviews and integrates. + +Scope is the shared solver seam only. `graph_fiscal_dense_calibration.py` still +calls `calibrate` without an observer, so this branch emits no snapshot for the +real US fiscal path; that host wiring is a separate step, and no observer +registry, `Node.params` callback or replay-reruns-the-optimizer claim was +invented here. See +[the lane experiment](experiments/grouped-target-snapshot-integration-20260912.md) +for scope, evidence and residual risks. ## Done @@ -44,14 +52,30 @@ this branch. tree, and neither describes the merged one. They are recomputed against the final merged sources in a later commit on this branch. +## Done (continued) + +- Instrumented `_optimize_grouped`: the grouped early return in `_optimize` + bypassed every hook S added, so grouped runs emitted only a closing snapshot. + The in-loop emission sits after the progress callback and before `backward()` + and reads the exact float32 estimate tensor the epoch's loss was computed + from. Returned weights, trajectory and RNG state are bit-identical with the + observer on and off, and no evaluation is added. +- Separated retain-best detection from the receipt for grouped runs, so a later + change that populated a grouped receipt cannot make the reused final emitter + claim a retained best. The receipt stays empty. +- Recomputed six source-derived identity artifacts against this checkout (the + five conflicted ones plus the country-bundle digests, which were not + conflicted because only G had touched them but which move for the same + reason). No value equals either branch's. The simulate and fit.qrf pins were + deliberately left alone. +- Added the checklist's cross product to the three existing grouped test files + rather than a new one, so every case reuses fixtures already there. + ## Next -1. Instrument `_optimize_grouped` so grouped runs emit snapshots (the grouped - early return in `_optimize` otherwise bypasses every hook S added). -2. Recompute the five source-derived identities against the final merged - checkout and the isolated interpreter. -3. Add the bounded cross-product tests from checklist section 7. -4. Record the lane's scope and evidence under `experiments/`. +Root's independent review and integration. Open follow-ups, none owned here: +the host observer seam for the US fiscal dense calibration path, a country-scale +cadence choice backed by a real measurement, and the dashboard consumer. --- diff --git a/experiments/grouped-target-snapshot-integration-20260912.json b/experiments/grouped-target-snapshot-integration-20260912.json new file mode 100644 index 000000000..5f7c0e72b --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.json @@ -0,0 +1,90 @@ +{ + "lane": "grouped-target-snapshot-integration-20260912", + "date": "2026-09-12", + "scope": "shared calibration solver seam only; no host wiring, no native build, no release action, nothing pushed", + "base": { + "origin_main": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b", + "moved_since_pin": false, + "note": "origin/main was fetched and is byte-identical to the shared base both reviewed heads were cut from, so no main drift had to be preserved" + }, + "integrated_heads": { + "grouped_us_solver": "536f1ceefcdafda3cc619c14b4da18e012a7be57", + "target_snapshots_pr914": "b43369dc49e175803f62020cc1e72fa53926aed8", + "merge_base_of_the_two": "116d46ee9dc2aafdc68259b7c06e4c3462522e8b" + }, + "pinned_source_hashes_verified_before_merge": { + "grouped/solve.py": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "grouped/calibrate__init__.py": "69ac527f4732cdb67c4807bcb8ee9faec56174621f663ec491140dfc324a705f", + "snapshots/solve.py": "f43be0efecdd21b3f5caa9789edc8463a127423c665523ac7f0c8bc48f4b4eed", + "snapshots/calibrate__init__.py": "6b66552827852dd16a36512bdef76c28e47842d9a6b0b59d176a4a18a71f48ae", + "all_four_matched_the_pin_file": true + }, + "commits": [ + {"sha": "fff20fca56f41060db55742b884a6df2693e139f", "what": "merge of the two reviewed heads; neither solver replaced"}, + {"sha": "e64d5a8850ab301b7e819a6b972f235c51688568", "what": "grouped Adam snapshot emission and retain-best separation"}, + {"sha": "7ac2b864a93977ffa98fa86674c09e04ea6b20b5", "what": "source-derived identity recomputation"}, + {"sha": "2f62bb8448010bcc9b9e1419ea24aafcb85917d1", "what": "grouped x snapshot cross-product controls"} + ], + "final_source_hashes": { + "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py": "e031ab5823d0f7bff43642e6ebb160ad867ec387e88658534ccd1e6ef62b273d", + "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py": "1841264fe0c09a96ccf6aacf467cfa201de92ee67185b00cd378456fc7631a78", + "packages/microcosm-calibrate/src/microcosm/calibrate/target_snapshots.py": "a86958e3e94ca2454d997567883e2223e832f7edb5d994f689f0a64ddad1a88a", + "packages/microcosm-calibrate/src/microcosm/calibrate/group_bounds.py": "19a440a31d164832f61c84b2c1277a7a8798681f6830700bf03511dce7524557" + }, + "identity_recomputation": { + "rule": "recomputed against the merged checkout and the isolated interpreter; no value equals either branch's, so none is an ours/theirs choice", + "values": [ + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_protocol", "grouped": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "snapshots": "d052fd875d014f6fa28d6f5b052735912f464d4efb2c98ecf373346b8cc454d3", "recomputed": "57e21b6522cfccc9b2d5e725503d9d39255abe5d5c9b391563d08ee664dea988"}, + {"artifact": "spec_engine/inventory_coverage.py EXPECTED_HASHES.seed_map", "grouped": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", "snapshots": "d5a9694aac3e351f987b4f08ef8add0c25fcd3390d6bb2a851246da03adde516", "recomputed": "4ae2a0634e3a92061beba5bf5059db7c30b4ff6f1898e639697d246cb7e73878"}, + {"artifact": "us resolved spec_sha256 (us-f0-coverage.json and test_us_multispine_pool_tool.py)", "grouped": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405", "snapshots": "ff2c97039853e18bfc0eb5388eb08c688cae33068b742f232d4c06a18b8efd88", "recomputed": "15e61194b4077de58272b14d83af95f7f54427aeab251429ada3cda01d93cbc8"}, + {"artifact": "test_spec_engine_loader.py minimal-loader golden", "grouped": "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3", "snapshots": "8a24089831b3a8f0cfe1be094769af7a30b7b74dbc7d4009e26e9f56444cba4c", "recomputed": "3868d14d5df04b5ab10ed4f0a56c2d9652c15c219840ed88d437f847fa481de7"}, + {"artifact": "parity kernels/calibrate/pins.json implementation_hash", "grouped": "59ca046b7c26bd872e56bd4cf3dd6170748fbe04e5a5da7b711e94aea8d3cc74", "snapshots": "46b52a20a7a5fac27d354c735a09ae5ba171d62bd9010fbc3079b06614cc2a32", "recomputed": "f2dd92eaaebba782e100cb92d90524d3ae6e2bab9ffedbacbe8df0257d5cae3d"}, + {"artifact": "parity kernels/calibrate/pins.json node_key", "grouped": "faaccb273d0aedeb1cca2e1b52c3e27678867512493f3c0379f596890c258cd1", "snapshots": "970720b71cb2579a5e2f714ca47deabc28c9249a83007ebfda83e2e7446d9311", "recomputed": "ae92142a3bf0934ac6b6751aca904c1847cdff3ca8533af83141d2cbf1ccf102"}, + {"artifact": "test_spec_engine_country_bundles.py am", "grouped": "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c", "snapshots": "untouched-by-S", "recomputed": "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae"}, + {"artifact": "test_spec_engine_country_bundles.py be", "grouped": "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", "snapshots": "untouched-by-S", "recomputed": "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1"}, + {"artifact": "test_spec_engine_country_bundles.py uk", "grouped": "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", "snapshots": "untouched-by-S", "recomputed": "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724"} + ], + "deliberately_not_repinned": [ + "packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json (G's value preserved; it hashes microcosm/frame/bundle.py, not calibrate.solve)", + "packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/ (neither head changed microcosm/fit/qrf.py)", + "parity direct.csv outputs (the calibrate kernel's outputs did not move, only its implementation hash)" + ], + "tooling": { + "us-f0-coverage.json": "tools/spec_engine_coverage.py (then --check exits 0)", + "calibrate parity pin": "tools/graph_parity_repin.py calibrate", + "loader golden and bundle digests": "taken from the observed value the failing identity assertion reports" + } + }, + "diff_summary": { + "vs_grouped_head": {"files": 25, "insertions": 4287, "deletions": 20, "note": "every deletion is an identity pin line, an extended import, an extended docstring, or a line S replaced; nothing from the grouped head was lost"}, + "vs_snapshot_head": {"files": 432, "insertions": 148099, "deletions": 3251, "note": "the grouped branch's whole delta"} + }, + "controls": { + "microcosm-calibrate shard": {"passed": 443, "failed": 0}, + "microcosm-graph shard + test_us_multispine_pool_tool + 4 spec-engine identity files": {"passed": 775, "failed": 0}, + "per_file": { + "test_target_snapshots.py": {"cases": 52, "added": 0}, + "test_grouped_fixed_support.py": {"cases": 42, "added": 18}, + "test_group_bounds.py": {"cases": 82, "added": 3}, + "test_solve_us_launch_integration.py": {"cases": 12, "added": 4} + }, + "tools/spec_engine_coverage.py --check": "exit 0", + "tools/ci_test_groups.py --verify": "verification=ok; no test file changed lane and none is [defaulted]", + "ruff": "format clean, check clean on packages/microcosm-calibrate" + }, + "environment": { + "interpreter": "borrowed root .venv/bin/python 3.14.4 run with -I -B -S and an explicit sys.path", + "sys_path_order": "this worktree's packages/*/src first, then the repo root, then the borrowed site-packages", + "imports_asserted_in_worktree": ["microcosm.frame", "microcosm.calibrate", "microcosm.build", "microcosm.fit", "microcosm.graph"], + "threads": "OMP/OPENBLAS/MKL/NUMEXPR/VECLIB all 1", + "pytest": "PYTEST_DISABLE_PLUGIN_AUTOLOAD=1, -p no:randomly", + "locked_versions": {"numpy": "2.4.6", "pandas": "3.0.3", "scipy": "1.17.1", "torch": "2.12.0", "platform": "arm64/darwin/py3.14"}, + "nothing_installed": true + }, + "not_done_on_purpose": [ + "host observer wiring in us_runtime/graph_fiscal_dense_calibration.py", + "any global observer registry, Node.params callback or filesystem path in graph params", + "any claim that a required replay reruns the optimizer", + "native build, PUF fixture, country build, dependency install, push, PR, publication or merge" + ] +} diff --git a/experiments/grouped-target-snapshot-integration-20260912.md b/experiments/grouped-target-snapshot-integration-20260912.md new file mode 100644 index 000000000..29e4793df --- /dev/null +++ b/experiments/grouped-target-snapshot-integration-20260912.md @@ -0,0 +1,124 @@ +# Grouped solver x calibration target snapshot integration — 2026-09-12 + +Scope, evidence and residual risks for branch +`grouped-target-snapshot-integration-20260912`. Source-only; no native build, +no PUF fixture, no country build, no release action, nothing pushed. + +## What this is + +The bounded integration of the reviewed calibration target snapshot observer +(microcosm#908, head `b43369dc49e175803f62020cc1e72fa53926aed8`) with the +reviewed US grouped/fixed-zero Adam solver (head +`536f1ceefcdafda3cc619c14b4da18e012a7be57`), on a fresh `origin/main` +(`116d46ee9dc2aafdc68259b7c06e4c3462522e8b`). Main had not moved from the +shared base, so no main drift had to be preserved. + +Executed against the read-only design checklist +`grouped-snapshot-integration-review.md` and its pins. All four pinned source +hashes on both heads matched the pin file byte for byte before the merge. + +## What it is not + +- **Not** host wiring. `us_runtime/graph_fiscal_dense_calibration.py` still + calls `calibrate` without an observer, so this branch produces no snapshot + file for the actual US fiscal path. Wiring it is a separate, explicitly + host-owned step. No global observer registry, no callback or filesystem path + in graph `Node.params`, and no claim that any replay reruns the optimizer. +- **Not** a dashboard, a staging upload, or a cadence/performance acceptance. +- **Not** a release, a publication or a promotion of any artifact. + +## The one real code change + +A clean textual merge leaves grouped runs uninstrumented: `_optimize` returns +early into `_optimize_grouped` before reaching any snapshot hook S added to the +ordinary Adam loop, so a caller passing `target_snapshots` to a grouped +`calibrate` received exactly one closing snapshot and no trajectory. + +1. `_optimize` forwards its bound observer into `_optimize_grouped` + (`snapshots=snapshots`), and `_optimize_grouped` takes a default-`None` + `snapshots` argument. +2. `_optimize_grouped` emits inside the epoch loop, after the existing progress + callback and before `backward()`, from the exact float32 `estimate` tensor + the epoch's loss was computed from: `iterate=current`, `precision=float32`, + `loss=trajectory[epoch]`, `best_retained={available: False, epoch: None, + loss: None}`. +3. `calibrate` reads retain-best off the solver actually used rather than + inferring it from an empty `iterate_selection_receipt`, and stamps the + grouped closing snapshot at `epoch == epochs`. The receipt itself stays + empty, exactly as before. +4. Grouped snapshots carry three bounded aggregate scalars in `selection`: + `rule: "closing_state"`, `constraint_mode: "grouped_upper_bounds"`, + `grouped_preserve_zeros: `. + +Nothing else in either solver moved. No optimizer or matrix dtype, active mask, +zero template, L2 denominator, log/exp operation, optimizer construction, seed, +projection call, accepted-buffer assignment or `log_w` overwrite changed; there +is still exactly one `log_w` overwrite per accepted projection, the final no-op +projection and accepted-byte equality are unchanged, and the stored-result +admission (weight bytes, fixed-zero mask, bounds, ordered household IDs) still +runs after the last sink. + +### Why the `selection` labels + +Grouped Adam is a closing-state algorithm; it never runs the retain-best rule. +Without a label, its `best_retained.available: False` is indistinguishable from +an ordinary run whose retain-best rule happened to be off, and a consumer would +read the two the same way. Checklist section 2 sanctions exactly this — +"Optional grouped labels must be bounded aggregate scalars, e.g. constraint +mode and whether fixed-zero support is enabled". The labels mirror the run's +own `options["iterate_selection"]` and reuse the receipt's `rule` vocabulary; +they are three JSON scalars carrying no record-level content. This is the one +place the integration goes beyond the checklist's literal field list for +in-loop snapshots, and it is a single expression to remove if root disagrees. + +## Evidence + +- Grouped runs are bit-identical with the observer on and off across + dense x CSR, positive x fixed-zero support, and plain / warm-started / L2: + returned weights, loss trajectory, options, closing loss, stored frame + weights, ordered IDs, fixed-zero bytes, and the full sequence of private + post-projection proof payloads. +- Exactly 20 `_apply_constraint` and 3 `problem.estimates` calls with the + observer on and off on the 20-epoch fixture: cadence buys observation, not + evaluation. Torch and numpy RNG state are identical after each pair of runs. +- Each in-loop row carries the exact float32 tensor the epoch's loss was + computed from, verified against a wrapped `_apply_constraint`, and differs on + all 20 epochs from a recomputation on the accepted vector the private + observer reports next. +- The closing row equals the reused float64 final diagnostics estimates. +- A sink that ruins every delivered payload changes neither the weights nor the + next payload. A sink that mutates live frame IDs at the closing emission + still trips the ordered-ID admission guard. +- Zero epochs through the internal grouped seam emits nothing and evaluates + nothing. +- The deliberately oscillating example still selects grouped closing output + while ordinary Adam selects an earlier best, and each solver's closing + snapshot now says which rule produced it. + +Identity recomputation. Editing `solve.py` — an attested kernel module — moves +the seed protocol digest, every resolved-spec digest that folds it in, and the +calibrate kernel's implementation hash. Each value below was recomputed against +this checkout with the isolated Python 3.14 interpreter; none equals either +branch's value, so none is an ours/theirs choice. The simulate parity pin and +the fit.qrf pin were deliberately left alone: neither hashes +`microcosm.calibrate.solve`, and both are byte-unchanged from the grouped head. + +## Residual risks + +1. **Cadence cost on a real US run is unmeasured here.** The only overhead + evidence in the tree is #908's synthetic benchmark; this branch adds no + native measurement. A country-scale cadence choice is still unowned. +2. **The `selection` labels are an integration judgement**, not a checklist + requirement, on the in-loop rows specifically. See above. +3. **Host wiring remains the gap between this and any dashboard.** Green tests + here say the solver seam is correct; they say nothing about whether any real + US calibration emits anything, because none does yet. +4. **The identity recomputation is only as good as its interpreter.** Digests + fold locked dependency versions; a lane on different locked versions will + compute different parity keys. The values here were taken under + numpy 2.4.6 / pandas 3.0.3 / scipy 1.17.1 / torch 2.12.0 on + arm64/darwin/py3.14, which is what the calibrate parity pin records. +5. **The engine-gated identity tests were exercised with the engines present** + in the borrowed environment. A lane without the US extra will skip + `test_spec_engine_loader.py`'s golden and the country-bundle proofs rather + than check them. From 13791d75a7fce94d12697b8d5ce9bdf46f83ebab Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:35:27 -0400 Subject: [PATCH 158/255] Record final fiscal snapshot evidence and historicize grouped lane notes --- PROGRESS.md | 6 ++++++ docs/us-uk-release-path.md | 9 +++++++++ experiments/fiscal-target-snapshot-host-20260912.md | 8 ++++++++ .../grouped-target-snapshot-integration-20260912.md | 6 ++++++ 4 files changed, 29 insertions(+) diff --git a/PROGRESS.md b/PROGRESS.md index b85581777..2af0074f8 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,5 +1,11 @@ # Grouped solver x calibration target snapshots integration - 2026-09-12 +Historical note, 12 September 2026: this grouped-lane journal was subsequently +integrated with PR #914's corrected iteration identities and the actual fiscal +host observer. Its 443-test evidence and statements that host wiring is absent +describe that earlier lane. Current source and acceptance are recorded in +[the fiscal-host evidence](experiments/fiscal-target-snapshot-host-20260912.md). + Lane: `microcosm-grouped-target-snapshot-integration-20260912`, branch `grouped-target-snapshot-integration-20260912`. Base: fresh `origin/main` `116d46ee9dc2aafdc68259b7c06e4c3462522e8b` (unmoved; it is exactly the shared diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index a9a5a51a9..25c41af84 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -131,6 +131,15 @@ their source: the complete-population host must authenticate the original sampling references and retained parent before and after use. Native fit, full-size memory and calibration acceptance remain unestablished. +The dense fiscal kernel now forwards an optional host-owned target-snapshot +observer to the actual grouped solver. Tiny invented graph tests verify equal +weights, artifacts, receipts and cache identity with observation on/off, plus +silent required replay and final mutation refusal after the sink. The integrated +suite has 473 passes and one explicitly recorded historical thread-configuration +skip; five final calibration-identity checks also pass. Source admission, +production cadence costs and a dashboard consumer remain separate. See +[the fiscal snapshot evidence](../experiments/fiscal-target-snapshot-host-20260912.md). + Complete-population ancestry remains an explicit requirement. The PUF host now retains its actual checked result for downstream consumption through `check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot diff --git a/experiments/fiscal-target-snapshot-host-20260912.md b/experiments/fiscal-target-snapshot-host-20260912.md index 8af4eae6a..824fe94e2 100644 --- a/experiments/fiscal-target-snapshot-host-20260912.md +++ b/experiments/fiscal-target-snapshot-host-20260912.md @@ -95,6 +95,14 @@ owned shard roots plus dependency site-packages explicitly. No source-identity check was relaxed. Its report SHA256 is `69666f687e025d9a27d9d76b2cc87fefc2622734a56fc4530674b9de019ff23d`. +Five focused checks of the final minimal/country spec goldens and calibration +parity identities then passed without failures or skips. They took 58.312 +seconds wall time with all 478 source pins unchanged. The receipt SHA256 is +`ee662e419c17acc27b76761b32f2bdbb096f2dc993e1efeb437d3f01006573e8`; +the JUnit result SHA256 is +`a60a4ffcc11b4afde6fe64a332c2bcfe54b8ac298f9aa523d9a4279089ccd116`. +No UK or US country-simulation test was selected. + ## Remaining acceptance This is solver and fiscal-host instrumentation, not a dataset release or diff --git a/experiments/grouped-target-snapshot-integration-20260912.md b/experiments/grouped-target-snapshot-integration-20260912.md index 29e4793df..5e317e0bf 100644 --- a/experiments/grouped-target-snapshot-integration-20260912.md +++ b/experiments/grouped-target-snapshot-integration-20260912.md @@ -1,5 +1,11 @@ # Grouped solver x calibration target snapshot integration — 2026-09-12 +Historical scope note, 12 September 2026: this records grouped core +`2f62bb8448010bcc9b9e1419ea24aafcb85917d1` and its earlier test run. The later +[fiscal-host integration](fiscal-target-snapshot-host-20260912.md) adds the +host observer and PR #914's completed-update convention. The 443-test result +and host-wiring gaps below remain evidence of this earlier revision. + Scope, evidence and residual risks for branch `grouped-target-snapshot-integration-20260912`. Source-only; no native build, no PUF fixture, no country build, no release action, nothing pushed. From 2ab15db92f640d14ea6d4b8b6680dd004fea4f2a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:23:40 -0400 Subject: [PATCH 159/255] Qualify ordinary and retirement-account ASEC interest sources --- .../current-asec-interest-source.added.md | 1 + docs/us-current-asec-interest-source.md | 91 ++++ .../current_asec_income_routing_source.py | 35 +- .../current_asec_interest_source.py | 463 ++++++++++++++++++ .../test_us_current_asec_interest_source.py | 391 +++++++++++++++ 5 files changed, 968 insertions(+), 13 deletions(-) create mode 100644 changelog.d/current-asec-interest-source.added.md create mode 100644 docs/us-current-asec-interest-source.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_interest_source.py diff --git a/changelog.d/current-asec-interest-source.added.md b/changelog.d/current-asec-interest-source.added.md new file mode 100644 index 000000000..73de9e322 --- /dev/null +++ b/changelog.d/current-asec-interest-source.added.md @@ -0,0 +1 @@ +Qualify ordinary and retirement-account ASEC interest from retained source owners, preserving published components, combined totals, knownness and allocation metadata without assigning tax treatment or balancing observations. diff --git a/docs/us-current-asec-interest-source.md b/docs/us-current-asec-interest-source.md new file mode 100644 index 000000000..d4cbfbfd8 --- /dev/null +++ b/docs/us-current-asec-interest-source.md @@ -0,0 +1,91 @@ +# Current ASEC interest observations + +`current_asec_interest_source.qualify_current_asec_interest(preparation)` +qualifies ordinary and retirement-account interest from the original 2025 ASEC +person member, referring to income in 2024. It preserves the already qualified +`INT_VAL` combined total and exposes the published components and their +reporting, account, allocation and disclosure metadata. + +The source is the [2025 ASEC public-use data dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The exact total-income domain is reused from the pinned +`asec_current_money_domains_v1.json`; the additional fields use this same +dictionary. Pages below are one-based PDF pages. + +| Fields | Meaning and printed universe | Page | +| --- | --- | --- | +| `INT_VAL`, `INT_YN` | Combined interest; amount universe `INT_YN = 1`, receipt universe age 15+ | 46 | +| `TRDINT_VAL` | Interest excluding retirement-account interest; `INT_YN = 1` | 50 | +| `RINT_YN` | Retirement-account interest receipt; age 15+ | 49 | +| `RINT_SC1`, `RINT_SC2` | Account identities, including 401k, 403b, Roth IRA, Regular IRA, KEOGH, SEP and other; `RINT_YN = 1` | 49 | +| `RINT_VAL1`, `RINT_VAL2` | Interest in the two reported account slots; corresponding account code greater than zero | 49 | +| `I_INTVAL`, `I_INTYN` | Combined-interest allocation flags, with distinct composite code systems | 56 | +| `I_RINTSC`, `I_RINTVAL1`, `I_RINTVAL2`, `I_RINTYN` | Retirement-account allocation flags; `I_RINTSC` names slot 1 only | 57 | +| `TRINT_VAL1`, `TRINT_VAL2`, `TTRDINT_VAL` | Published top-code indicators for the respective positive component amounts | 60 | + +The account, receipt and applicable allocation code systems reuse the income +routing qualifier. `I_INTVAL` accepts the printed codes 0 and 11–15; its header +interval does not make codes 1–10 valid. There is no individually published +allocation flag for `TRDINT_VAL` or `RINT_SC2` in this dictionary. Allocation +labels describe publisher processing, not observation validity or tax status. + +## Amounts and knownness + +The result keeps raw source literals, parsed published amounts, parse status, +reporting status and canonical amount knownness separately. Missing and +malformed components remain unknown. Interest amounts must be nonnegative +integers within each field's published encoding; these disclosure ranges are +not latent-income bounds. + +`TRDINT_VAL` describes its range as a dollar value, so a published zero with +combined-interest receipt `yes` is an observed zero ordinary component. +`INT_VAL` and `RINT_VAL1/2` describe zero as none or NIU, so zero with receipt +`yes` remains ambiguous. A retirement-interest recipient with account code 0 +and slot amount 0 has an `unreported_account_slot`, not an imputed account +amount. Receipt `no` plus account code 0 and amount 0 establishes known +nonreceipt. Under-15 NIU values never become analytical zeros. + +The diagnostic `combined_minus_published_components` subtracts the three +parsed published component cells from `INT_VAL`. Its knownness means that all +four published numbers are readable; it does not establish that all three +components are reported observations. `all_component_amounts_observed` +separately records that stronger condition. The diagnostic may be negative; +the qualifier never balances, clips or replaces any published number. + +## Source lifetime and composition + +Qualification borrows the actual `AuthenticatedSurveyPopulationPreparation` +and its retained native ASEC/current-money owners. It matches the original +member's archive digest, member digest, byte count and row count, captures that +member once, and verifies the captured file after reading. Current-cohort +`PERIDNUM` keys must form an exact roster; household coordinates, person line +numbers and raw ages must match the retained parent. Source row order is +irrelevant. The original `INT_VAL` validity and float64 bits must agree with +the current-money owner before selected ASEC people are projected. + +`CurrentAsecInterestValues(person, asec_literals, evidence)` is descriptive +transport. Its seal covers both complete tables, axes, nullable masks and +backing storage, exact float bits and evidence. The qualifier rechecks the +actual preparation after its last source I/O and checks the complete value +seal before returning. A consumer must retain the actual preparation, +requalify it immediately before consumption and after its own last relevant +I/O, and compare the full value seal before returning or exporting a +successor. A copied dataclass, evidence dictionary or matching digest cannot +grant source authority. + +The only existing helper change is an optional explicit column/amount-grammar +roster on the private income-routing literal reader. Its original defaults +remain in effect for the income-routing qualifier. This module adds no host, +run issuer, graph execution or model fitting. + +## Bridge work still required + +[ACS response guidance](https://www.census.gov/programs-surveys/acs/respond/get-help.html) +includes interest credited to retirement accounts in its broad +interest/dividend/property item. The live guide currently refers to the 2025 +questionnaire; it is not an archived 2024 instruction artifact. This +ASEC qualification therefore keeps ordinary interest and retirement-account +interest separate for a later, reviewed measurement bridge. It does not +declare either component taxable, tax-exempt, regularly withdrawn, or equal +to an ACS component. It does not alter tax leaves, original or PUF clones, +financial imputation, calibration or release eligibility. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index 9c679a3db..c26f6abcd 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -976,8 +976,20 @@ def _other_income(raw, ages): return out -def _read_capture(path, *, rows): - """Bounded literal reader; its path and row count establish no authority.""" +def _read_capture(path, *, rows, columns=None, patterns=None): + """Bounded literal reader; its path and row count establish no authority. + + Private source extensions may choose an explicit literal roster and amount + grammars. The original routing qualifier retains its unchanged defaults. + """ + columns = READ_COLUMNS if columns is None else tuple(columns) + patterns = amount_patterns() if patterns is None else dict(patterns) + require( + len(set(columns)) == len(columns) + and set(COORDINATE_COLUMNS) <= set(columns) + and set(patterns) <= set(columns), + "READ_COLUMN_CONTRACT", + ) reader = source_csv_builtin.capture_csv_reader(csv) require(reader is not None, "CSV_READER_CHANGED") records, keys, coordinates = [], set(), set() @@ -988,13 +1000,13 @@ def _read_capture(path, *, rows): bool(header) and all(header) and len(header) == len(set(header)) - and set(READ_COLUMNS) <= set(header), + and set(columns) <= set(header), "HEADER", ) - positions = [header.index(c) for c in READ_COLUMNS] + positions = [header.index(c) for c in columns] for row in stream: require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") - record = dict(zip(READ_COLUMNS, (row[i] for i in positions), strict=True)) + record = dict(zip(columns, (row[i] for i in positions), strict=True)) key = record["PERIDNUM"] require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") for name, width in COORDINATE_WIDTHS.items(): @@ -1010,23 +1022,20 @@ def _read_capture(path, *, rows): min(pair) > 0 and pair not in coordinates and key not in keys, "DUPLICATE_OR_INVALID_COORDINATE", ) - for name, pattern in amount_patterns().items(): + for name, pattern in patterns.items(): require( record[name] == "" or pattern.fullmatch(record[name]) is not None, "AMOUNT_TOKEN:" + name, ) - for name in ( - *RECEIPT_ENTRIES, - *ACCOUNT_ENTRIES, - "OI_OFF", - *ALLOCATION_ENTRIES, - ): + for name in columns: + if name in COORDINATE_COLUMNS or name in patterns: + continue require(len(record[name]) <= TOKEN_MAX_CHARS, "TOKEN_BOUND:" + name) records.append(record) keys.add(key) coordinates.add(pair) require(len(records) == rows, "ROW_COUNT") - return pd.DataFrame(records, columns=READ_COLUMNS).set_index("PERIDNUM", drop=False) + return pd.DataFrame(records, columns=columns).set_index("PERIDNUM", drop=False) def amount_observations(field, positions, literals): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py new file mode 100644 index 000000000..36f3a89af --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py @@ -0,0 +1,463 @@ +"""Source observations of ordinary and retirement-account ASEC interest. + +No tax treatment, model, component balancing or source issuer lives here. +The consuming host retains and requalifies the actual preparation around I/O. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-interest-source.v1" +AMOUNT_FIELDS = ("INT_VAL", "TRDINT_VAL", "RINT_VAL1", "RINT_VAL2") +# Width, position, PDF page, printed page, maximum, universe, zero wording. +# INT_VAL is read from the already pinned current-money domain resource. +ADDITIONAL_AMOUNT_ENTRIES = { + "TRDINT_VAL": (5, 665, 50, "6C-29", 99999, "INT_YN = 1", "dollar value"), + "RINT_VAL1": (6, 608, 49, "6C-28", 999999, "RINT_SC1 > 0", "none or niu"), + "RINT_VAL2": (6, 614, 49, "6C-28", 999999, "RINT_SC2 > 0", "none or niu"), +} +RECEIPT_ENTRIES = { + "INT_YN": (1, 543, 46, "6C-25", "All Persons aged 15+", "niu"), + "RINT_YN": (1, 620, 49, "6C-28", "All Persons aged 15+", "niu"), +} +ACCOUNT_ENTRIES = { + "RINT_SC1": (1, 606, 49, "6C-28", "RINT_YN = 1"), + "RINT_SC2": (1, 607, 49, "6C-28", "RINT_YN = 1"), +} +# These published component and composite flags have different code systems. +ALLOCATION_ENTRIES = { + "I_INTVAL": (2, 838, 56, "6C-35", "INT_VAL> 0", (0, 11, 12, 13, 14, 15)), + "I_INTYN": (2, 840, 56, "6C-35", "INT_YN > 0", routing.ALLOCATION_COMPOSITE_CODES), + "I_RINTSC": (1, 857, 57, "6C-36", "RINT_SC1 > 0", routing.ALLOCATION_ANNVAL_CODES), + "I_RINTVAL1": ( + 1, + 858, + 57, + "6C-36", + "RINT_VAL1 > 0", + routing.ALLOCATION_ANNVAL_CODES, + ), + "I_RINTVAL2": ( + 1, + 859, + 57, + "6C-36", + "RINT_VAL2 > 0", + routing.ALLOCATION_ANNVAL_CODES, + ), + "I_RINTYN": (1, 860, 57, "6C-36", "RINT_YN > 0", routing.ALLOCATION_ANNVAL_CODES), +} +TOPCODE_ENTRIES = { + "TRINT_VAL1": (1, 915, 60, "6C-39", "RINT_VAL1 > 0"), + "TRINT_VAL2": (1, 916, 60, "6C-39", "RINT_VAL2 > 0"), + "TTRDINT_VAL": (1, 918, 60, "6C-39", "TRDINT_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *ACCOUNT_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_INTEREST_SOURCE_" + reason) + + +@lru_cache(maxsize=1) +def amount_entries(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + field = next(f for f in json.loads(payload)["fields"] if f["name"] == "INT_VAL") + vintage = next(v for v in field["vintages"] if v["income_year"] == 2024) + require( + vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + return { + "INT_VAL": ( + vintage["ascii_length_as_printed"], + vintage["ascii_position_as_printed"], + vintage["pdf_page_1based"], + vintage["printed_page"], + field["domain"]["encoded_range_inclusive"]["maximum"], + vintage["universe_as_printed"], + field["domain"]["zero_semantics"], + ), + **ADDITIONAL_AMOUNT_ENTRIES, + } + + +def _slot_status(age, receipt, account, amount): + code, status = account + value, amount_status = amount + receipt_code, receipt_state = receipt + # Reuse the shared universe/receipt classifier before refining account slots. + kind = "missing" if value is None else ("zero" if value == 0 else "nonzero") + base = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + if age < 15: + return base if code == 0 else "contradictory_outside_reporting_universe" + if receipt_state != "in_printed_range": + return base + if amount_status != "in_printed_range": + return ( + "missing_amount" if amount_status == "missing" else "invalid_amount_literal" + ) + if status != "in_printed_range": + return ( + "missing_account_literal" + if status == "missing" + else "invalid_account_literal" + ) + if receipt_code != 1: + return base if code == 0 else "account_without_receipt" + if code == 0: + return "unreported_account_slot" if value == 0 else "amount_without_account" + return base + + +def project_interest_literals(ordered): + """Descriptive projection; a DataFrame argument grants no source authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + out = pd.DataFrame(index=range(len(ordered))) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + parsed = {} + for name, entry in amount_entries().items(): + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) + for t in ordered[name] + ] + parsed[name] = pairs + out[name + "_published_amount"] = pd.array( + [v for v, _ in pairs], dtype="Float64" + ) + out[name + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + codes = {} + for name in (*RECEIPT_ENTRIES, *ACCOUNT_ENTRIES): + allowed = ( + routing.RECEIPT_CODE_DOMAIN + if name in RECEIPT_ENTRIES + else routing.ACCOUNT_CODES + ) + labels = ( + {0: RECEIPT_ENTRIES[name][5], 1: "yes", 2: "no"} + if name in RECEIPT_ENTRIES + else routing.ACCOUNT_CODES + ) + frame, values, statuses = routing._codes_frame( + name, ordered[name], allowed, labels + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name in AMOUNT_FIELDS: + labels = [] + for i, age in enumerate(ages): + value, status = parsed[name][i] + if name.startswith("RINT_VAL"): + label = _slot_status( + age, + codes["RINT_YN"][i], + codes["RINT_SC" + name[-1]][i], + parsed[name][i], + ) + else: + kind = ( + "missing" + if value is None + else ("zero" if value == 0 else "nonzero") + ) + label = routing.receipt_status( + bool(age >= 15), + codes["INT_YN"][i], + kind, + net_measure=False, + zero_is_dollars=name == "TRDINT_VAL", + ) + if status not in ("missing", "in_printed_range"): + label = "invalid_amount_literal" + # Unlike INT_VAL, TRDINT_VAL prints 'dollar value', not 'none or niu'. + if name == "TRDINT_VAL" and label == "known_recipient_zero": + label = "observed_zero_component" + labels.append(label) + known = np.array( + [ + s in (*routing.KNOWN_AMOUNT_STATUSES, "observed_zero_component") + for s in labels + ] + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + for name in ACCOUNT_ENTRIES: + out[name + "_account_known"] = [ + bool(age >= 15 and receipt[0] == 1 and code is not None and code > 0) + for age, receipt, (code, _) in zip( + ages, codes["RINT_YN"], codes[name], strict=True + ) + ] + flags = {} + for name, entry in ALLOCATION_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, ordered[name], entry[5], width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + flags[name] = (values, statuses) + out["allocation_origin"] = routing._allocation_origin( + flags, ("TRDINT_VAL", "RINT_SC2") + ) + for name, entry in TOPCODE_ENTRIES.items(): + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=entry[0]) + out = pd.concat([out, frame], axis=1) + published = [out[name + "_published_amount"] for name in AMOUNT_FIELDS] + out["published_components_sum"] = published[1] + published[2] + published[3] + out["combined_minus_published_components"] = ( + published[0] - out.published_components_sum + ) + out["component_discrepancy_known"] = out.combined_minus_published_components.notna() + out["all_component_amounts_observed"] = out[ + [name + "_amount_known" for name in AMOUNT_FIELDS[1:]] + ].all(axis=1) + out["source_age"] = ages + return out + + +@dataclass(frozen=True) +class CurrentAsecInterestValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def interest_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecInterestValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="interest columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="interest" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-interest-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_total(ready, positions, literals): + field = ready.field("INT_VAL") + entry = amount_entries()["INT_VAL"] + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) for t in literals + ] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "TOTAL_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "TOTAL_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "TOTAL_BITS", + ) + return field + + +def qualify_current_asec_interest(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == 2024 + and header["semantic"] == "annual_current_money" + and native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "PERIOD", + ) + pins = [p for p in routing.coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + field = _compare_total(ready, positions, ordered.INT_VAL) + basis = project_interest_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in ("statuses", "validity", "zero_origin"): + basis["INT_VAL_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "receipt_entries": RECEIPT_ENTRIES, + "account_entries": ACCOUNT_ENTRIES, + "account_codes": routing.ACCOUNT_CODES, + "allocation_entries": ALLOCATION_ENTRIES, + "topcode_entries": TOPCODE_ENTRIES, + "fields_without_individual_allocation_flag": ["TRDINT_VAL", "RINT_SC2"], + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": 2024, + "survey_year": 2025, + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "component_discrepancy_is_diagnostic_only": True, + "unreported_slot_imputed_zero": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecInterestValues(out, literals, json.loads(json.dumps(evidence))) + seal = interest_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(interest_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/tests/test_us_current_asec_interest_source.py b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py new file mode 100644 index 000000000..f69dc380c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py @@ -0,0 +1,391 @@ +"""Invented literal and real-source-owner controls for the interest extension.""" + +import copy +import hashlib +import json +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_interest_source as owner + + +def row(**changes): + return { + **{c: "0" for c in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "INT_VAL": "140", + "INT_YN": "1", + "TRDINT_VAL": "100", + "RINT_YN": "1", + "RINT_SC1": "4", + "RINT_VAL1": "40", + **changes, + } + + +def project(**changes): + return owner.project_interest_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +def test_component_totals_remain_observed_and_discrepancy_is_not_balanced(): + value = project( + INT_VAL="500", TRDINT_VAL="0", RINT_VAL1="400", RINT_SC2="7", RINT_VAL2="200" + ) + assert value.INT_VAL_amount == 500 + assert value.TRDINT_VAL_amount == 0 and value.TRDINT_VAL_amount_known + assert value.TRDINT_VAL_reporting_status == "observed_zero_component" + assert value.RINT_VAL1_amount == 400 and value.RINT_VAL2_amount == 200 + assert value.published_components_sum == 600 + assert value.combined_minus_published_components == -100 + assert value.component_discrepancy_known and value.all_component_amounts_observed + + +def test_an_unreported_account_slot_is_not_an_observed_zero(): + value = project() + assert value.RINT_VAL2_published_amount == 0 + assert value.RINT_VAL2_reporting_status == "unreported_account_slot" + assert not value.RINT_VAL2_amount_known and pd.isna(value.RINT_VAL2_amount) + assert not value.RINT_SC2_account_known + assert ( + value.component_discrepancy_known + and value.combined_minus_published_components == 0 + ) + assert not value.all_component_amounts_observed + + +@pytest.mark.parametrize( + "changes,field,status,known", + [ + ({"INT_VAL": "0"}, "INT_VAL", "ambiguous_recipient_zero", False), + ({"RINT_VAL1": "0"}, "RINT_VAL1", "ambiguous_recipient_zero", False), + ({"TRDINT_VAL": ""}, "TRDINT_VAL", "missing_amount", False), + ({"TRDINT_VAL": "-1"}, "TRDINT_VAL", "invalid_amount_literal", False), + ({"RINT_VAL1": "-1"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "4.0"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_VAL1": "NA"}, "RINT_VAL1", "invalid_amount_literal", False), + ({"RINT_SC1": ""}, "RINT_VAL1", "missing_account_literal", False), + ({"RINT_SC1": "9"}, "RINT_VAL1", "invalid_account_literal", False), + ({"RINT_SC1": "0"}, "RINT_VAL1", "amount_without_account", False), + ({"RINT_YN": ""}, "RINT_VAL1", "missing_receipt_literal", False), + ({"RINT_YN": "2"}, "RINT_VAL1", "account_without_receipt", False), + ({"INT_YN": "2"}, "TRDINT_VAL", "contradictory_no_nonzero", False), + ({"INT_YN": "0", "TRDINT_VAL": "0"}, "TRDINT_VAL", "niu", False), + ({"INT_YN": "2", "TRDINT_VAL": "0"}, "TRDINT_VAL", "known_nonreceipt", True), + ], +) +def test_receipt_account_and_amount_conflicts_remain_explicit( + changes, field, status, known +): + value = project(**changes) + assert value[field + "_reporting_status"] == status + assert bool(value[field + "_amount_known"]) is known + if not known: + assert pd.isna(value[field + "_amount"]) + + +def test_under15_universe_is_checked_without_analytical_zeros(): + value = project( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + assert value.published_components_sum == 0 # Published-cell diagnostic only. + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("1", "outside_printed_range"), + ("11", "in_printed_range"), + ("15", "in_printed_range"), + ("x", "malformed"), + ], +) +def test_composite_allocation_uses_printed_codes_not_header_interval(token, status): + value = project(I_INTVAL=token) + assert value.I_INTVAL_literal_status == status + assert ( + value.RINT_VAL1_amount_known + ) # Allocation never changes observation validity. + if token in ("11", "15"): + assert value.allocation_origin == "publisher_allocated" + + +def test_missing_component_or_flag_never_becomes_false_or_zero(): + value = project(RINT_VAL2="", TRINT_VAL2="", I_RINTVAL2="") + assert pd.isna(value.published_components_sum) + assert not value.component_discrepancy_known + assert value.TRINT_VAL2_literal_status == "missing" + assert value.allocation_origin == "allocation_flag_not_populated" + + +def test_shared_account_codes_and_pinned_total_domain_are_reused(): + assert owner.amount_entries()["INT_VAL"][4] == 999999 + assert owner.amount_entries()["TRDINT_VAL"][4] == 99999 + value = project(RINT_SC1="3") + assert value.RINT_SC1_label == owner.routing.ACCOUNT_CODES[3] == "Roth IRA" + assert value.RINT_SC1_account_known + value = project(INT_YN="0", RINT_YN="0") + assert value.INT_YN_label == value.RINT_YN_label == "niu" + + +def test_nullable_float_seal_preserves_masks_hidden_values_and_exact_bits(): + raw = pd.DataFrame([row()]) + values = owner.CurrentAsecInterestValues( + owner.project_interest_literals(raw), raw, {"descriptive": True} + ) + original = owner.interest_values_seal(values) + assert owner.interest_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "hidden", "mask"): + altered = copy.deepcopy(values) + if change == "bit": + array = altered.person.INT_VAL_amount.array + array._data[0] = np.nextafter(array._data[0], np.inf) + else: + array = altered.person.RINT_VAL2_amount.array + assert array._mask[0] + if change == "hidden": + array._data[0] = 17.0 + else: + array._mask[0] = False + assert owner.interest_values_seal(altered) != original + + +@pytest.mark.parametrize("change", ["bit", "validity", "literal"]) +def test_retained_combined_total_requires_exact_amount_and_validity(change): + field = SimpleNamespace(amounts=np.array([140.0]), validity=np.array([1])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0]), ["140"] + assert owner._compare_total(ready, positions, literals) is field + if change == "bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[0] = 0 + else: + literals[0] = "-1" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def interest_arguments(tmp_path, monkeypatch): + """Amend original invented bytes before any preparation is issued.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + amounts, ages = people.INT_VAL.to_numpy(copy=True), people.A_AGE.to_numpy(copy=True) + literals = { + 105: row(), + 106: row( + A_AGE="14", + INT_VAL="0", + INT_YN="0", + TRDINT_VAL="0", + RINT_YN="0", + RINT_SC1="0", + RINT_VAL1="0", + ), + 107: row( + A_AGE="70", + INT_VAL="500", + TRDINT_VAL="0", + RINT_VAL1="400", + RINT_SC2="7", + RINT_VAL2="200", + I_INTVAL="11", + TRINT_VAL1="1", + ), + 108: row( + INT_VAL="0", + INT_YN="2", + TRDINT_VAL="0", + RINT_YN="2", + RINT_SC1="0", + RINT_VAL1="0", + ), + } + for person_id, values in literals.items(): + amounts[ids == person_id], ages[ids == person_id] = ( + int(values["INT_VAL"]), + int(values["A_AGE"]), + ) + _changed_parent( + parent_path, attachment, monkeypatch, {"INT_VAL": amounts, "A_AGE": ages} + ) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["INT_VAL"] = [str(int(updated.loc[key, "INT_VAL"])) for key in raw.PERIDNUM] + raw["A_AGE"] = [str(int(updated.loc[key, "A_AGE"])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - {"INT_VAL"} + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "interest-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **interest_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_join_keeps_source_total_and_requalifies(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_interest(parent) + values = result.person.set_index("native_person_id") + assert values.loc[107, "INT_VAL_amount"] == 500 + assert values.loc[107, "combined_minus_published_components"] == -100 + assert values.loc[107, "TRINT_VAL1_code"] == 1 + assert values.loc[105, "RINT_SC1_label"] == "Regular IRA" + assert values.loc[106, "INT_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[108, "RINT_VAL2_reporting_status"] == "known_nonreceipt" + assert owner.interest_values_seal( + owner.qualify_current_asec_interest(parent) + ) == owner.interest_values_seal(result) + assert ( + not result.evidence["tax_treatment_assigned"] + and not result.evidence["source_admission_issued"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(copy.copy(parent)) + + +@pytest.mark.parametrize("coordinate", ["PH_SEQ", "A_LINENO", "A_AGE", "PERIDNUM"]) +def test_actual_owner_refuses_changed_source_coordinates( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + + +def test_actual_owner_refuses_mutation_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + called = [] + + def changed(*args): + raw = capture(*args) + parent._checked()[2].frame.person.iloc[ + 0, parent._checked()[2].frame.person.columns.get_loc("age") + ] += 1 + called.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_interest(parent) + assert called + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.interest_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + column = value.person["INT_VAL_amount"].array + i = np.flatnonzero(~column._mask)[0] + column._data[i] = np.nextafter(column._data[i], np.inf) + elif mutation == "masked_backing": + column = value.person["RINT_VAL2_amount"].array + i = np.flatnonzero(column._mask)[0] + column._data[i] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("RINT_SC1") + ] = "7" + else: + value.evidence["tax_treatment_assigned"] = True + return before + + monkeypatch.setattr(owner, "interest_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_interest(parent) + + +def test_default_reader_and_explicit_original_roster_agree(tmp_path): + from test_us_current_asec_income_routing import _member_row, _write_member + + path = tmp_path / "routing.csv" + _write_member(path, [_member_row()]) + default = owner.routing._read_capture(path, rows=1) + explicit = owner.routing._read_capture( + path, + rows=1, + columns=owner.routing.READ_COLUMNS, + patterns=owner.routing.amount_patterns(), + ) + pd.testing.assert_frame_equal(default, explicit) From 4a8f1d9edeacc99a36a9b2e43175081a8757f0fa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:31:19 -0400 Subject: [PATCH 160/255] Respect unpublished ASEC farm allocation code meanings --- .../asec-farm-allocation-meaning.fixed.md | 1 + docs/us-current-asec-income-routing-source.md | 6 ++- .../current_asec_income_routing_source.py | 17 ++++++- ...test_us_current_asec_allocation_meaning.py | 44 +++++++++++++++++++ 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 changelog.d/asec-farm-allocation-meaning.fixed.md create mode 100644 packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py diff --git a/changelog.d/asec-farm-allocation-meaning.fixed.md b/changelog.d/asec-farm-allocation-meaning.fixed.md new file mode 100644 index 000000000..ab570a7c3 --- /dev/null +++ b/changelog.d/asec-farm-allocation-meaning.fixed.md @@ -0,0 +1 @@ +Keep farm allocation provenance unresolved when only I_FRMYN has a nonzero code: the published dictionary supplies its range but no code meanings. Preserve allocation established by documented flags. diff --git a/docs/us-current-asec-income-routing-source.md b/docs/us-current-asec-income-routing-source.md index ed6c59408..22968e633 100644 --- a/docs/us-current-asec-income-routing-source.md +++ b/docs/us-current-asec-income-routing-source.md @@ -203,7 +203,11 @@ the under-58 source codes is unresolved; `I_DSTSC`'s label uses the same `DST_SC(2)` notation, so mapping it to both `DST_SC1` and `DST_SC2` reads the parenthesis as a slot count that the dictionary does not state; and `I_FRMYN` prints an empty `Values:` block, so only its `(0:9)` range header is published -and no meaning is claimed for its codes. No row is ever labelled a raw +and no meaning is claimed for its codes. A nonzero `I_FRMYN` alone yields +`allocation_code_meaning_unpublished`; a documented nonzero allocation flag +such as `I_ERNYN` can still establish `publisher_allocated`. All-zero readings +remain descriptions of the published codes, not assertions of nonallocation. +The original flag values and parse statuses remain unchanged. No row is ever labelled a raw respondent value. ## Constants diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index c26f6abcd..cef6f66c8 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -238,6 +238,7 @@ class AccountEntry(NamedTuple): # I_FRMYN's printed Values block is empty; its (0:9) range header is all that is # published, so no code meaning is claimed for it. ALLOCATION_PRINTED_RANGE_CODES = tuple(range(10)) +ALLOCATION_CODE_MEANINGS_UNPUBLISHED = frozenset({"I_FRMYN"}) class AllocationEntry(NamedTuple): @@ -547,16 +548,28 @@ def _allocation_origin(flags, unflagged): reading is not an assertion of non-allocation: on a non-recipient row the flag is outside its own printed universe. The zero readings are reported as exactly that, never as publisher-confirmed absence of allocation. + A readable nonzero code whose meaning is unpublished cannot establish + allocation; a documented allocation flag can still establish it. """ origins = [] count = len(next(iter(flags.values()))[0]) for i in range(count): - codes = [flags[f][0][i] for f in flags] + codes = {f: flags[f][0][i] for f in flags} statuses = [flags[f][1][i] for f in flags] if any(s not in ("missing", "in_printed_range") for s in statuses): origins.append("unresolved_allocation_provenance") - elif any(c is not None and c != 0 for c in codes): + elif any( + c is not None and c != 0 + for f, c in codes.items() + if f not in ALLOCATION_CODE_MEANINGS_UNPUBLISHED + ): origins.append("publisher_allocated") + elif any( + c is not None and c != 0 + for f, c in codes.items() + if f in ALLOCATION_CODE_MEANINGS_UNPUBLISHED + ): + origins.append("allocation_code_meaning_unpublished") elif any(s == "missing" for s in statuses): origins.append("allocation_flag_not_populated") elif unflagged: diff --git a/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py new file mode 100644 index 000000000..f250f65c2 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_allocation_meaning.py @@ -0,0 +1,44 @@ +"""Published allocation meanings over source-pure invented projections.""" + +import copy + +import pytest +from test_us_current_asec_income_routing import _pure_rows, _set_amount + +from microcosm.build.us_runtime import current_asec_income_routing_source as owner + + +@pytest.mark.parametrize( + "earn_code,farm_code,farm_status,expected", + [ + (0, 4, "in_printed_range", "allocation_code_meaning_unpublished"), + (4, 4, "in_printed_range", "publisher_allocated"), + (4, 0, "in_printed_range", "publisher_allocated"), + (4, None, "missing", "publisher_allocated"), + (0, 0, "in_printed_range", "published_flags_all_zero_with_unflagged_fields"), + (0, None, "missing", "allocation_flag_not_populated"), + (0, None, "outside_printed_range", "unresolved_allocation_provenance"), + ], +) +def test_farm_allocation_requires_a_published_code_meaning( + earn_code, farm_code, farm_status, expected +): + raw, ages = _pure_rows(1, [40.0], ERN_YN=["1"], FRMOTR=["1"], FRSE_YN=["1"]) + _set_amount(raw, "FRSE_VAL", [100.0]) + raw["allocations"]["I_ERNYN"] = ([earn_code], ["in_printed_range"]) + raw["allocations"]["I_FRMYN"] = ([farm_code], [farm_status]) + before = copy.deepcopy(raw["allocations"]) + output = owner.project_income_routing(raw, ages) + assert output.farm_allocation_origin.iloc[0] == expected + assert output.farm_known_amount.iloc[0] == 100.0 + assert raw["allocations"] == before + assert "meaning of its codes" in owner.AMBIGUOUS_FLAG_COVERAGE["I_FRMYN"] + + +@pytest.mark.parametrize( + "name,code", [("I_RNTVAL", 4), ("I_DSTSC", 9), ("I_DSTYNCOMP", 11)] +) +def test_documented_nonzero_flags_still_establish_allocation(name, code): + assert code in owner.ALLOCATION_ENTRIES[name][5] + flags = {name: ([code], ["in_printed_range"])} + assert owner._allocation_origin(flags, unflagged=False)[0] == "publisher_allocated" From adc8928cfe3f10bc6a507451e6325575145db6e2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:54:25 -0400 Subject: [PATCH 161/255] Expose signed component reconciliation in the calculation graph --- .../graph-signed-reconciliation.added.md | 1 + docs/graph-signed-reconciliation.md | 57 ++++ .../fit/graph_signed_reconciliation.py | 235 ++++++++++++++ .../tests/test_graph_signed_reconciliation.py | 288 ++++++++++++++++++ 4 files changed, 581 insertions(+) create mode 100644 changelog.d/graph-signed-reconciliation.added.md create mode 100644 docs/graph-signed-reconciliation.md create mode 100644 packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py create mode 100644 packages/microcosm-fit/tests/test_graph_signed_reconciliation.py diff --git a/changelog.d/graph-signed-reconciliation.added.md b/changelog.d/graph-signed-reconciliation.added.md new file mode 100644 index 000000000..96be9e554 --- /dev/null +++ b/changelog.d/graph-signed-reconciliation.added.md @@ -0,0 +1 @@ +Expose signed-total component reconciliation as a graph node with explicit scales, bounds, and row-level adjustment diagnostics. diff --git a/docs/graph-signed-reconciliation.md b/docs/graph-signed-reconciliation.md new file mode 100644 index 000000000..8cba5aceb --- /dev/null +++ b/docs/graph-signed-reconciliation.md @@ -0,0 +1,57 @@ +# Signed component reconciliation in the graph + +`microcosm.fit.graph_signed_reconciliation.signed_reconciliation_node` exposes +the existing weighted Euclidean projection as an ordinary graph calculation. +Its parameters name the anchor, input draws, output components, nonnegative +bounds, positive scales and numerical tolerances. The country model must +declare and justify those choices. The operator supplies no default scale or +tax interpretation. + +For the proposed ACS property-income family, the components are ordinary +interest, retirement-account interest, dividends and signed property income. +The first three have nonnegative bounds; property income can offset them. +A zero reported total therefore does not imply that every component is zero. +Retirement-account interest remains a separate survey component and is not +relabeled as taxable or tax-exempt interest. Source qualification, a defensible +ASEC-to-ACS measurement bridge, fitting and tax conversion remain separate +upstream/downstream operations; this numeric node alone does not complete them. + +The graph retains the anchor and raw draw columns. It adds the reconciled +components, each component's adjustment and bound activity, the sum residual +and the projection objective. A typed summary artifact records the complete +rule, row count, adjusted-row count, maximum absolute residual and bound counts. +It describes the numerical result and grants no source or release authority. +These declarations and columns make the rule inspectable in the shared graph +viewer without another viewer implementation. + +All input amounts must be finite float64 values. Unknown anchors or draws +refuse; the country host must declare how it selects or models those cases. +Inputs cannot share names with the new outputs, and no original observation +is overwritten. The kernel never reads survey origin, weights or an RNG and +returns no membership or weight changes. Its numerical contract is bitwise on +one platform; the explicit projection tolerances test constraints within each +run and are not a claim of cross-platform equality. + +## Verification on 12 September 2026 + +Twenty-four guarded tests pass, including direct comparison to the independently +tested pure projection; negative and zero-net anchors; exact IDs above 2^53; +row permutation; missing, infinite and incorrectly typed inputs; invalid +bounds/scales and column collisions; and declaration/context disagreement. +An actual two-node graph executes cold, retains the complete input Frame, +metadata and zero-weight record, then reopens the store under required replay. +The replay hits both nodes and preserves artifact identity. Changing scales +reuses the source node and invalidates the reconciliation node. + +The closed run used 3.69 seconds wall time and 417,775,616 bytes peak RSS. +All 1,000 source and eight owned hashes, plus 15 resource hashes, were unchanged; +there were no unexpected denied operations or child processes. These are +invented-data graph checks, not native Microcosm acceptance. Receipt SHA-256: +`7e0ed8fefc911b9d7969f48e0d2c382c189e40a0427a120a163a8ce9c8ffae01`. +JUnit SHA-256: +`914745d61c768ae4f445df408db89601d4a4268374ba4f0e6caba6085ea6be6c`. + +The first run preserved 22 passing tests and two fixture failures: the test +attempted an empty `Weights` object and constructed unsorted group IDs. +Correcting those fixture inputs produced the result above; production source +was unchanged between the two runs. Independent review is recorded separately. diff --git a/packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py b/packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py new file mode 100644 index 000000000..901978b99 --- /dev/null +++ b/packages/microcosm-fit/src/microcosm/fit/graph_signed_reconciliation.py @@ -0,0 +1,235 @@ +"""A visible graph operation for reconciling component draws to a signed total. + +The caller owns the meanings, source qualification and modeling choices. This +operator preserves its inputs and exposes the adjustments as separate columns; +it does not assign tax treatment or turn a modeled component into an observation. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import signed_reconciliation as projection +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +SUMMARY_TYPE = ArtifactType("microcosm.fit.signed_total_reconciliation_summary", 1) +PROTOCOL = "microcosm.fit.signed-total-reconciliation.v1" + + +def _require(condition, reason): + if not condition: + raise ValueError("GRAPH_SIGNED_RECONCILIATION_" + reason) + + +def _name(value): + return type(value) is str and bool(value) and value.strip() == value + + +def _diagnostics(components, prefix): + return ( + tuple(prefix + "_adjustment_" + c for c in components), + tuple(prefix + "_active_bound_" + c for c in components), + prefix + "_residual", + prefix + "_objective", + ) + + +def signed_reconciliation_node( + node_id: str, + *, + population: str, + entity: str, + anchor: str, + draws: tuple[str, ...], + components: tuple[str, ...], + nonnegative: tuple[bool, ...], + scales: tuple[float, ...], + atol: float, + rtol: float, + diagnostic_prefix: str, +) -> Node: + """Declare the complete numeric rule, with no implicit component or scale. + + The input draw names and output component names must differ. Original draw + and anchor columns remain available alongside component adjustments, bound + activity, residual and projection objective. All rows must have a finite + anchor and finite draws. Source unknowns require an explicit upstream + selection or modeled completion, never a zero supplied by this operator. + """ + _require(all(_name(v) for v in (entity, anchor, diagnostic_prefix)), "NAMES") + for values in (draws, components): + _require( + type(values) is tuple + and bool(values) + and all(_name(c) for c in values) + and len(set(values)) == len(values), + "COLUMN_ROSTER", + ) + _require(len(draws) == len(components), "COMPONENT_COUNT") + _require( + type(nonnegative) is tuple + and len(nonnegative) == len(components) + and all(type(v) is bool for v in nonnegative), + "BOUND_MASK", + ) + _require( + type(scales) is tuple + and len(scales) == len(components) + and all(type(v) is float for v in scales) + and type(atol) is type(rtol) is float, + "NUMERIC_PARAMETERS", + ) + # The pure implementation is the single authority for numeric parameter + # domains. An empty batch validates them without inventing a country model. + projection.reconcile_signed_total( + np.empty((0, len(draws))), + np.empty(0), + components=components, + nonnegative=np.array(nonnegative, dtype=bool), + scales=np.array(scales), + atol=atol, + rtol=rtol, + ) + adjustments, active, residual, objective = _diagnostics( + components, diagnostic_prefix + ) + output_names = (*components, *adjustments, *active, residual, objective) + _require( + len(set(output_names)) == len(output_names) + and len({anchor, *draws}) == 1 + len(draws) + and not set(output_names).intersection((anchor, *draws, f"{entity}_id")), + "COLUMN_COLLISION", + ) + return Node( + node_id, + SignedReconciliationKernel.ref, + population=population, + inputs=(Slice(entity, (anchor, *draws)),), + outputs=tuple( + Owned(entity, c, "bool" if c in active else "float64") for c in output_names + ), + params={ + "entity": entity, + "anchor": anchor, + "draws": draws, + "components": components, + "nonnegative": nonnegative, + "scales": scales, + "atol": atol, + "rtol": rtol, + "diagnostic_prefix": diagnostic_prefix, + }, + artifact_outputs=(ArtifactOutput("summary", SUMMARY_TYPE),), + description="Reconcile joint component draws to a supplied signed total; retain original draws, adjustments and bound diagnostics.", + ) + + +class SignedReconciliationKernel(KernelBase): + ref = "fit.signed_reconciliation@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + projection, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + expected = signed_reconciliation_node( + context.node.id, + population=context.node.population, + **dict(context.params), + ) + _require(context.node.normative() == expected.normative(), "DECLARATION") + params = expected.params + entity = params["entity"] + _require( + set(context.tables) == {entity} + and not context.sources + and not context.artifacts, + "CONTEXT", + ) + table = context.tables[entity] + id_column = f"{entity}_id" + _require( + id_column in table + and table[id_column].dtype == np.dtype("int64") + and table[id_column].is_unique, + "ROW_IDENTITY", + ) + for name in (params["anchor"], *params["draws"]): + _require( + name in table + and table[name].dtype == np.dtype("float64") + and np.isfinite(table[name].to_numpy()).all(), + "FINITE_FLOAT64_INPUT:" + name, + ) + result = projection.reconcile_signed_total( + table.loc[:, list(params["draws"])].to_numpy(copy=True), + table[params["anchor"]].to_numpy(copy=True), + components=params["components"], + nonnegative=np.array(params["nonnegative"], dtype=bool), + scales=np.array(params["scales"]), + atol=params["atol"], + rtol=params["rtol"], + ) + index = pd.Index(table[id_column].to_numpy(copy=True), name=id_column) + adjustments, active, residual, objective = _diagnostics( + result.components, params["diagnostic_prefix"] + ) + columns = {} + for j, component in enumerate(result.components): + for name, array in ( + (component, result.values[:, j]), + (adjustments[j], result.adjustments[:, j]), + (active[j], result.active_bounds[:, j]), + ): + columns[(entity, name)] = pd.Series( + array.copy(), index=index, name=name + ) + columns[(entity, residual)] = pd.Series( + result.residuals.copy(), index=index, name=residual + ) + columns[(entity, objective)] = pd.Series( + result.objective.copy(), index=index, name=objective + ) + summary = { + "protocol": PROTOCOL, + "rule": dict(params), + "rows": len(table), + "max_absolute_residual": float(np.max(abs(result.residuals), initial=0)), + "adjusted_rows": int(np.any(result.adjustments != 0, axis=1).sum()), + "active_bounds_by_component": { + name: int(result.active_bounds[:, j].sum()) + for j, name in enumerate(result.components) + }, + } + return KernelResult( + columns=columns, + artifacts={"summary": canonical_json(summary)}, + receipt={"protocol": PROTOCOL, "rows": len(table)}, + ) diff --git a/packages/microcosm-fit/tests/test_graph_signed_reconciliation.py b/packages/microcosm-fit/tests/test_graph_signed_reconciliation.py new file mode 100644 index 000000000..1a2233362 --- /dev/null +++ b/packages/microcosm-fit/tests/test_graph_signed_reconciliation.py @@ -0,0 +1,288 @@ +"""Actual graph replay and numerical boundaries for anchored component draws.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.fit.graph_signed_reconciliation import ( + SignedReconciliationKernel, + signed_reconciliation_node, +) +from microcosm.fit.signed_reconciliation import reconcile_signed_total +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + load_source, + run_graph, +) + + +def rule(**changes): + return signed_reconciliation_node( + "reconcile", + **{ + "population": "source", + "entity": "tax_unit", + "anchor": "reported_property", + "draws": ( + "draw_interest", + "draw_retirement_interest", + "draw_dividend", + "draw_property", + ), + "components": ("interest", "retirement_interest", "dividend", "property"), + "nonnegative": (True, True, True, False), + "scales": (1.0, 1.0, 1.0, 1.0), + "atol": 1e-10, + "rtol": 1e-12, + "diagnostic_prefix": "property_reconciliation", + **changes, + }, + ) + + +def table(): + return pd.DataFrame( + { + "tax_unit_id": np.array([2**53 + 17, 19, 31], dtype=np.int64), + "reported_property": [-50.0, 0.0, 130.0], + "draw_interest": [20.0, 10.0, 30.0], + "draw_retirement_interest": [10.0, 20.0, 40.0], + "draw_dividend": [10.0, 30.0, 50.0], + "draw_property": [-40.0, -60.0, 10.0], + } + ) + + +def context(values=None, node=None): + values = table() if values is None else values + node = rule() if node is None else node + return KernelContext( + node, + {"tax_unit": values}, + {} + if len(values) == 0 + else { + "tax_unit": Weights( + np.array([1.0, 2.0, 0.0])[: len(values)], WeightKind.DESIGN + ) + }, + pd.Series(dtype=str), + node.params, + np.random.default_rng(7), + ) + + +def test_matches_pure_projection_preserves_input_and_exact_ids(): + ctx = context() + before = ctx.tables["tax_unit"].copy(deep=True) + rng = json.dumps(ctx.rng.bit_generator.state, sort_keys=True) + output = SignedReconciliationKernel().run(ctx) + p = ctx.params + expected = reconcile_signed_total( + before[list(p["draws"])].to_numpy(), + before[p["anchor"]].to_numpy(), + components=p["components"], + nonnegative=np.array(p["nonnegative"]), + scales=np.array(p["scales"]), + atol=p["atol"], + rtol=p["rtol"], + ) + for j, c in enumerate(p["components"]): + result = output.columns[("tax_unit", c)] + np.testing.assert_array_equal(result, expected.values[:, j]) + np.testing.assert_array_equal(result.index, before.tax_unit_id) + assert result.index.dtype == np.dtype("int64") + np.testing.assert_array_equal( + output.columns[("tax_unit", "property_reconciliation_adjustment_" + c)], + expected.adjustments[:, j], + ) + # A zero net total need not mean no component income. + assert output.columns[("tax_unit", "interest")].iloc[1] > 0 + assert output.columns[("tax_unit", "property")].iloc[1] < 0 + pd.testing.assert_frame_equal(before, ctx.tables["tax_unit"]) + assert json.dumps(ctx.rng.bit_generator.state, sort_keys=True) == rng + assert not output.weights and output.frame is None + + +def test_row_permutation_preserves_id_association_and_diagnostics(): + kernel = SignedReconciliationKernel() + original = kernel.run(context()) + permuted = kernel.run(context(table().iloc[[2, 0, 1]])) + for key in original.columns: + pd.testing.assert_series_equal( + original.columns[key].sort_index(), permuted.columns[key].sort_index() + ) + assert original.artifacts == permuted.artifacts + + +@pytest.mark.parametrize("bad", [np.nan, np.inf, -np.inf]) +@pytest.mark.parametrize("column", ["reported_property", "draw_interest"]) +def test_unknown_or_nonfinite_anchor_or_draw_refuses(column, bad): + values = table() + values.loc[0, column] = bad + with pytest.raises(ValueError, match="FINITE_FLOAT64_INPUT"): + SignedReconciliationKernel().run(context(values)) + + +@pytest.mark.parametrize( + "change", + [ + {"scales": (1.0, 0.0, 1.0, 1.0)}, + {"scales": (1.0, float("inf"), 1.0, 1.0)}, + {"nonnegative": (True, True, True, True)}, + {"nonnegative": (1, True, True, False)}, + {"components": ("interest", "interest", "dividend", "property")}, + {"anchor": "draw_interest"}, + { + "components": ( + "draw_interest", + "retirement_interest", + "dividend", + "property", + ) + }, + {"atol": -1.0}, + {"draws": ("draw_interest",)}, + ], +) +def test_invalid_numeric_rule_or_column_collision_refuses_at_declaration(change): + with pytest.raises(ValueError): + rule(**change) + + +def test_direct_context_cannot_change_declared_scales_or_output_roster(): + ctx = context() + altered = replace(ctx, params={**ctx.params, "scales": (10.0, 1.0, 1.0, 1.0)}) + with pytest.raises(ValueError, match="DECLARATION"): + SignedReconciliationKernel().run(altered) + altered = replace(ctx, node=replace(ctx.node, outputs=ctx.node.outputs[:-1])) + with pytest.raises(ValueError, match="DECLARATION"): + SignedReconciliationKernel().run(altered) + + +@pytest.mark.parametrize( + "defect", ("id_float", "id_duplicate", "nullable", "bool_draw") +) +def test_no_silent_identity_or_dtype_coercion(defect): + values = table() + if defect == "id_float": + values["tax_unit_id"] = values.tax_unit_id.astype(float) + elif defect == "id_duplicate": + values.loc[1, "tax_unit_id"] = values.loc[0, "tax_unit_id"] + elif defect == "nullable": + values["draw_interest"] = pd.array(values.draw_interest, dtype="Float64") + else: + values["draw_interest"] = True + with pytest.raises(ValueError): + SignedReconciliationKernel().run(context(values)) + + +def test_empty_projection_has_explicit_zero_diagnostics(): + result = SignedReconciliationKernel().run(context(table().iloc[:0])) + assert all(len(s) == 0 for s in result.columns.values()) + summary = json.loads(result.artifacts["summary"]) + assert summary["rows"] == summary["adjusted_rows"] == 0 + assert summary["max_absolute_residual"] == 0 + + +class SourceKernel(KernelBase): + ref = "test.signed_reconciliation_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, ctx): + return KernelResult(frame=load_source("frame-store", ctx.sources["source"])) + + +def test_actual_cold_and_required_graph_replay_preserves_complete_frame(tmp_path): + values = table().sort_values("tax_unit_id").reset_index(drop=True) + ids = values.tax_unit_id.to_numpy(copy=True) + frame = Frame( + { + "person": pd.DataFrame({"person_id": ids, "person_tax_unit_id": ids}), + "tax_unit": values, + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(np.array([1.0, 2.0, 0.0]), WeightKind.DESIGN)}, + metadata={"survey": "invented", "preserve": {"version": 2}}, + ) + source = Node( + "source", + SourceKernel.ref, + sources=("source",), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned("tax_unit", c, "float64") for c in values if c != "tax_unit_id" + ), + ) + node = rule() + graph = compile_graph( + Graph( + country="us", + sources=(SourceRef("source", "frame-store"),), + nodes=(source, node), + ) + ) + kernels = KernelRegistry() + kernels.register(SourceKernel()) + kernels.register(SignedReconciliationKernel()) + store = ContentStore(tmp_path) + key = hashlib.sha256(values.to_json().encode()).hexdigest() + sources = {"source": store.put_frame(key, frame)} + observed = {} + first = run_graph( + graph, + sources=sources, + store=store, + kernels=kernels, + _population_observer=lambda name, pop: observed.update({name: pop}), + ) + complete = observed["reconcile"].frame + pd.testing.assert_frame_equal(complete.person, frame.person) + pd.testing.assert_frame_equal(complete.table("tax_unit")[list(values)], values) + np.testing.assert_array_equal( + complete.weights_for("tax_unit").values, frame.weights_for("tax_unit").values + ) + assert complete.metadata == frame.metadata + assert not any(n.hit for n in first.nodes.values()) + second = run_graph( + graph, + sources=sources, + store=ContentStore(tmp_path), + kernels=kernels, + resume="require", + ) + assert first.key == second.key + assert all(n.hit for n in second.nodes.values()) + assert ( + first.node("reconcile").opaque_artifacts + == second.node("reconcile").opaque_artifacts + ) + changed = compile_graph( + Graph( + country="us", + sources=(SourceRef("source", "frame-store"),), + nodes=(source, rule(scales=(2.0, 1.0, 1.0, 1.0))), + ) + ) + third = run_graph(changed, sources=sources, store=store, kernels=kernels) + assert third.node("source").hit and not third.node("reconcile").hit + assert third.node("reconcile").key != first.node("reconcile").key From 0e8d5a72a6c776df7e193b0c8b744df57e576047 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:55:17 -0400 Subject: [PATCH 162/255] Declare pinned fiscal leaf policies with strict execution boundaries --- changelog.d/us-fiscal-leaf-policy.added.md | 1 + docs/fiscal-leaf-policy.md | 114 ++++ docs/us-uk-release-path.md | 8 + .../build/us_runtime/fiscal_leaf_policy.py | 255 +++++++++ .../us_runtime/graph_fiscal_measurement.py | 109 +++- .../tests/test_us_fiscal_leaf_policy.py | 511 ++++++++++++++++++ 6 files changed, 989 insertions(+), 9 deletions(-) create mode 100644 changelog.d/us-fiscal-leaf-policy.added.md create mode 100644 docs/fiscal-leaf-policy.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py create mode 100644 packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py diff --git a/changelog.d/us-fiscal-leaf-policy.added.md b/changelog.d/us-fiscal-leaf-policy.added.md new file mode 100644 index 000000000..dee0da8d5 --- /dev/null +++ b/changelog.d/us-fiscal-leaf-policy.added.md @@ -0,0 +1 @@ +Add optional pinned fiscal input policies with strict producer defaults, complete-population collision checks and private-copy preparation. Literal assumptions remain non-executable pending checked parent integration; sampled traces cannot exempt inactive inputs. diff --git a/docs/fiscal-leaf-policy.md b/docs/fiscal-leaf-policy.md new file mode 100644 index 000000000..823603506 --- /dev/null +++ b/docs/fiscal-leaf-policy.md @@ -0,0 +1,114 @@ +# Fiscal input policies + +The declared fiscal measurement stage requires an input producer for every +leaf in its static model closure. Missing and nonfinite values still refuse +evaluation. The optional leaf policy records more precise intent without +weakening that default. + +An explicit policy can describe producer ownership or a proposed literal +scenario assumption. **Assumption-bearing executions currently refuse with +`FISCAL_MEASUREMENT_ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED`.** No US assumption +resource is enabled, and no dataset column or producer requirement changes. + +## Exact policy identity + +`load_fiscal_leaf_policy(payload, expected_sha256=...)` accepts bounded, +canonical JSON bytes and an explicit SHA-256 pin. It performs no file or +network I/O. The same `FiscalLeafPolicy` goes to `fiscal_measurement_node` and +`FiscalMeasurementKernel`. Its pin is rechecked when used; the parsed document +is detached. Omitting the argument, or passing `None`, preserves the previous +all-producer declaration shape and runtime behavior. + +A policy has schema version 1, artifact kind +`microcosm.us.fiscal_leaf_policy`, one positive integer `period`, an exact +ordered list of model `roots`, and an `entries` mapping covering the static +closure exactly. Missing entries and stale entries refuse. Each entry's +entity must agree with the independently derived model metadata. + +| Kind | Required entry fields | Behavior | +| --- | --- | --- | +| `producer` | `kind`, `entity`, `producer` | The declared input column must exist and every cell must be known and finite. | +| `assumption` | `kind`, `entity`, `value`, `interpretation`, `period`, `roots`, `affected_roots`, `affected_programs`, `rationale`, `reviewer`, `source_issue`, `reform_sensitivity` | Validated and inspectable; execution remains unsupported. | +| `inactive` | No supported shape | Always refused. | + +An assumption's interpretation is explicitly `baseline_behavior` or +`scenario_parameter`. Neither category means the program is insensitive to +the choice. The affected roots must include every declared root whose static +closure contains that leaf, in declaration order. Scope must exactly match +the policy's period and roots; program names and review fields cannot be +empty. These fields describe claimed review and intent. They do not prove +reviewer authorization, establish a source observation, or make an observed +attribute an acceptable assumption. + +V1 supports finite Boolean, signed 64-bit integer and floating-point literals +with explicit engine type checks. Strings, enums, null, containers and an +`engine default` reference refuse. The actual engine metadata and whether a +default is available (with its literal value when present) are recorded +beside the proposed value. The host never chooses a value by default lookup. +The exact policy and engine/default record enter `model_contract`, so policy, +scope, ownership, review or engine-default changes alter the declared node and +its cache key. The same declaration is embedded in a successful measurement +artifact. In this version, only producer policies can produce that artifact. + +A column constant at an engine default remains admissible as a producer at +this boundary. The label does not establish observation or modeled variation. +The source owner and existing release input-coverage/variation gates retain +that responsibility. Behavioral take-up and filing propensities are not +automatically classified as innocuous scenario settings. No prior wages are +added by this infrastructure. + +## Complete-parent admission still required + +The graph kernel receives only its declared input slices. It cannot prove a +column is absent from the complete parent when a caller omitted that column +from the slice. Declaration checks reject an assumption name present on any +declared entity, but this is insufficient to enable execution. + +The future country host must retain its independently checked complete +population and use `validate_fiscal_leaf_policy_population(frame, policy)` +before projection, around relevant I/O, and during cache admission. The +validator rejects any incumbent assumption name on any entity, including +wholly unknown or partially unknown columns. Passing a projection or a +caller-authored column roster is not absence proof. The validator itself +does not authenticate or issue a parent. + +`private_fiscal_assumption_frame(frame, policy)` is a detached preparation +helper for that future integration. It repeats the collision check and +creates engine-only tables. It preserves the supplied frame's values, +membership, weights, strata and metadata. The current fiscal kernel does not +call it. A future host must validate the exact policy against the admitted +engine, retain source and complete-parent checks before and after I/O, bind +the engine/default record, evaluate only the private copy, and keep assumption +columns out of the population and measurement adapter. A cached artifact +cannot supply missing source or parent authority. + +## Why inactivity is unsupported + +`validation_input_coverage._provision_input_leaves` traces a single empty +one-person simulation. It observes the leaves read by that execution. A +conditional formula or `defined_for` branch may read additional leaves for +other households. Absence from that trace cannot establish universal +inactivity, even at the same period and roots. + +Future inactivity support needs exhaustive engine-, period- and root-scoped +dependency proof, including conditional paths and declared state roots. The +static closure remains conservative; neither it nor a sampled trace supplies +this exemption today. + +## Bounded implementation evidence + +The local source/mock test run passed 91 cases, including the existing +invented fiscal measurement, calibration and observer graphs. The new controls +check strict-default compatibility, malformed policies, exact scope and +closure coverage, type mismatches, private-copy isolation, omitted-column +collisions, late policy mutation, graph key changes and required replay with +a forbidden model call. There were no country engine imports, network +connections or subprocess operations; no native population or source data +was used. This is infrastructure validation, not actual engine-policy +admission or candidate fiscal validation. + +The source and selected test files were sealed before and after the run. +Receipt SHA-256: +`c7aed643333638460d7464091be0c068b910baba2eefbb76cfa1208af6f8265f`. +JUnit SHA-256: +`54215b02c7a8a85488db48b2fee339b0ddbad907a0985502019c185b9a1c3adb`. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 25c41af84..53e873a6b 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -140,6 +140,14 @@ skip; five final calibration-identity checks also pass. Source admission, production cadence costs and a dashboard consumer remain separate. See [the fiscal snapshot evidence](../experiments/fiscal-target-snapshot-host-20260912.md). +Fiscal leaf-policy infrastructure now distinguishes required producers from +explicitly documented proposed assumptions. It passes 91 source/mock and +invented fiscal graph checks. The default remains strict all-producer; +assumption execution and inactive-leaf exemptions remain unsupported until +complete-parent admission and exhaustive dependency proof exist. No US +assumptions or new dataset inputs are enabled. See +[the policy contract and boundary](fiscal-leaf-policy.md). + Complete-population ancestry remains an explicit requirement. The PUF host now retains its actual checked result for downstream consumption through `check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py new file mode 100644 index 000000000..b50944904 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_leaf_policy.py @@ -0,0 +1,255 @@ +"""Explicit fiscal input policy, without source or complete-parent authority. + +Documents describe reviewed intent, not verified approval. No assumptions are +enabled in a US candidate by this module. A dynamic trace of one population +cannot prove universal inactivity, so inactive entries are unsupported. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass + +import numpy as np + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json + +MAX_POLICY_BYTES = 1024**2 +_POLICY_KIND = "microcosm.us.fiscal_leaf_policy" +_PRODUCER_FIELDS = {"kind", "entity", "producer"} +_ASSUMPTION_FIELDS = { + "kind", + "entity", + "value", + "interpretation", + "period", + "roots", + "affected_roots", + "affected_programs", + "rationale", + "reviewer", + "source_issue", + "reform_sensitivity", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_LEAF_POLICY_" + reason) + + +def _text(value): + return type(value) is str and bool(value.strip()) + + +def _names(value): + return ( + type(value) is list + and bool(value) + and all(_text(item) for item in value) + and len(set(value)) == len(value) + ) + + +def _literal(value): + return ( + type(value) is bool + or (type(value) is int and -(2**63) <= value < 2**63) + or (type(value) is float and math.isfinite(value)) + ) + + +@dataclass(frozen=True) +class FiscalLeafPolicy: + """Exact bytes and digest; descriptive only, never a parent-admission token.""" + + payload: bytes + sha256: str + + +def load_fiscal_leaf_policy( + payload: bytes, *, expected_sha256: str +) -> FiscalLeafPolicy: + """Verify caller-supplied policy bytes against an explicitly reviewed pin. + + No file or network access. An entry's reviewer/issue text is provenance, + not authenticated approval and not permission to run assumptions. + """ + result = FiscalLeafPolicy(payload, expected_sha256) + fiscal_leaf_policy_document(result) + return result + + +def fiscal_leaf_policy_document(policy: FiscalLeafPolicy) -> dict: + """Recheck the pin and schema, returning a detached parsed document.""" + _require(type(policy) is FiscalLeafPolicy, "TYPE") + _require( + type(policy.payload) is bytes and 0 < len(policy.payload) <= MAX_POLICY_BYTES, + "BOUND", + ) + _require( + type(policy.sha256) is str + and hashlib.sha256(policy.payload).hexdigest() == policy.sha256, + "FINGERPRINT", + ) + try: + raw = json.loads(policy.payload) + canonical = canonical_json(raw) + except (ValueError, TypeError, UnicodeError, RecursionError): + raise ValueError("FISCAL_LEAF_POLICY_JSON") from None + _require(canonical == policy.payload, "CANONICAL") + _require( + type(raw) is dict + and set(raw) + == {"schema_version", "artifact_kind", "period", "roots", "entries"}, + "FIELDS", + ) + _require( + type(raw["schema_version"]) is int + and raw["schema_version"] == 1 + and raw["artifact_kind"] == _POLICY_KIND, + "SCHEMA", + ) + _require( + type(raw["period"]) is int and raw["period"] > 0 and _names(raw["roots"]), + "SCOPE", + ) + _require( + type(raw["entries"]) is dict and all(_text(name) for name in raw["entries"]), + "ENTRIES", + ) + for entry in raw["entries"].values(): + _require(type(entry) is dict, "ENTRY") + kind = entry.get("kind") + _require(kind != "inactive", "INACTIVE_UNSUPPORTED") + _require(kind in ("producer", "assumption"), "ENTRY_KIND") + _require(entry.get("entity") in US_SCHEMA.entities, "ENTITY") + if kind == "producer": + _require( + set(entry) == _PRODUCER_FIELDS and _text(entry["producer"]), + "PRODUCER_FIELDS", + ) + continue + _require(set(entry) == _ASSUMPTION_FIELDS, "ASSUMPTION_FIELDS") + _require(_literal(entry["value"]), "LITERAL_UNSUPPORTED") + _require( + entry["interpretation"] in ("baseline_behavior", "scenario_parameter"), + "INTERPRETATION", + ) + _require( + type(entry["period"]) is int + and entry["period"] == raw["period"] + and entry["roots"] == raw["roots"], + "ASSUMPTION_SCOPE", + ) + _require( + _names(entry["affected_roots"]) + and set(entry["affected_roots"]) <= set(raw["roots"]) + and _names(entry["affected_programs"]), + "AFFECTED_SCOPE", + ) + _require( + all( + _text(entry[key]) + for key in ( + "rationale", + "reviewer", + "source_issue", + "reform_sensitivity", + ) + ), + "REVIEW_SCOPE", + ) + return raw + + +def classify_fiscal_leaf_policy(policy, *, period, roots, leaves): + """Bind a policy exactly to the independently derived static input closure.""" + raw = fiscal_leaf_policy_document(policy) + _require(raw["period"] == period, "POLICY_PERIOD") + _require(raw["roots"] == list(roots), "POLICY_ROOTS") + _require(set(leaves) <= set(raw["entries"]), "UNPOLICIED_MODEL_LEAF") + _require(set(raw["entries"]) <= set(leaves), "STALE_MODEL_LEAF") + _require( + all(entry["entity"] == leaves[name] for name, entry in raw["entries"].items()), + "LEAF_ENTITY", + ) + return raw + + +def fiscal_assumption_engine_records(document, *, metadata, defaults): + """Validate literal types and retain actual engine metadata/default records. + + The caller derives metadata/defaults from the engine pinned in the fiscal + contract. Defaults are recorded, never selected by reference. Enum/string + assumptions require an additional reviewed representation and are refused. + """ + records = {} + for name, entry in document["entries"].items(): + if entry["kind"] != "assumption": + continue + item = metadata[name] + _require( + item["name"] == name and item["entity"] == entry["entity"], "ENGINE_ENTITY" + ) + value = entry["value"] + _require( + (item["dtype"] == "bool" and type(value) is bool) + or (item["dtype"] == "int" and type(value) is int) + or (item["dtype"] == "float" and type(value) in (int, float)), + "ENGINE_LITERAL_TYPE", + ) + default = {"available": name in defaults} + if name in defaults: + _require(_literal(defaults[name]), "ENGINE_DEFAULT_UNSUPPORTED") + default["value"] = defaults[name] + records[name] = {"metadata": dict(item), "default": default} + return records + + +def validate_fiscal_leaf_policy_population( + frame: Frame, policy: FiscalLeafPolicy +) -> None: + """Refuse any incumbent assumption column in an actual complete Frame. + + The country host must supply its independently admitted complete parent, + before projection and again around cache/export I/O. Passing a projection + is not absence proof. This function does not issue or authenticate a Frame. + Even wholly or partially unknown incumbent columns count as collisions. + """ + raw = fiscal_leaf_policy_document(policy) + _require(frame.schema == US_SCHEMA, "POPULATION_SCHEMA") + supplied = { + column for entity in frame.entities for column in frame.table(entity).columns + } + for name, entry in raw["entries"].items(): + if entry["kind"] == "assumption": + _require(name not in supplied, "ASSUMPTION_COLUMN_COLLISION:" + name) + + +def private_fiscal_assumption_frame(frame: Frame, policy: FiscalLeafPolicy) -> Frame: + """Prepare detached engine-only tables; not called by the fiscal kernel. + + Future host integration must separately admit the complete parent, bind + policy plus engine records, and call the collision validator around I/O. + This pure helper is not an assumption-enabled graph execution path. + """ + validate_fiscal_leaf_policy_population(frame, policy) + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + tables.update({name: frame.link(name).copy(deep=True) for name in frame.links}) + for name, entry in fiscal_leaf_policy_document(policy)["entries"].items(): + if entry["kind"] == "assumption": + tables[entry["entity"]][name] = np.full( + frame.n(entry["entity"]), entry["value"] + ) + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + strata=frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py index 6b6ce6bde..f2761f07f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -7,8 +7,10 @@ The host owns source/target activation and complete-enrichment ancestry. Computed roots use the existing real US adapter only after its static closure -is explicitly present, with no missing cells or default allowlist. Measurements -and model outputs live only in private tables and a bounded CSR artifact. +is explicitly present, with no missing cells or default allowlist. Optional +leaf policies bind producer intent or planned literal assumptions; assumption +execution remains unsupported until a complete-parent host admits it. +Measurements and model outputs live only in private tables and a CSR artifact. """ from __future__ import annotations @@ -46,6 +48,7 @@ from microcosm.graph.executor import _context_digest from . import asec_engine_evaluation as runtime +from . import fiscal_leaf_policy as leaf_policies MEASUREMENT_TYPE = ArtifactType("microcosm.us.declared_fiscal_measurement", 1) PROTOCOL = "microcosm.us.declared-fiscal-measurement.v1" @@ -161,6 +164,62 @@ def _model_contract(roots): return {"evaluation": "real_baseline", "runtime": identity, "roots": contracts} +def _resolved_model_contract(roots, leaf_policy=None): + # Validate malformed policies before importing any engine metadata/runtime. + document = ( + None + if leaf_policy is None + else leaf_policies.fiscal_leaf_policy_document(leaf_policy) + ) + if document is not None: + _require(document["roots"] == list(roots), "POLICY_ROOTS") + model = _model_contract(roots) + if document is None: + return model + leaves = {} + for contract in model["roots"]: + for name, entity in contract["leaves"].items(): + _require(name not in leaves or leaves[name] == entity, "LEAF_ENTITY") + leaves[name] = entity + document = leaf_policies.classify_fiscal_leaf_policy( + leaf_policy, + period=document["period"], + roots=roots, + leaves=leaves, + ) + assumptions = [ + name + for name, entry in document["entries"].items() + if entry["kind"] == "assumption" + ] + for name in assumptions: + affected = [ + contract["closure"]["root"] + for contract in model["roots"] + if name in contract["leaves"] + ] + _require( + document["entries"][name]["affected_roots"] == affected, + "ASSUMPTION_AFFECTED_ROOTS:" + name, + ) + records = {} + if assumptions: + index = policyengine_us.PolicyEngineUSVariableMetadataIndex() + records = leaf_policies.fiscal_assumption_engine_records( + document, + metadata={ + name: asdict(index.variable_metadata(name)) for name in assumptions + }, + defaults=policyengine_us.PolicyEngineUSEngine().default_values(assumptions), + ) + model["leaf_policy"] = { + "sha256": leaf_policy.sha256, + "document": document, + "engine_records": records, + } + return model + + def fiscal_measurement_node( registry: TargetRegistry, *, @@ -171,6 +230,7 @@ def fiscal_measurement_node( geography_vintage: str, period: int, model_outputs: tuple[str, ...] = (), + leaf_policy: leaf_policies.FiscalLeafPolicy | None = None, node_id: str = "us.fiscal_measurement", ) -> Node: """Declare a bounded measurement stage; supplied targets are not activated here. @@ -180,7 +240,10 @@ def fiscal_measurement_node( group tables are reconstructed as sorted IDs only, using Frame's exact membership contract; none of their values is inferred or used. Every required input leaf is checked against this declaration before model - evaluation. Neither this node nor its artifact admits a complete parent. + evaluation. None retains the strict all-producer contract. A policy with + assumptions may be declared for inspection but cannot execute in this + kernel: complete-parent collision admission has not been implemented. + Neither this node nor its artifact admits a complete parent. """ _require(type(period) is int and period > 0, "PERIOD") _require(type(geography_vintage) is str and bool(geography_vintage), "VINTAGE") @@ -190,6 +253,10 @@ def fiscal_measurement_node( and len(set(model_outputs)) == len(model_outputs), "MODEL_OUTPUTS", ) + if leaf_policy is not None: + policy_document = leaf_policies.fiscal_leaf_policy_document(leaf_policy) + _require(policy_document["period"] == period, "POLICY_PERIOD") + _require(policy_document["roots"] == list(model_outputs), "POLICY_ROOTS") registry = _registry(_registry_document(registry)) scopes = _scopes([asdict(s) for s in advertised_scopes]) entities = (US_SCHEMA.person_entity, *US_SCHEMA.group_entities) @@ -262,7 +329,12 @@ def fiscal_measurement_node( ) _require(used_bindings == set(contract_targets), "UNUSED_BINDINGS") _require(target_scopes == keys, "UNCOVERED_ADVERTISED_SCOPE") - model = _model_contract(model_outputs) + model = _resolved_model_contract(model_outputs, leaf_policy) + entries = {} if leaf_policy is None else model["leaf_policy"]["document"]["entries"] + supplied = {name for names in columns.values() for name in names} + for name, entry in entries.items(): + if entry["kind"] == "assumption": + _require(name not in supplied, "ASSUMPTION_COLUMN_COLLISION:" + name) for contract in model["roots"]: root = contract["closure"]["root"] _require(root not in columns[contract["entity"]], "FORMULA_INPUT_COLLISION") @@ -274,7 +346,8 @@ def fiscal_measurement_node( "FORMULA_INPUT_COLLISION", ) for leaf, entity in contract["leaves"].items(): - _require(leaf in columns[entity], "UNDECLARED_MODEL_LEAF:" + leaf) + if leaf not in entries or entries[leaf]["kind"] == "producer": + _require(leaf in columns[entity], "UNDECLARED_MODEL_LEAF:" + leaf) params = { "registry": _json(_registry_document(registry)), "contract_targets": _json(contract_targets), @@ -513,8 +586,11 @@ class FiscalMeasurementKernel(KernelBase): dependencies=("numpy", "pandas", "scipy"), ) - def __init__(self, *, model_outputs=()): + def __init__(self, *, model_outputs=(), leaf_policy=None): self.model_outputs = model_outputs + if leaf_policy is not None: + leaf_policies.fiscal_leaf_policy_document(leaf_policy) + self.leaf_policy = leaf_policy def implementation_hash(self): return _sha( @@ -532,10 +608,13 @@ def implementation_hash(self): # to this helper's separate defining module. policyengine_us.engine_tables, runtime, + leaf_policies, _context_digest, dependencies=self.capabilities.dependencies, ), - "model": _model_contract(self.model_outputs), + "model": _resolved_model_contract( + self.model_outputs, self.leaf_policy + ), } ) ) @@ -554,9 +633,21 @@ def run(self, context: KernelContext) -> KernelResult: geography_vintage=params["geography_vintage"], period=params["period"], model_outputs=self.model_outputs, + leaf_policy=self.leaf_policy, node_id=context.node.id, ) _require(context.node.normative() == expected.normative(), "NODE_DECLARATION") + model = json.loads(params["model_contract"]) + _require( + not any( + entry["kind"] == "assumption" + for entry in model.get("leaf_policy", {}) + .get("document", {}) + .get("entries", {}) + .values() + ), + "ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED", + ) projection = _context_digest(context).hex() tables = {e: t.copy(deep=True) for e, t in context.tables.items()} for entity in US_SCHEMA.group_entities: @@ -575,7 +666,6 @@ def run(self, context: KernelContext) -> KernelResult: strata=context.strata.copy(), ) adapter = _Adapter(frame) - model = json.loads(params["model_contract"]) if self.model_outputs: for contract in model["roots"]: for leaf, entity in contract["leaves"].items(): @@ -666,7 +756,8 @@ def run(self, context: KernelContext) -> KernelResult: decode_fiscal_measurement(payload) _require(_context_digest(context).hex() == projection, "INPUT_MUTATION") _require( - _json(_model_contract(self.model_outputs)) == params["model_contract"], + _json(_resolved_model_contract(self.model_outputs, self.leaf_policy)) + == params["model_contract"], "MODEL_DRIFT", ) return KernelResult( diff --git a/packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py b/packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py new file mode 100644 index 000000000..2586935e1 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_fiscal_leaf_policy.py @@ -0,0 +1,511 @@ +"""Invented policy/graph boundaries; no country engine or population source reads.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_fiscal_measurement import InventedSource, context, declaration, frame + +from microcosm.build.us_runtime import fiscal_leaf_policy as policy +from microcosm.build.us_runtime import graph_fiscal_measurement as stage +from microcosm.frame import VariableMetadata +from microcosm.frame.adapters.policyengine_us import VariableDependencyClosure +from microcosm.graph import ( + ContentStore, + Graph, + KernelRegistry, + Node, + NodeRejectedError, + Owned, + SourceRef, + StoreMissError, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import node_key + + +def document(*, assumption=True): + entries = { + "amount": {"kind": "producer", "entity": "person", "producer": "survey.amount"}, + "scenario_multiplier": { + "kind": "producer", + "entity": "person", + "producer": "survey.behavior", + }, + } + if assumption: + entries["scenario_multiplier"] = { + "kind": "assumption", + "entity": "person", + "value": 1.0, + "interpretation": "baseline_behavior", + "period": 2024, + "roots": ["invented_fiscal"], + "affected_roots": ["invented_fiscal"], + "affected_programs": ["invented program"], + "rationale": "Invented boundary control, not a supported US assumption.", + "reviewer": "Invented reviewer; no real approval implied", + "source_issue": "fixture:invented-policy-review", + "reform_sensitivity": "Constant participation changes the response to eligibility reforms.", + } + return { + "schema_version": 1, + "artifact_kind": "microcosm.us.fiscal_leaf_policy", + "period": 2024, + "roots": ["invented_fiscal"], + "entries": entries, + } + + +def load(raw=None): + payload = canonical_json(document() if raw is None else raw) + return policy.load_fiscal_leaf_policy( + payload, expected_sha256=hashlib.sha256(payload).hexdigest() + ) + + +@pytest.fixture +def mocked_model(monkeypatch): + """Replace only country metadata/evaluation; actual fiscal graph stays real.""" + calls = {"materialize": 0, "defaults": 0} + default = {"value": 0.0} + + class Index: + def variable_dependency_closure(self, root): + return VariableDependencyClosure( + "invented", + root, + ("amount", "scenario_multiplier"), + (root,), + ((root, "amount"), (root, "scenario_multiplier")), + "a" * 64, + ) + + def variable_metadata(self, name): + return VariableMetadata(name, "person", "float", "year") + + class Engine: + def default_values(self, names): + calls["defaults"] += 1 + return {name: default["value"] for name in names} + + def materialize(self, value, names, period): + calls["materialize"] += 1 + assert period == 2024 + return { + name: value.table("person").amount.to_numpy(copy=True) for name in names + } + + monkeypatch.setattr( + stage.runtime, "engine_runtime_identity", lambda: {"fixture": "engine"} + ) + monkeypatch.setattr( + stage.policyengine_us, "PolicyEngineUSVariableMetadataIndex", Index + ) + monkeypatch.setattr(stage.policyengine_us, "PolicyEngineUSEngine", Engine) + return calls, default + + +def arguments(leaf_policy=None): + result = declaration( + value_variable="invented_fiscal", model_outputs=("invented_fiscal",) + ) + if leaf_policy is not None: + result["leaf_policy"] = leaf_policy + return result + + +def test_pinned_document_is_canonical_detached_and_detects_changed_bytes(): + pinned = load() + raw = policy.fiscal_leaf_policy_document(pinned) + raw["entries"]["scenario_multiplier"]["value"] = 3.0 + assert policy.fiscal_leaf_policy_document(pinned) == document() + with pytest.raises(ValueError, match="FINGERPRINT"): + policy.load_fiscal_leaf_policy( + pinned.payload + b" ", expected_sha256=pinned.sha256 + ) + with pytest.raises(ValueError, match="CANONICAL"): + altered = json.dumps(document(), indent=2).encode() + policy.load_fiscal_leaf_policy( + altered, expected_sha256=hashlib.sha256(altered).hexdigest() + ) + with pytest.raises(ValueError, match="FINGERPRINT"): + policy.fiscal_leaf_policy_document( + replace(pinned, payload=pinned.payload + b" ") + ) + + +@pytest.mark.parametrize( + "field", + [ + "rationale", + "reviewer", + "source_issue", + "value", + "period", + "roots", + "affected_roots", + "affected_programs", + "reform_sensitivity", + "interpretation", + ], +) +def test_missing_assumption_review_or_scope_refuses_without_engine(field): + raw = document() + del raw["entries"]["scenario_multiplier"][field] + with pytest.raises(ValueError, match="ASSUMPTION_FIELDS"): + load(raw) + + +@pytest.mark.parametrize( + "value", + [None, "engine default", {"default": True}, [0], float("nan"), float("inf"), 2**64], +) +def test_only_finite_bounded_numeric_or_boolean_literals_are_supported(value): + raw = document() + raw["entries"]["scenario_multiplier"]["value"] = value + with pytest.raises((ValueError, TypeError)): + load(raw) + + +@pytest.mark.parametrize( + "changed", + [ + "inactive", + "wrong_period", + "wrong_roots", + "empty_programs", + "empty_rationale", + "unknown_field", + ], +) +def test_inactivity_or_invalid_review_scope_never_becomes_an_allowlist(changed): + raw = document() + entry = raw["entries"]["scenario_multiplier"] + if changed == "inactive": + entry["kind"] = "inactive" + elif changed == "wrong_period": + entry["period"] = 2026 + elif changed == "wrong_roots": + entry["roots"] = ["other_root"] + elif changed == "empty_programs": + entry["affected_programs"] = [] + elif changed == "empty_rationale": + entry["rationale"] = " " + else: + entry["allow_missing"] = True + with pytest.raises(ValueError, match="FISCAL_LEAF_POLICY"): + load(raw) + + +def test_exact_closure_coverage_and_entity_are_required(mocked_model): + for change, reason in ( + ("missing", "UNPOLICIED"), + ("extra", "STALE"), + ("entity", "ENTITY"), + ): + raw = document() + if change == "missing": + del raw["entries"]["amount"] + elif change == "extra": + raw["entries"]["unrelated"] = { + "kind": "producer", + "entity": "person", + "producer": "fixture", + } + else: + raw["entries"]["amount"]["entity"] = "household" + with pytest.raises(ValueError, match=reason): + stage.fiscal_measurement_node(**arguments(load(raw))) + + +def test_exact_root_period_and_supplied_assumption_collision(mocked_model): + params = arguments(load()) + params["period"] = 2026 + with pytest.raises(ValueError, match="POLICY_PERIOD"): + stage.fiscal_measurement_node(**params) + params = arguments(load()) + params["model_outputs"] = ("different_root",) + with pytest.raises(ValueError, match="POLICY_ROOTS"): + stage.fiscal_measurement_node(**params) + params = arguments(load()) + params["input_columns"]["tax_unit"] += ("scenario_multiplier",) + with pytest.raises(ValueError, match="ASSUMPTION_COLUMN_COLLISION"): + stage.fiscal_measurement_node(**params) + + +def test_default_remains_strict_and_explicit_producers_keep_unknownness(mocked_model): + with pytest.raises(ValueError, match="UNDECLARED_MODEL_LEAF"): + stage.fiscal_measurement_node(**arguments()) + value = frame() + value.table("person")["scenario_multiplier"] = 0.0 + params = arguments() + params["input_columns"]["person"] += ("scenario_multiplier",) + default_node = stage.fiscal_measurement_node(**params) + assert default_node == stage.fiscal_measurement_node(**params, leaf_policy=None) + default_result = stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"] + ).run(context(params, value)) + assert mocked_model[0]["materialize"] == 1 + assert "leaf_policy" not in json.loads(default_node.params["model_contract"]) + assert mocked_model[0]["defaults"] == 0 + params["leaf_policy"] = load(document(assumption=False)) + result = stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=params["leaf_policy"] + ).run(context(params, value)) + np.testing.assert_array_equal( + stage.decode_fiscal_measurement( + default_result.artifacts["measurement"] + ).matrix.toarray(), + stage.decode_fiscal_measurement( + result.artifacts["measurement"] + ).matrix.toarray(), + ) + value.table("person").loc[0, "scenario_multiplier"] = np.nan + with pytest.raises(ValueError, match="MISSING_MODEL_INPUT"): + stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=params["leaf_policy"] + ).run(context(params, value)) + assert mocked_model[0]["materialize"] == 2 + + +def test_assumption_policy_binds_literal_review_and_engine_record_but_cannot_execute( + mocked_model, +): + params = arguments(load()) + c = context(params) + model = json.loads(c.params["model_contract"]) + assert model["leaf_policy"]["document"] == document() + assert model["leaf_policy"]["engine_records"]["scenario_multiplier"]["default"] == { + "available": True, + "value": 0.0, + } + with pytest.raises(ValueError, match="ASSUMPTION_PARENT_ADMISSION_UNSUPPORTED"): + stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=params["leaf_policy"] + ).run(c) + assert mocked_model[0]["materialize"] == 0 + changed = document() + changed["entries"]["scenario_multiplier"]["value"] = 2.0 + assert ( + stage.fiscal_measurement_node(**arguments(load(changed))).normative() + != c.node.normative() + ) + changed["entries"]["scenario_multiplier"]["value"] = 1.0 + changed["entries"]["scenario_multiplier"]["reviewer"] = "Another reviewer" + assert ( + stage.fiscal_measurement_node(**arguments(load(changed))).normative() + != c.node.normative() + ) + mocked_model[1]["value"] = 0.5 + assert stage.fiscal_measurement_node(**params).normative() != c.node.normative() + + +@pytest.mark.parametrize( + "values", [[1.0, 2.0, 3.0, 4.0], [np.nan] * 4, [np.nan, 2.0, 3.0, 4.0]] +) +def test_full_parent_validator_catches_omitted_and_partially_unknown_columns(values): + value = frame() + value.table("person")["scenario_multiplier"] = values + assert "scenario_multiplier" not in declaration()["input_columns"]["person"] + with pytest.raises(ValueError, match="ASSUMPTION_COLUMN_COLLISION"): + policy.validate_fiscal_leaf_policy_population(value, load()) + with pytest.raises(ValueError, match="ASSUMPTION_COLUMN_COLLISION"): + policy.private_fiscal_assumption_frame(value, load()) + + +def test_private_materialization_never_modifies_or_aliases_the_supplied_population(): + value = frame() + before = {e: value.table(e).copy(deep=True) for e in value.entities} + policy.validate_fiscal_leaf_policy_population(value, load()) + work = policy.private_fiscal_assumption_frame(value, load()) + np.testing.assert_array_equal(work.table("person").scenario_multiplier, np.ones(4)) + work.table("person").loc[0, "amount"] = 999.0 + for entity in value.entities: + pd.testing.assert_frame_equal(value.table(entity), before[entity]) + np.testing.assert_array_equal( + work.weights_for("household").values, value.weights_for("household").values + ) + + +def test_affected_roots_must_include_every_declared_consumer(mocked_model): + raw = document() + raw["roots"].append("other_root") + raw["entries"]["scenario_multiplier"]["roots"].append("other_root") + params = arguments(load(raw)) + params["model_outputs"] = tuple(raw["roots"]) + with pytest.raises(ValueError, match="ASSUMPTION_AFFECTED_ROOTS"): + stage.fiscal_measurement_node(**params) + raw["entries"]["scenario_multiplier"]["affected_roots"] = list(raw["roots"]) + params["leaf_policy"] = load(raw) + stage.fiscal_measurement_node(**params) + + +@pytest.mark.parametrize( + "dtype,value", + [("bool", 1), ("int", True), ("int", 1.5), ("float", True), ("str", 1)], +) +def test_engine_metadata_prevents_literal_type_coercion(dtype, value): + raw = document() + raw["entries"]["scenario_multiplier"]["value"] = value + raw = policy.fiscal_leaf_policy_document(load(raw)) + metadata = { + "scenario_multiplier": { + "name": "scenario_multiplier", + "entity": "person", + "dtype": dtype, + "period": "year", + } + } + with pytest.raises(ValueError, match="ENGINE_LITERAL_TYPE"): + policy.fiscal_assumption_engine_records(raw, metadata=metadata, defaults={}) + + +def test_final_model_fence_rejects_changed_policy_after_materialization( + mocked_model, monkeypatch +): + raw = document(assumption=False) + value = frame() + value.table("person")["scenario_multiplier"] = 0.0 + params = arguments(load(raw)) + params["input_columns"]["person"] += ("scenario_multiplier",) + kernel = stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=params["leaf_policy"] + ) + original = stage.policyengine_us.PolicyEngineUSEngine.materialize + + def mutate(self, *args, **kwargs): + outputs = original(self, *args, **kwargs) + changed = document(assumption=False) + changed["entries"]["amount"]["producer"] = "different.source" + kernel.leaf_policy = load(changed) + return outputs + + monkeypatch.setattr( + stage.policyengine_us.PolicyEngineUSEngine, "materialize", mutate + ) + with pytest.raises(ValueError, match="MODEL_DRIFT"): + kernel.run(context(params, value)) + + +def _source_node(*, has_scenario): + outputs = ( + Owned("person", "amount", "float64"), + Owned("person", "keep", "bool"), + Owned("tax_unit", "tax_unit_amount", "float64"), + Owned("household", "state_fips", "int64"), + Owned("household", "congressional_district_geoid", "int64"), + ) + if has_scenario: + outputs += (Owned("person", "scenario_multiplier", "float64"),) + return Node( + "invented", + InventedSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=outputs, + ) + + +def test_literal_policy_and_engine_defaults_change_actual_graph_key(mocked_model): + create = _source_node(has_scenario=False) + keys, implementation_hashes = [], [] + for value in (1.0, 2.0, 1.0): + raw = document() + raw["entries"]["scenario_multiplier"]["value"] = value + if len(keys) == 2: + mocked_model[1]["value"] = 0.5 + pinned = load(raw) + measure = stage.fiscal_measurement_node(**arguments(pinned)) + compiled = compile_graph( + Graph("us", (SourceRef("fixture", "frame-store"),), (create, measure)) + ) + kernel = stage.FiscalMeasurementKernel( + model_outputs=("invented_fiscal",), leaf_policy=pinned + ) + implementation = kernel.implementation_hash() + implementation_hashes.append(implementation) + keys.append( + node_key( + compiled, + measure.id, + {create.id: "a" * 64}, + implementation, + {}, + kernel_capabilities=kernel.capabilities, + ) + ) + assert len(set(keys)) == len(set(implementation_hashes)) == 3 + assert mocked_model[0]["materialize"] == 0 + + +def test_explicit_producer_actual_graph_cold_required_and_policy_rekey( + tmp_path, mocked_model, monkeypatch +): + original = frame() + original.table("person")["scenario_multiplier"] = 0.0 + raw = document(assumption=False) + pinned = load(raw) + params = arguments(pinned) + params["input_columns"]["person"] += ("scenario_multiplier",) + create = _source_node(has_scenario=True) + measure = stage.fiscal_measurement_node(**params) + sources = (SourceRef("fixture", "frame-store"),) + compiled = compile_graph(Graph("us", sources, (create, measure))) + registry = KernelRegistry() + registry.register(InventedSource()) + registry.register( + stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=pinned + ) + ) + store = ContentStore(tmp_path / "store") + source = store.put_frame( + hashlib.sha256(b"invented-policy-parent").hexdigest(), original + ) + run_args = {"sources": {"fixture": source}, "kernels": registry, "store": store} + cold = run_graph(compiled, **run_args) + cold_bytes = store.load_bytes(cold.node(measure.id).opaque_artifacts["measurement"]) + + def forbidden(*args, **kwargs): + pytest.fail("required replay executed country model") + + monkeypatch.setattr( + stage.policyengine_us.PolicyEngineUSEngine, "materialize", forbidden + ) + warm = run_graph(compiled, **run_args, resume="require") + assert all(node.store_hit for node in warm.nodes.values()) + assert cold.key == warm.key + assert cold_bytes == store.load_bytes( + warm.node(measure.id).opaque_artifacts["measurement"] + ) + for entity in original.entities: + pd.testing.assert_frame_equal( + warm.population(create.id).table(entity), original.table(entity) + ) + changed = document(assumption=False) + changed["entries"]["amount"]["producer"] = "other.source" + changed_policy = load(changed) + params["leaf_policy"] = changed_policy + changed_registry = KernelRegistry() + changed_registry.register(InventedSource()) + changed_registry.register( + stage.FiscalMeasurementKernel( + model_outputs=params["model_outputs"], leaf_policy=changed_policy + ) + ) + changed_graph = compile_graph( + Graph("us", sources, (create, stage.fiscal_measurement_node(**params))) + ) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph( + changed_graph, **{**run_args, "kernels": changed_registry}, resume="require" + ) From 2be80c2e925d4bc0f187a7adbf7b990db42789e4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:02:29 -0400 Subject: [PATCH 163/255] Classify reviewed US source graph modules and verify integration --- ...s-integrated-source-acceptance-20260912.md | 72 +++++++++++++++++++ .../tests/test_us_spine_blindness.py | 40 +++++++++++ 2 files changed, 112 insertions(+) create mode 100644 experiments/us-integrated-source-acceptance-20260912.md diff --git a/experiments/us-integrated-source-acceptance-20260912.md b/experiments/us-integrated-source-acceptance-20260912.md new file mode 100644 index 000000000..c01643a54 --- /dev/null +++ b/experiments/us-integrated-source-acceptance-20260912.md @@ -0,0 +1,72 @@ +# Integrated US source acceptance, 2026-09-12 + +The accepted current-income routing, interest and allocation-source changes +compose with the accepted fiscal target-snapshot host. This is an invented-source +integration check, not a native build, model evaluation, or release verdict. + +The integration preserves the full histories of current main +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`, the income source branch +`4a8f1d9edeacc99a36a9b2e43175081a8757f0fa`, and the fiscal snapshot branch +`13791d75a7fce94d12697b8d5ce9bdf46f83ebab`. The merges needed no production +source edits: each runtime/calibration source difference from one accepted +parent is byte-identical to the other parent. In particular, the routing +NamedTuple declarations, private interest-reader seam, and unpublished farm +allocation-code distinction coexist. + +## Runtime classification + +The source-spine inventory now includes twelve already reviewed graph modules. +Seven qualify source observations, validate original-person joins, or declare +those joins' inputs: ACS income anchors, ASEC interest, ASEC unemployment, +current survey amounts, the health source and recode/attachment helper, and the +health graph fragment. These receive explicit source-provenance ownership +comments. Neither attachment uses source channel to choose a PUF-detail model; +it verifies the source-person identity and attaches to the same two clones. + +Fiscal measurement, PUF route attachment, the PUF composition host, and the +combined amount/health host retain source-spine scanning. Their reviewed dynamic +selectors index exact geography fields, numeric masks, typed artifact names, +producer keys, or retained node histories. Sixteen direct counterexamples verify +that provenance columns and factories still fail in each of those four modules. +The dense fiscal solver needs no exception. Scanner functions and classes, +registered population operators, and the physical source-accessor gate remain +unchanged. + +## Validation + +The frozen combined suite passed **138 tests**: routing 88, interest 40, and +allocation meaning 10. The first launch stopped during collection because the +handoff's routing count was 64; no tests executed in that attempt. The second +launch corrected only the expected count and owned scratch paths, preserving +all test and production bytes and the existing audit-control functions. + +The passing run took 216.86 seconds wall time and 211.50 seconds CPU, with +405,897,216 bytes peak RSS. All 1,011 source/owned hashes and 15 resource hashes +were unchanged; there were no unexpected refusals and Torch thread counts were +1/1. Limits remained 300 CPU seconds, 600 wall seconds, 8 GiB RSS, no children, +no network and no country engine. The existing denied optional timezone-resource +probe produced the sole warning. Native inputs remained inaccessible. + +The exact passing receipt SHA256 is +`21d4c5723558ad539c2bee4a70d3296b004dea2b2fe951be599a1c07f5b680f6`; +JUnit SHA256 is +`c9d6bd0420417ca6d246b25fbd48701374877a4f2924504e245533c91bf16d44`. +Only the source-spine test roster changed after this frozen source run; runtime, +resource and selected source-test bytes did not change. + +The seven existing runtime source scans first demonstrated the unclassified +modules and source-owner/dynamic-selector findings. Their other four checks +already passed. The full scanner is CPU-heavy in the existing +`atomic_block_support.py`; its first 120-second capped attempt was retained +without claiming a result, and the completed red run took 260.66 seconds. The +classification changes do not optimize or weaken this scanner. All seven green +checks passed in 238.44 seconds, with unchanged roster-test bytes during the +run. An observational helper wrapper recorded module names, timing and the exact +returned findings; it did not alter scanner inputs or findings. + +Local evidence is retained under +`codex-integrated-source-acceptance-20260912`: source-v1 preserves the count +refusal, source-v2 holds the frozen passing acceptance, and separate roster +receipts, source-equivalence proof and direct counterexamples delimit the test +inventory change. Fiscal numerical/callback acceptance remains the separately +reviewed 13791 parent evidence; no duplicate fiscal or PUF fixture was run here. diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 06a52e9c8..3ed1f83e6 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -134,6 +134,15 @@ # qualifier beside it does; it routes no PUF-detail imputation and # models nothing. "current_asec_income_routing_source.py", + # Source-qualified graph additions reviewed 2026-09-12. These are + # original-record joins or declarations, not source-routed PUF models. + "current_acs_income_anchor_source.py", # ACS anchor -> exact original ids. + "current_asec_interest_source.py", # ASEC interest -> original money owner. + "current_asec_unemployment_source.py", # UC literal -> original ASEC ids. + "current_survey_amounts.py", # Validate origin join; fan out to both clones. + "current_survey_health_source.py", # Qualify exact ACS/ASEC source rosters. + "current_survey_health_coverage.py", # Validate the same two-clone join. + "graph_current_survey_health.py", # Declare the attachment's identity inputs. "current_social_security_source.py", # Source-owned SS totals per row. "current_survey_geography.py", # Origin roster and draw keys. "current_survey_predictors.py", # Source-qualified predictor prep. @@ -357,6 +366,30 @@ # listed separately in _SOURCE_SPINE_PROVENANCE_OWNERS with their reason. _US_LAUNCH_GRAPH_RUNTIME_MODULES = frozenset( { + # Qualified raw ACS INTP/RETP anchors, preserving literal knownness + "current_acs_income_anchor_source.py", + # Qualified ASEC interest components and unreconciled total diagnostics + "current_asec_interest_source.py", + # Qualified UC receipt/amount observations, preserving unknown zeros + "current_asec_unemployment_source.py", + # Current UC/health amount qualification and exact two-clone attachment + "current_survey_amounts.py", + # Pure current-coverage recodes and original-person clone attachment + "current_survey_health_coverage.py", + # Authenticated original ACS/ASEC health-coverage observations + "current_survey_health_source.py", + # Typed source, recode and attachment health graph fragment + "graph_current_survey_health.py", + # Grouped dense calibration over retained fiscal measurements + "graph_fiscal_dense_calibration.py", + # Measured fiscal inputs and exact retained population/target axes + "graph_fiscal_measurement.py", + # Source-blind PUF clone placement and original-channel preservation + "graph_puf55_route_attachment.py", + # Complete PUF composition and retained checked-output lifetime + "graph_survey_puf55.py", + # Current amount/health composition over the checked PUF parent + "graph_us_survey_enrichment.py", # Bounded, checksummed codec preserving target-only money source authority "_asec_current_money_codec.py", # Validate the measured person-signal summary shape before gate decisions @@ -3413,6 +3446,13 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: # accepted, and only for the listed modules. _REVIEWED_DYNAMIC_SELECTOR_MODULES = frozenset( { + # Reviewed 2026-09-12: exact geography fields/CSR masks, typed artifact + # names and producer keys, retained registry entries and node histories. + # These four remain scanned for provenance columns and accessor calls. + "graph_fiscal_measurement.py", + "graph_puf55_route_attachment.py", + "graph_survey_puf55.py", + "graph_us_survey_enrichment.py", "acs_native_coverage_binding.py", "acs_person_coverage_columns.py", "acs_population_catalogue.py", From bf7976d1bc4a050ee2d701fba18c49bc5047cdb0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 12:49:33 -0400 Subject: [PATCH 164/255] Qualify received and paid ASEC child-support observations --- ...current-asec-child-support-source.added.md | 1 + docs/us-current-asec-child-support-source.md | 79 +++ .../current_asec_child_support_source.py | 452 ++++++++++++++++++ ...st_us_current_asec_child_support_source.py | 321 +++++++++++++ 4 files changed, 853 insertions(+) create mode 100644 changelog.d/current-asec-child-support-source.added.md create mode 100644 docs/us-current-asec-child-support-source.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_child_support_source.py diff --git a/changelog.d/current-asec-child-support-source.added.md b/changelog.d/current-asec-child-support-source.added.md new file mode 100644 index 000000000..4b951e9a3 --- /dev/null +++ b/changelog.d/current-asec-child-support-source.added.md @@ -0,0 +1 @@ +Qualify retained ASEC child-support amounts, receipt and obligation answers, preserving paid NIU and unknown voluntary payments alongside original source and allocation metadata. diff --git a/docs/us-current-asec-child-support-source.md b/docs/us-current-asec-child-support-source.md new file mode 100644 index 000000000..c55593637 --- /dev/null +++ b/docs/us-current-asec-child-support-source.md @@ -0,0 +1,79 @@ +# Current ASEC child-support observations + +`qualify_current_asec_child_support(preparation)` in +`microcosm.build.us_runtime.current_asec_child_support_source` qualifies the +original received- and paid-support observations from the retained 2025 ASEC +person source. The dollar amounts refer to the current 2024 income cohort. + +The source is the [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA-256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The qualifier reuses the exact packaged monetary domains and temporal +authority for `CSP_VAL` and `CHSP_VAL`. It adds source questions and flags from +the same dictionary, with their positions, widths and printed universes. + +| Fields | Source meaning | One-based PDF page | +| --- | --- | --- | +| `CSP_VAL`, `CSP_YN` | Amount received and actual receipt answer; receipt universe age 15+ | 52 | +| `CHSP_VAL`, `CHSP_YN` | Annual amount paid and whether payment is required; the latter is an obligation question | 52 | +| `CHELSEW_YN` | Whether a child lives outside the household; age 15+ | 52 | +| `I_CHELSEWYN`, `I_CHSPVAL`, `I_CHSPYN`, `I_CSPVAL`, `I_CSPYN` | Published allocation flags, using the shared I_ANNVAL code meanings | 54 | +| `TCHSP_VAL`, `TCSP_VAL` | Published top-code indicators for positive amounts | 59 | + +## Received amounts + +The printed zero for `CSP_VAL` means none or NIU. An in-universe receipt-no +answer plus zero establishes known nonreceipt. A receipt-yes answer plus zero +remains ambiguous; the qualifier does not complete it with a zero-dollar +observation. Positive receipt amounts are retained, including valid allocated +observations. Under-15 NIU, missing answers, malformed literals and +contradictory receipt/amount pairs remain distinguishable and unknown. + +## Payments and obligations + +`CHSP_YN` asks whether the person is required to pay child support. A no +answer does not rule out voluntary payment. `CHSP_VAL` describes actual annual +payments and prints zero as NIU, so every paid zero remains a published code +with an unknown canonical payment amount. No obligation answer turns it into +an observed zero. + +The paid amount's universe is `CHSP_YN = 1`. The obligation question itself +prints only the bare field `CHELSEW_YN`, while the allocation flag `I_CHSPYN` +prints `CHELSEW_YN = 1`. That discrepancy remains in the evidence. The +qualifier conservatively accepts positive payments on the observed +intersection age 15+, child-elsewhere yes and required-to-pay yes. Other +routes keep the published positive amount and an explicit unresolved status. +This intersection is a qualification choice, not a correction of the printed +universe or proof that voluntary payments do not occur. + +The obligation and child-elsewhere entries do not independently state a +reference year. Their source context is retained; the qualifier does not +issue a current-at-interview or eligibility input from those answers. + +## Provenance and consumption + +The result has two complete tables: selected ASEC people on the common +preparation's person identities, and the original current-cohort literals on +native identities. Source amount validity, statuses and zero origin are +retained alongside the descriptive projection. Allocation/top-code literal +values and parse statuses remain separate from amount knownness. All-zero +allocation flags describe the published codes; they are not a blanket claim +of unallocated or respondent-reported data. + +The qualifier takes the actual authenticated preparation, captures the +pinned original member once through the shared bounded literal reader, +checks its bytes and complete native-key roster, verifies household/line/raw +age coordinates, and compares both source amounts to their retained +current-money values with exact validity and float64 bits. Source row order +does not define person identity. + +`CurrentAsecChildSupportValues` is descriptive transport, not an issuer. +`child_support_values_seal` covers both complete tables, axes, nullable masks +and backing storage, float bits and detached evidence. The actual preparation +is requalified after source I/O, followed by final owner and physical-value +checks. A consumer must retain the actual owner, requalify immediately before +consumption and after its own last relevant I/O, and compare complete value +seals before returning or exporting a successor. Copied data or a matching +digest cannot grant source authority. + +This module assigns no engine leaf, changes no clone, fits no ACS model and +issues no new source admission or release eligibility. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py new file mode 100644 index 000000000..ba1cd9533 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py @@ -0,0 +1,452 @@ +"""Observed child support received, paid and required-to-pay source answers. + +Obligation is not payment. Returned values are descriptive and issue no source +authority; a consuming host retains/requalifies the actual preparation. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-child-support-source.v1" +AMOUNT_FIELDS = ("CSP_VAL", "CHSP_VAL") +# Width, position, PDF page, printed page, printed universe, zero label. +RESPONSE_ENTRIES = { + "CHELSEW_YN": (1, 711, 52, "6C-31", "All Persons aged 15+", "Niu"), + "CHSP_YN": (1, 717, 52, "6C-31", "CHELSEW_YN", "Niu"), + "CSP_YN": (1, 723, 52, "6C-31", "All Persons aged 15+", "Niu"), +} +RESPONSE_MEANINGS = { + "CHELSEW_YN": "has a child living outside the household", + "CHSP_YN": "required to pay child support; not whether payment occurred", + "CSP_YN": "received child support payments", +} +ALLOCATION_ENTRIES = { + "I_CHELSEWYN": (1, 807, 54, "6C-33", "CHELSEW_YN > 0"), + "I_CHSPVAL": (1, 808, 54, "6C-33", "CHSP_YN = 1"), + "I_CHSPYN": (1, 809, 54, "6C-33", "CHELSEW_YN = 1"), + "I_CSPVAL": (1, 810, 54, "6C-33", "CSP_YN = 1"), + "I_CSPYN": (1, 811, 54, "6C-33", "CSP_YN > 0"), +} +TOPCODE_ENTRIES = { + "TCHSP_VAL": (1, 899, 59, "6C-38", "CHSP_VAL > 0"), + "TCSP_VAL": (1, 901, 59, "6C-38", "CSP_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RESPONSE_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) +OBLIGATION_UNIVERSE_NOTE = ( + "CHSP_YN prints bare CHELSEW_YN, whereas I_CHSPYN prints CHELSEW_YN = 1. " + "The bare field is not silently compiled as a general predicate. Positive " + "paid amounts are qualified only on the observed intersection of age15+, " + "CHELSEW_YN=1 and CHSP_YN=1; other routes remain explicit and unresolved." +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_CHILD_SUPPORT_SOURCE_" + reason) + + +@lru_cache(maxsize=1) +def amount_entries(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + entries = {} + for field in json.loads(payload)["fields"]: + if field["name"] not in AMOUNT_FIELDS: + continue + vintage = next( + v + for v in field["vintages"] + if v["income_year"] == routing.CURRENT_INCOME_YEAR + ) + require( + vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + entries[field["name"]] = { + "width": vintage["ascii_length_as_printed"], + "position": vintage["ascii_position_as_printed"], + "pdf_page": vintage["pdf_page_1based"], + "printed_page": vintage["printed_page"], + "universe": vintage["universe_as_printed"], + "domain": field["domain"], + "period": field["period"], + "temporal_authority": field["temporal_authority"], + } + require(set(entries) == set(AMOUNT_FIELDS), "AMOUNT_ROSTER") + return entries + + +def _amount_pair(token, entry): + bounds = entry["domain"]["encoded_range_inclusive"] + return routing.literal_code( + token, range(bounds["minimum"], bounds["maximum"] + 1), width=entry["width"] + ) + + +def _paid_status(age, elsewhere, obligation, amount): + value, parse_status = amount + if parse_status != "in_printed_range": + return ( + "missing_amount" if parse_status == "missing" else "invalid_amount_literal" + ) + if age < 15: + return ( + "outside_reporting_universe" + if elsewhere[0] == obligation[0] == 0 and value == 0 + else "contradictory_outside_reporting_universe" + ) + if value == 0: + return "declared_niu_amount" + if elsewhere[1] != "in_printed_range": + return "unresolved_child_elsewhere_literal" + if obligation[1] != "in_printed_range": + return ( + "missing_obligation_literal" + if obligation[1] == "missing" + else "invalid_obligation_literal" + ) + if elsewhere[0] != 1: + return "unresolved_child_elsewhere_route" + if obligation[0] != 1: + return "payment_outside_published_obligation_universe" + return "observed_positive_payment" + + +def project_child_support_literals(ordered): + """Descriptive projection only; arbitrary literals grant no authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + out = pd.DataFrame(index=range(len(ordered))) + parsed, codes = {}, {} + for name, entry in amount_entries().items(): + pairs = [_amount_pair(token, entry) for token in ordered[name]] + parsed[name] = pairs + out[name + "_literal"] = pd.array(ordered[name].tolist(), dtype="string") + out[name + "_literal_status"] = pd.array( + [status for _, status in pairs], dtype="string" + ) + out[name + "_published_amount"] = pd.array( + [value for value, _ in pairs], dtype="Float64" + ) + for name, entry in RESPONSE_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, + ordered[name], + routing.RECEIPT_CODE_DOMAIN, + {0: entry[5], 1: "Yes", 2: "No"}, + width=entry[0], + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name in AMOUNT_FIELDS: + labels = [] + for i, age in enumerate(ages): + value, status = parsed[name][i] + if name == "CHSP_VAL": + label = _paid_status( + age, codes["CHELSEW_YN"][i], codes["CHSP_YN"][i], parsed[name][i] + ) + elif status not in ("missing", "in_printed_range"): + label = "invalid_amount_literal" + else: + kind = ( + "missing" + if value is None + else ("zero" if value == 0 else "nonzero") + ) + label = routing.receipt_status( + bool(age >= 15), + codes["CSP_YN"][i], + kind, + net_measure=False, + zero_is_dollars=False, + ) + labels.append(label) + known = np.array( + [ + label in (*routing.KNOWN_AMOUNT_STATUSES, "observed_positive_payment") + for label in labels + ] + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + flags = {} + for name, entry in ALLOCATION_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, ordered[name], routing.ALLOCATION_ANNVAL_CODES, width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + flags[name] = (values, statuses) + out["received_allocation_origin"] = routing._allocation_origin( + {name: flags[name] for name in ("I_CSPVAL", "I_CSPYN")}, unflagged=False + ) + out["paid_allocation_origin"] = routing._allocation_origin( + {name: flags[name] for name in ("I_CHELSEWYN", "I_CHSPVAL", "I_CHSPYN")}, + unflagged=False, + ) + for name, entry in TOPCODE_ENTRIES.items(): + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=entry[0]) + out = pd.concat([out, frame], axis=1) + out["source_age"] = ages + out["obligation_implies_payment"] = False + out["voluntary_payment_absence_known"] = False + return out + + +@dataclass(frozen=True) +class CurrentAsecChildSupportValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def child_support_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecChildSupportValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="child support columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="child support" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-child-support-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_amount(ready, positions, name, literals): + field = ready.field(name) + entry = amount_entries()[name] + pairs = [_amount_pair(t, entry) for t in literals] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "AMOUNT_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "AMOUNT_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "AMOUNT_BITS", + ) + return field + + +def qualify_current_asec_child_support(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == routing.CURRENT_INCOME_YEAR + and header["semantic"] == "annual_current_money" + and native_document["source_year"] + == native_document["income_year"] + == routing.CURRENT_INCOME_YEAR + and native_document["survey_year"] == (routing.CURRENT_INCOME_YEAR + 1), + "PERIOD", + ) + pins = [ + p for p in routing.coverage._MEMBER_PINS if p[0] == routing.CURRENT_INCOME_YEAR + ] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero( + np.asarray(parent.scope.person_years) == routing.CURRENT_INCOME_YEAR + ) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + fields = { + name: _compare_amount(ready, positions, name, ordered[name]) + for name in AMOUNT_FIELDS + } + basis = project_child_support_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for field_name, field in fields.items(): + for name in ("statuses", "validity", "zero_origin"): + basis[field_name + "_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "response_entries": RESPONSE_ENTRIES, + "response_meanings": RESPONSE_MEANINGS, + "obligation_universe_note": OBLIGATION_UNIVERSE_NOTE, + "response_period_note": ( + "CHSP_YN and CHELSEW_YN do not state an independent reference year " + "in their printed entries. Their 2025 source context is preserved; " + "no new current-at-interview or policy eligibility input is issued." + ), + "allocation_entries": ALLOCATION_ENTRIES, + "topcode_entries": TOPCODE_ENTRIES, + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": routing.CURRENT_INCOME_YEAR, + "survey_year": (routing.CURRENT_INCOME_YEAR + 1), + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "obligation_implies_payment": False, + "voluntary_payment_absence_known": False, + "paid_niu_completed_with_zero": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecChildSupportValues( + out, literals, json.loads(json.dumps(evidence)) + ) + seal = child_support_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(child_support_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py new file mode 100644 index 000000000..a3b4f0121 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py @@ -0,0 +1,321 @@ +"""Child-support source semantics and invented actual-owner boundaries.""" + +import copy +import hashlib +import shutil + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_child_support_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "CSP_VAL": "1200", + "CSP_YN": "1", + "CHSP_VAL": "2000", + "CHSP_YN": "1", + "CHELSEW_YN": "1", + **changes, + } + + +def project(**changes): + return owner.project_child_support_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "receipt,amount,status,known", + [ + ("1", "1200", "known_receipt", True), + ("1", "0", "ambiguous_recipient_zero", False), + ("2", "0", "known_nonreceipt", True), + ("2", "1200", "contradictory_no_nonzero", False), + ("0", "0", "niu", False), + ("", "1200", "missing_receipt_literal", False), + ], +) +def test_received_support_uses_receipt_but_does_not_complete_recipient_zeros( + receipt, amount, status, known +): + value = project(CSP_YN=receipt, CSP_VAL=amount) + assert value.CSP_VAL_reporting_status == status + assert bool(value.CSP_VAL_amount_known) is known + + +@pytest.mark.parametrize("obligation", ["0", "1", "2", ""]) +def test_paid_zero_is_niu_under_every_obligation_answer(obligation): + value = project(CHSP_VAL="0", CHSP_YN=obligation) + assert value.CHSP_VAL_published_amount == 0 + assert value.CHSP_VAL_reporting_status == "declared_niu_amount" + assert not value.CHSP_VAL_amount_known and pd.isna(value.CHSP_VAL_amount) + assert not value.voluntary_payment_absence_known + assert not value.obligation_implies_payment + + +@pytest.mark.parametrize( + "elsewhere,obligation,status", + [ + ("1", "1", "observed_positive_payment"), + ("1", "2", "payment_outside_published_obligation_universe"), + ("1", "0", "payment_outside_published_obligation_universe"), + ("2", "1", "unresolved_child_elsewhere_route"), + ("0", "1", "unresolved_child_elsewhere_route"), + ("", "1", "unresolved_child_elsewhere_literal"), + ("1", "", "missing_obligation_literal"), + ("1", "x", "invalid_obligation_literal"), + ], +) +def test_positive_paid_cells_remain_visible_when_routing_is_unresolved( + elsewhere, obligation, status +): + value = project(CHELSEW_YN=elsewhere, CHSP_YN=obligation) + assert value.CHSP_VAL_reporting_status == status + assert value.CHSP_VAL_published_amount == 2000 + assert bool(value.CHSP_VAL_amount_known) is (status == "observed_positive_payment") + + +@pytest.mark.parametrize("field", ["CSP_VAL", "CHSP_VAL"]) +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.5", "malformed"), + ("NA", "malformed"), + ], +) +def test_amount_missing_malformed_and_negative_literals_remain_distinct( + field, token, status +): + value = project(**{field: token}) + assert value[field + "_literal"] == token + assert value[field + "_literal_status"] == status + assert not value[field + "_amount_known"] + + +def test_under15_observations_do_not_become_analytical_zeros(): + value = project( + A_AGE="14", CSP_VAL="0", CSP_YN="0", CHSP_VAL="0", CHSP_YN="0", CHELSEW_YN="0" + ) + for name in owner.AMOUNT_FIELDS: + assert value[name + "_reporting_status"] == "outside_reporting_universe" + assert not value[name + "_amount_known"] + + +def test_allocated_values_remain_known_and_missing_flags_remain_unknown(): + value = project(I_CHSPVAL="4", I_CSPVAL="", TCHSP_VAL="1") + assert value.CHSP_VAL_amount == 2000 and value.CHSP_VAL_amount_known + assert value.paid_allocation_origin == "publisher_allocated" + assert value.received_allocation_origin == "allocation_flag_not_populated" + assert value.I_CSPVAL_literal_status == "missing" + assert value.TCHSP_VAL_code == 1 + + +def test_pinned_domains_and_universe_discrepancy_are_preserved(): + entries = owner.amount_entries() + assert entries["CHSP_VAL"]["domain"]["zero_semantics"] == "niu" + assert entries["CHSP_VAL"]["domain"]["valid_dollar_range_inclusive"]["minimum"] == 1 + assert owner.RESPONSE_ENTRIES["CHSP_YN"][4] == "CHELSEW_YN" + assert owner.ALLOCATION_ENTRIES["I_CHSPYN"][4] == "CHELSEW_YN = 1" + assert "bare" in owner.OBLIGATION_UNIVERSE_NOTE + + +def test_nullable_seal_covers_values_masks_and_hidden_backing(): + raw = pd.DataFrame([row(CHSP_VAL="0")]) + values = owner.CurrentAsecChildSupportValues( + owner.project_child_support_literals(raw), raw, {} + ) + original = owner.child_support_values_seal(values) + assert owner.child_support_values_seal(copy.deepcopy(values)) == original + for change in ("bit", "mask", "hidden"): + other = copy.deepcopy(values) + array = ( + other.person.CSP_VAL_amount.array + if change == "bit" + else other.person.CHSP_VAL_amount.array + ) + if change == "bit": + array._data[0] = np.nextafter(array._data[0], np.inf) + elif change == "mask": + array._mask[0] = False + else: + array._data[0] = 17.0 + assert owner.child_support_values_seal(other) != original + + +def child_support_arguments(tmp_path, monkeypatch): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_CHSPVAL="4"), + 106: row( + A_AGE="14", + CSP_VAL="0", + CSP_YN="0", + CHSP_VAL="0", + CHSP_YN="0", + CHELSEW_YN="0", + ), + 107: row(A_AGE="70", CSP_VAL="0", CSP_YN="2", CHSP_VAL="0", CHSP_YN="2"), + 108: row(CSP_VAL="0", CHSP_VAL="0"), + } + changes = {} + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.AMOUNT_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RESPONSE_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "child-support-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **child_support_arguments(tmp_path, monkeypatch) + ) + + +def test_actual_owner_keeps_obligation_distinct_and_refuses_copied_owner( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_child_support(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "CHSP_VAL_amount"] == 2000 + assert values.loc[105, "CSP_VAL_amount"] == 1200 + assert values.loc[107, "CSP_VAL_amount"] == 0 + assert not values.loc[107, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CHSP_VAL_amount_known"] + assert not values.loc[108, "CSP_VAL_amount_known"] + assert not result.evidence["paid_niu_completed_with_zero"] + assert owner.child_support_values_seal( + owner.qualify_current_asec_child_support(parent) + ) == owner.child_support_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(copy.copy(parent)) + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "CSP_VAL", "CHSP_VAL"]) +def test_actual_owner_requires_original_source_identity_and_amounts( + tmp_path, monkeypatch, field +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_child_support(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.child_support_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.CSP_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.CHSP_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("CHSP_YN") + ] = "2" + else: + value.evidence["paid_niu_completed_with_zero"] = True + return before + + monkeypatch.setattr(owner, "child_support_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_child_support(parent) From 3b201755702a9631448033970a337fe55b84ea0c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:15:38 -0400 Subject: [PATCH 165/255] Define shared columns for the property income bridge --- .../build/us_runtime/property_income_constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py b/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py new file mode 100644 index 000000000..1b1456ad4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/property_income_constants.py @@ -0,0 +1,10 @@ +"""Canonical columns for the declared ACS/ASEC property-income bridge.""" + +PROPERTY_COMPONENTS = ( + "property_ordinary_interest", + "property_retirement_interest", + "property_dividends", + "property_broad_receipts", +) +PROPERTY_REPORTED_TOTAL = "property_reported_total" +PROPERTY_DRAW_COLUMNS = tuple("draw_" + name for name in PROPERTY_COMPONENTS) From 90bc1bfbc2592530c8fe914f5ed60b3bc7b2e1e3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:17:44 -0400 Subject: [PATCH 166/255] Qualify current ASEC dividends and survivor property routes --- .../us-current-asec-dividend-source.added.md | 1 + docs/us-current-asec-dividend-source.md | 109 ++++ .../current_asec_dividend_source.py | 519 ++++++++++++++++++ .../test_us_current_asec_dividend_source.py | 497 +++++++++++++++++ 4 files changed, 1126 insertions(+) create mode 100644 changelog.d/us-current-asec-dividend-source.added.md create mode 100644 docs/us-current-asec-dividend-source.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_dividend_source.py diff --git a/changelog.d/us-current-asec-dividend-source.added.md b/changelog.d/us-current-asec-dividend-source.added.md new file mode 100644 index 000000000..a7d453233 --- /dev/null +++ b/changelog.d/us-current-asec-dividend-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC dividend observations and survivor source routes from the retained survey owner, preserving ambiguous zeros, allocation-codebook conflicts and source flags without assigning tax treatment or selecting donors. diff --git a/docs/us-current-asec-dividend-source.md b/docs/us-current-asec-dividend-source.md new file mode 100644 index 000000000..5439e50c7 --- /dev/null +++ b/docs/us-current-asec-dividend-source.md @@ -0,0 +1,109 @@ +# Current ASEC dividend and survivor routes + +`current_asec_dividend_source.qualify_current_asec_dividend(preparation)` +qualifies current ASEC dividend observations and the recorded survivor-income +source routes that could overlap with a broad property-income concept. It +reads the original 2025 person member, whose annual money amounts refer to +2024. It returns source observations and provenance, without modeling ACS +values, assigning tax treatment or authorizing a dataset release. + +The [2025 ASEC public-use dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +is pinned to SHA-256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The dividend domain also comes from the existing pinned +`asec_current_money_domains_v1.json`. Page numbers below are one-based PDF +pages; positions and widths refer to the printed ASCII layout. + +| Field | Position / width | Page | Printed universe | +| --- | --- | --- | --- | +| `DIV_VAL` | 478 / 6 | 45 | `DIV_YN = 1` | +| `DIV_YN` | 484 / 1 | 45 | Age 15+ | +| `SUR_SC1`, `SUR_SC2` | 648 / 2, 650 / 2 | 50 | `SUR_YN = 1` | +| `SUR_YN` | 664 / 1 | 50 | Age 15+ | +| `I_DIVVAL` | 819 / 1 | 54 | `DIV_YN = 1` | +| `I_DIVYN` | 820 / 1 | 54 | Age 15+ | +| `TDIV_VAL` | 905 / 1 | 59 | `DIV_VAL > 0` | +| `TRNT_VAL` | 917 / 1 | 60 | `RNT_VAL > 0` | + +## Dividend values and source flags + +The projection preserves the exact `DIV_VAL_literal`, parsed +`DIV_VAL_published_amount`, literal and reporting statuses, and a separate +`DIV_VAL_amount_known` mask. `DIV_VAL_amount` contains only the values whose +reporting status is resolved. An in-universe receipt-yes and positive amount +is known; receipt-no and zero establish known nonreceipt. Receipt-yes and +zero remain ambiguous because the dictionary combines none and NIU in its +zero code. Missing literals, NIU and contradictory combinations retain +their source evidence and an unknown canonical amount. No analytical zero +is inferred solely from age. + +Source flags never change this knownness or select donors. A known published +cell can still have publisher allocation or topcoding metadata. It is not a +claim about an uncensored latent amount or a respondent-only report. +`I_DIVVAL` refers to the allocation meanings under `I_ANNVAL` on page 53. +`TDIV_VAL` and `TRNT_VAL` retain codes 0 (not topcoded) and 1 (topcoded), +including missing or malformed flags. `TRNT_VAL` is retained only for future +diagnostics: this qualifier does not read `RNT_VAL` or evaluate that flag's +positive-rent universe. + +The dictionary is internally inconsistent for `I_DIVYN`: its header prints +0–1, while its Values section refers to the `I_ANNVAL` codes 0–9. The result +preserves the literal and code, physical syntax status, separate +`I_DIVYN_header_range_status` and `I_DIVYN_referenced_values_status`, and an +explicit `I_DIVYN_codebook_status`. Codes 2–9 therefore retain the conflict; +they are not silently dropped or admitted under a preferred interpretation. + +## Survivor source routes + +`SUR_YN`, `SUR_SC1` and `SUR_SC2` each retain raw literals, parsed codes, +literal status and source labels. Code 8 identifies regular payments from +estates or trusts. It signals a possible property-income overlap, without +establishing an amount or tax classification. Code 10 means other or don't +know: its literal is valid, but its property route remains unspecified. + +`survivor_property_route_clear` is nullable. It is true only for an +in-universe known receipt-no with both source slots NIU, or receipt-yes with +every slot readable, at least one active slot, and neither code 8 nor code +10. Receipt-yes with an explicit code 8 sets it false. Missing or unreadable +receipts, unresolved slots, contradictory nonreceipt routes, NIU and code 10 +leave it unknown. Under-15 rows remain unknown; incomplete under-15 literals +are labeled unresolved rather than contradictory. Independent booleans +retain the presence of codes 8 and 10 even when the receipt is unresolved. + +This clearance describes the recorded source routes. It does not establish +that the person has no property income. No survivor amount is captured, no +donor exclusion is applied, and a missing slot is never an observed zero +amount. A later measurement bridge must explicitly choose and validate how +these routes affect donor eligibility. + +## Retained source boundaries + +The qualifier borrows the original authenticated survey preparation and its +retained native ASEC/current-money owner. Before capture it checks the live +money domain against the pinned dividend contract. The domain reader +requires a unique field and current vintage, refuses unsupported negative, +missing-code or zero semantics, and caches only immutable values. Public +entry mappings are detached on every call. + +The exact member's archive and member digests, row count and byte count +must agree with the retained owner. A single bounded capture is checked +again after reading. All current-year native keys, original household and +person-line coordinates, and source ages are joined before selection. +`DIV_VAL` validity and every valid float64 bit must match the actual money +owner, including people omitted from the selected survey. Original +`DIV_VAL_parent_statuses`, `DIV_VAL_parent_validity` and +`DIV_VAL_parent_zero_origin` remain in the projection. + +`CurrentAsecDividendValues(person, asec_literals, evidence)` is descriptive +transport. `person` is indexed by `person_id` and includes `native_person_id` +and `source_age`; `asec_literals` retains the complete current-year literal +table indexed by native person ID. `dividend_values_seal(values)` covers +both tables, their axes, exact float bits, nullable masks and backing +storage, and detached evidence. The qualifier rechecks the original +preparation after source I/O and verifies the complete seal before return. + +Consumers must retain and requalify the actual preparation before +consumption and after their last relevant I/O, and compare the complete +values seal. Matching JSON, a copied dataclass or a digest does not grant +source authority. No host, graph attachment, tax leaf, source issuer, +original/PUF clone change or release gate is added here. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py new file mode 100644 index 000000000..8a760bb80 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_dividend_source.py @@ -0,0 +1,519 @@ +"""Current ASEC dividend observations and possible survivor-property routes. + +Source flags are provenance, never donor filters. No survivor amount, tax +treatment, ACS imputation or new source authority is issued by this module. +""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-dividend-source.v1" +AMOUNT_FIELDS = ("DIV_VAL",) +# Width, position, PDF page, printed page, printed universe, zero wording. +RECEIPT_ENTRIES = { + "DIV_YN": (1, 484, 45, "6C-24", "All Persons aged 15+", "niu"), + "SUR_YN": (1, 664, 50, "6C-29", "All Persons aged 15+", "niu"), +} +SURVIVOR_ENTRIES = { + "SUR_SC1": (2, 648, 50, "6C-29", "SUR_YN = 1"), + "SUR_SC2": (2, 650, 50, "6C-29", "SUR_YN = 1"), +} +SURVIVOR_CODES = { + 0: "none or niu", + 1: "company or union survivor pension", + 2: "federal government", + 3: "US military retirement survivor pension", + 4: "state or local government survivor pension", + 5: "US railroad retirement survivor pension", + 6: "worker compensation survivor", + 7: "black lung", + 8: "regular payments from estates or trusts", + 9: "regular payments from annuities or paid-up life insurance", + 10: "other or don't know", +} +ALLOCATION_ENTRIES = { + "I_DIVVAL": (1, 819, 54, "6C-33", "DIV_YN = 1", "0:9", "See I_ANNVAL"), + "I_DIVYN": (1, 820, 54, "6C-33", "All Persons 15+", "0:1", "See I_ANNVAL"), +} +ALLOCATION_CONFLICT_NOTE = ( + "I_DIVYN prints header range0:1 but references I_ANNVAL values0:9. " + "Both axes are preserved; a code2..9 is not silently admitted or discarded. " + "Flag values do not alter dividend amount knownness or select donors." +) +TOPCODE_ENTRIES = { + "TDIV_VAL": (1, 905, 59, "6C-38", "DIV_VAL > 0"), + "TRNT_VAL": (1, 917, 60, "6C-39", "RNT_VAL > 0"), +} +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *SURVIVOR_ENTRIES, + *ALLOCATION_ENTRIES, + *TOPCODE_ENTRIES, +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_DIVIDEND_SOURCE_" + reason) + + +def _decode_amount_entry(data): + fields = [f for f in data["fields"] if f["name"] == "DIV_VAL"] + require(len(fields) == 1, "DOMAIN_FIELD_ROSTER") + field = fields[0] + vintages = [ + v for v in field["vintages"] if v["income_year"] == routing.CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE") + vintage, domain = vintages[0], field["domain"] + require( + vintage["dictionary_spelling"] == "DIV_VAL" + and vintage["survey_year"] == 2025 + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + and vintage["source_url"] == routing.DICTIONARY_URL, + "DICTIONARY_PIN", + ) + require( + field["entity"] == "person" + and domain["encoded_range_inclusive"] == {"minimum": 0, "maximum": 999999} + and domain["valid_dollar_range_inclusive"] == {"minimum": 0, "maximum": 999999} + and domain["negative_dollars_permitted"] is False + and domain["declared_negative_nonmoney_codes"] == [] + and domain["declared_other_missing_codes"] == [] + and domain["declared_niu_codes"] == [0] + and domain["valid_dollar_range_excludes"] == [] + and domain["zero_semantics"] + == "none_or_niu_not_distinguishable_from_amount_alone", + "DOMAIN_UNSUPPORTED", + ) + return ( + vintage["ascii_length_as_printed"], + vintage["ascii_position_as_printed"], + vintage["pdf_page_1based"], + vintage["printed_page"], + domain["encoded_range_inclusive"]["maximum"], + vintage["universe_as_printed"], + domain["zero_semantics"], + ) + + +@lru_cache(maxsize=1) +def _amount_entry(): + payload = ( + resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() + ) + require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") + # Every cached value is immutable; callers never receive a shared mapping. + return _decode_amount_entry(json.loads(payload)) + + +def amount_entries(): + return {"DIV_VAL": _amount_entry()} + + +def _domain_agreement(ready): + domains = [d for d in ready.bindings.spec.fields if d.name == "DIV_VAL"] + require(len(domains) == 1, "MONEY_DOMAIN_ROSTER") + domain, entry = domains[0], amount_entries()["DIV_VAL"] + require( + domain.entity == "person" + and domain.minimum == 0 + and domain.maximum == entry[4] + and domain.zero_semantics == entry[6], + "MONEY_DOMAIN_DISAGREEMENT", + ) + + +def _survivor_route(age, receipt, slots): + """Clear only resolved non-property source routes; never infer absent amounts.""" + code, status = receipt + values = [pair[0] for pair in slots] + readable = all(pair[1] == "in_printed_range" for pair in slots) + if age < 15: + if status != "in_printed_range" or not readable: + return "unresolved_outside_reporting_universe", None + return ( + "outside_reporting_universe" + if code == 0 and readable and values == [0, 0] + else "contradictory_outside_reporting_universe", + None, + ) + if status != "in_printed_range": + return ( + "missing_receipt_literal" + if status == "missing" + else "unrecognized_receipt_literal", + None, + ) + if code == 0: + return ( + "niu" if readable and values == [0, 0] else "unresolved_niu_route", + None, + ) + if code == 2: + return ( + ("known_nonreceipt", True) + if readable and values == [0, 0] + else ("unresolved_nonreceipt_source_route", None) + ) + if 8 in values: + return "possible_estate_or_trust_route", False + if not readable: + return "unresolved_source_literal", None + if 10 in values: + return "source_type_unspecified", None + if values == [0, 0]: + return "unreported_source_slots", None + return "known_other_survivor_sources", True + + +def _dividend_allocation_conflict(tokens): + # Width1 admits only one decimal digit. Parsing physical syntax against + # 0..9 does not decide which conflicting codebook range is authoritative. + frame, values, statuses = routing._codes_frame("I_DIVYN", tokens, range(10)) + frame["I_DIVYN_literal_status"] = pd.array( + [ + "well_formed" if status == "in_printed_range" else status + for status in statuses + ], + dtype="string", + ) + header = [routing.literal_code(t, (0, 1), width=1)[1] for t in tokens] + frame["I_DIVYN_header_range_status"] = pd.array(header, dtype="string") + frame["I_DIVYN_referenced_values_status"] = pd.array(statuses, dtype="string") + frame["I_DIVYN_codebook_status"] = pd.array( + [ + "literal_unresolved" + if status != "in_printed_range" + else ( + "header_reference_conflict" + if value not in (0, 1) + else "header_reference_agree" + ) + for value, status in zip(values, statuses, strict=True) + ], + dtype="string", + ) + return frame + + +def project_dividend_literals(ordered): + """Descriptive source projection; an arbitrary DataFrame grants no authority.""" + require( + type(ordered) is pd.DataFrame and set(READ_COLUMNS) <= set(ordered), "COLUMNS" + ) + ages = ordered.A_AGE.to_numpy(dtype=np.int64) + require(((ages >= 0) & (ages <= 99)).all(), "AGE_RANGE") + out = pd.DataFrame(index=range(len(ordered))) + entry = amount_entries()["DIV_VAL"] + amounts = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) + for t in ordered.DIV_VAL + ] + out["DIV_VAL_literal"] = pd.array(ordered.DIV_VAL.tolist(), dtype="string") + out["DIV_VAL_published_amount"] = pd.array([v for v, _ in amounts], dtype="Float64") + out["DIV_VAL_literal_status"] = pd.array([s for _, s in amounts], dtype="string") + codes = {} + for name, item in (*RECEIPT_ENTRIES.items(), *SURVIVOR_ENTRIES.items()): + allowed = ( + routing.RECEIPT_CODE_DOMAIN if name in RECEIPT_ENTRIES else SURVIVOR_CODES + ) + labels = ( + {0: "niu", 1: "yes", 2: "no"} if name in RECEIPT_ENTRIES else SURVIVOR_CODES + ) + frame, values, statuses = routing._codes_frame( + name, ordered[name], allowed, labels, width=item[0] + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + labels = [] + for age, receipt, (value, status) in zip( + ages, codes["DIV_YN"], amounts, strict=True + ): + kind = "missing" if value is None else ("zero" if value == 0 else "nonzero") + label = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + labels.append( + label + if status in ("missing", "in_printed_range") + else "invalid_amount_literal" + ) + known = np.array([s in routing.KNOWN_AMOUNT_STATUSES for s in labels]) + out["DIV_VAL_reporting_status"] = pd.array(labels, dtype="string") + out["DIV_VAL_amount_known"] = known + out["DIV_VAL_amount"] = out.DIV_VAL_published_amount.where(known) + routes = [ + _survivor_route(age, receipt, (one, two)) + for age, receipt, one, two in zip( + ages, codes["SUR_YN"], codes["SUR_SC1"], codes["SUR_SC2"], strict=True + ) + ] + out["survivor_property_route_status"] = pd.array( + [s for s, _ in routes], dtype="string" + ) + out["survivor_property_route_clear"] = pd.array( + [v for _, v in routes], dtype="boolean" + ) + out["survivor_estate_or_trust_code_present"] = [ + one[0] == 8 or two[0] == 8 + for one, two in zip(codes["SUR_SC1"], codes["SUR_SC2"], strict=True) + ] + out["survivor_unspecified_source_code_present"] = [ + one[0] == 10 or two[0] == 10 + for one, two in zip(codes["SUR_SC1"], codes["SUR_SC2"], strict=True) + ] + flags, _, _ = routing._codes_frame( + "I_DIVVAL", ordered.I_DIVVAL, routing.ALLOCATION_ANNVAL_CODES + ) + out = pd.concat( + [out, flags, _dividend_allocation_conflict(ordered.I_DIVYN)], axis=1 + ) + for name, item in TOPCODE_ENTRIES.items(): + flags, _, _ = routing._codes_frame( + name, + ordered[name], + (0, 1), + {0: "not topcoded", 1: "topcoded"}, + width=item[0], + ) + out = pd.concat([out, flags], axis=1) + out["source_age"] = ages + return out + + +@dataclass(frozen=True) +class CurrentAsecDividendValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def dividend_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecDividendValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="dividend columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="dividend" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-dividend-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_total(ready, positions, literals): + field = ready.field("DIV_VAL") + entry = amount_entries()["DIV_VAL"] + pairs = [ + routing.literal_code(t, range(entry[4] + 1), width=entry[0]) for t in literals + ] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "TOTAL_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "TOTAL_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "TOTAL_BITS", + ) + return field + + +def qualify_current_asec_dividend(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == 2024 + and header["semantic"] == "annual_current_money" + and native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "PERIOD", + ) + pins = [p for p in routing.coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + field = _compare_total(ready, positions, ordered.DIV_VAL) + basis = project_dividend_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for name in ("statuses", "validity", "zero_origin"): + basis["DIV_VAL_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": amount_entries(), + "receipt_entries": RECEIPT_ENTRIES, + "survivor_entries": SURVIVOR_ENTRIES, + "survivor_codes": SURVIVOR_CODES, + "allocation_entries": ALLOCATION_ENTRIES, + "allocation_conflict_note": ALLOCATION_CONFLICT_NOTE, + "topcode_entries": TOPCODE_ENTRIES, + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": 2024, + "survey_year": 2025, + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "source_flags_used_as_donor_filter": False, + "survivor_amounts_read": False, + "survivor_code8_is_possible_property_scope_only": True, + "survivor_code10_property_clearance_known": False, + "trnt_flag_universe_evaluated": False, + "tax_treatment_assigned": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecDividendValues(out, literals, json.loads(json.dumps(evidence))) + seal = dividend_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(dividend_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py new file mode 100644 index 000000000..70483f62f --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_dividend_source.py @@ -0,0 +1,497 @@ +"""Invented dividend literals, survivor routes and actual retained source owners.""" + +import copy +import hashlib +import json +import shutil +from fractions import Fraction +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_dividend_source as owner + + +def row(**changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "DIV_YN": "1", + "DIV_VAL": "140", + "SUR_YN": "2", + **changes, + } + + +def project(**changes): + return owner.project_dividend_literals(pd.DataFrame([row(**changes)])).iloc[0] + + +@pytest.mark.parametrize( + "changes,status,amount", + [ + ({}, "known_receipt", 140), + ({"DIV_VAL": "0", "DIV_YN": "2"}, "known_nonreceipt", 0), + ({"DIV_VAL": "0"}, "ambiguous_recipient_zero", None), + ({"DIV_VAL": "0", "DIV_YN": "0"}, "niu", None), + ({"DIV_YN": "0"}, "contradictory_niu_nonzero", None), + ({"DIV_YN": "2"}, "contradictory_no_nonzero", None), + ({"DIV_YN": ""}, "missing_receipt_literal", None), + ({"DIV_YN": "x"}, "unrecognized_receipt_literal", None), + ({"DIV_VAL": ""}, "missing_amount", None), + ({"DIV_VAL": "-1"}, "invalid_amount_literal", None), + ({"DIV_VAL": "1.0"}, "invalid_amount_literal", None), + ( + {"A_AGE": "14", "DIV_VAL": "0", "DIV_YN": "0"}, + "outside_reporting_universe", + None, + ), + ({"A_AGE": "14"}, "contradictory_outside_reporting_universe", None), + ], +) +def test_dividend_receipt_unknownness_never_infers_age_only_or_recipient_zeros( + changes, status, amount +): + value = project(**changes) + assert value.DIV_VAL_reporting_status == status + assert bool(value.DIV_VAL_amount_known) is (amount is not None) + assert ( + pd.isna(value.DIV_VAL_amount) + if amount is None + else value.DIV_VAL_amount == amount + ) + assert value.DIV_VAL_literal == changes.get("DIV_VAL", "140") + + +@pytest.mark.parametrize( + "token,header,reference,agreement", + [ + ("0", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("1", "in_printed_range", "in_printed_range", "header_reference_agree"), + ("5", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("9", "outside_printed_range", "in_printed_range", "header_reference_conflict"), + ("", "missing", "missing", "literal_unresolved"), + ("x", "malformed", "malformed", "literal_unresolved"), + ("01", "malformed", "malformed", "literal_unresolved"), + ], +) +def test_dividend_receipt_allocation_preserves_both_conflicting_codebook_axes( + token, header, reference, agreement +): + value = project(I_DIVYN=token) + assert value.I_DIVYN_literal == token + assert value.I_DIVYN_header_range_status == header + assert value.I_DIVYN_referenced_values_status == reference + assert value.I_DIVYN_codebook_status == agreement + if reference == "in_printed_range": + assert value.I_DIVYN_code == int(token) + assert value.I_DIVYN_literal_status == "well_formed" + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + + +@pytest.mark.parametrize( + "changes,status,clear", + [ + ({}, "known_nonreceipt", True), + ({"SUR_YN": "1", "SUR_SC1": "1"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "9"}, "known_other_survivor_sources", True), + ({"SUR_YN": "1", "SUR_SC2": "8"}, "possible_estate_or_trust_route", False), + ( + {"SUR_YN": "1", "SUR_SC1": "8", "SUR_SC2": ""}, + "possible_estate_or_trust_route", + False, + ), + ({"SUR_YN": "1", "SUR_SC1": "10"}, "source_type_unspecified", None), + ({"SUR_YN": "1"}, "unreported_source_slots", None), + ( + {"SUR_YN": "1", "SUR_SC1": "1", "SUR_SC2": ""}, + "unresolved_source_literal", + None, + ), + ({"SUR_YN": "1", "SUR_SC1": "11"}, "unresolved_source_literal", None), + ({"SUR_YN": "1", "SUR_SC1": "x"}, "unresolved_source_literal", None), + ({"SUR_YN": "", "SUR_SC1": "8"}, "missing_receipt_literal", None), + ({"SUR_YN": "x"}, "unrecognized_receipt_literal", None), + ({"SUR_SC1": "8"}, "unresolved_nonreceipt_source_route", None), + ({"SUR_SC1": ""}, "unresolved_nonreceipt_source_route", None), + ({"SUR_YN": "0"}, "niu", None), + ({"SUR_YN": "0", "SUR_SC1": "1"}, "unresolved_niu_route", None), + ({"A_AGE": "14", "SUR_YN": "0"}, "outside_reporting_universe", None), + ({"A_AGE": "14", "SUR_YN": ""}, "unresolved_outside_reporting_universe", None), + ( + {"A_AGE": "14", "SUR_YN": "0", "SUR_SC1": ""}, + "unresolved_outside_reporting_universe", + None, + ), + ], +) +def test_survivor_clearance_requires_resolved_receipt_and_sources( + changes, status, clear +): + value = project(**changes) + assert value.survivor_property_route_status == status + assert ( + pd.isna(value.survivor_property_route_clear) + if clear is None + else bool(value.survivor_property_route_clear) is clear + ) + assert value.DIV_VAL_published_amount == 140 # Routes never change donor values. + if "8" in (changes.get("SUR_SC1"), changes.get("SUR_SC2")): + assert value.survivor_estate_or_trust_code_present + if changes.get("SUR_SC1") == "10": + assert value.SUR_SC1_literal_status == "in_printed_range" + assert value.survivor_unspecified_source_code_present + + +@pytest.mark.parametrize( + "field,token", + [ + ("I_DIVVAL", "9"), + ("I_DIVVAL", ""), + ("I_DIVVAL", "x"), + ("I_DIVYN", "5"), + ("TDIV_VAL", "1"), + ("TDIV_VAL", ""), + ("TRNT_VAL", "1"), + ("TRNT_VAL", ""), + ], +) +def test_source_allocation_topcode_flags_are_never_donor_filters(field, token): + value = project(**{field: token}) + assert value.DIV_VAL_amount == 140 and value.DIV_VAL_amount_known + assert bool(value.survivor_property_route_clear) + assert value[field + "_literal"] == token + + +def test_source_roster_and_primary_pin_have_no_survivor_amount_or_tax_outputs(): + assert owner.amount_entries()["DIV_VAL"][:4] == (6, 478, 45, "6C-24") + assert owner.ALLOCATION_ENTRIES["I_DIVYN"][5:] == ("0:1", "See I_ANNVAL") + assert owner.SURVIVOR_CODES[8] == "regular payments from estates or trusts" + assert owner.SURVIVOR_CODES[10] == "other or don't know" + assert not { + "SUR_VAL1", + "SUR_VAL2", + "SRVS_VAL", + "OI_VAL", + "RNT_VAL", + "CAP_VAL", + } & set(owner.READ_COLUMNS) + + +def test_cached_domain_entry_does_not_expose_a_shared_mutable_mapping(): + original = owner.amount_entries() + changed = owner.amount_entries() + changed["DIV_VAL"] = (99, *changed["DIV_VAL"][1:]) + changed["invented"] = () + assert owner.amount_entries() == original + assert type(owner._amount_entry()) is tuple + assert project().DIV_VAL_published_amount == 140 + + +@pytest.mark.parametrize( + "change", + [ + "duplicate_field", + "absent_vintage", + "duplicate_vintage", + "negative_code", + "negative_dollars", + "zero_semantics", + "missing_code", + "minimum", + "dictionary", + ], +) +def test_supported_domain_and_unique_current_vintage_are_required(change): + data = json.loads( + owner.resources.files(owner.__package__) + .joinpath(owner.routing.DOMAINS_RESOURCE) + .read_bytes() + ) + field = next(f for f in data["fields"] if f["name"] == "DIV_VAL") + current = next(v for v in field["vintages"] if v["income_year"] == 2024) + assert owner._decode_amount_entry(data) == owner.amount_entries()["DIV_VAL"] + if change == "duplicate_field": + data["fields"].append(copy.deepcopy(field)) + elif change == "absent_vintage": + field["vintages"].remove(current) + elif change == "duplicate_vintage": + field["vintages"].append(copy.deepcopy(current)) + elif change == "negative_code": + field["domain"]["declared_negative_nonmoney_codes"] = [-1] + elif change == "negative_dollars": + field["domain"]["negative_dollars_permitted"] = True + elif change == "zero_semantics": + field["domain"]["zero_semantics"] = "valid_zero_dollars" + elif change == "missing_code": + field["domain"]["declared_other_missing_codes"] = [999999] + elif change == "minimum": + field["domain"]["encoded_range_inclusive"]["minimum"] = -1 + else: + current["dictionary_spelling"] = "WRONG" + with pytest.raises(ValueError, match="ASEC_DIVIDEND_SOURCE_(DOMAIN_|DICTIONARY_)"): + owner._decode_amount_entry(data) + + +@pytest.mark.parametrize( + "change", ["missing", "duplicate", "entity", "minimum", "maximum", "zero_semantics"] +) +def test_live_ready_money_domain_must_match_the_pinned_source_contract(change): + domain = SimpleNamespace( + name="DIV_VAL", + entity="person", + minimum=0, + maximum=999999, + zero_semantics="none_or_niu_not_distinguishable_from_amount_alone", + ) + fields = [domain] + ready = SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + owner._domain_agreement(ready) + if change == "missing": + fields.clear() + elif change == "duplicate": + fields.append(copy.copy(domain)) + else: + setattr( + domain, change, "wrong" if change in ("entity", "zero_semantics") else 1 + ) + with pytest.raises(ValueError, match="MONEY_DOMAIN_"): + owner._domain_agreement(ready) + + +@pytest.mark.parametrize("change", ["amount_bit", "validity", "literal", "missing"]) +def test_current_money_total_requires_exact_bits_and_literal_validity(change): + field = SimpleNamespace(amounts=np.array([140.0, 0.0]), validity=np.array([1, 0])) + ready = SimpleNamespace(field=lambda name: field) + positions, literals = np.array([0, 1]), ["140", ""] + assert owner._compare_total(ready, positions, literals) is field + if change == "amount_bit": + field.amounts[0] = np.nextafter(140.0, np.inf) + elif change == "validity": + field.validity[1] = 1 + elif change == "literal": + literals[0] = "-1" + else: + literals[1] = "0" + with pytest.raises(ValueError, match="TOTAL_"): + owner._compare_total(ready, positions, literals) + + +def test_physical_seal_covers_nullable_boolean_and_money_hidden_backing(): + raw = pd.DataFrame([row(DIV_VAL="0", SUR_YN="1")]) + values = owner.CurrentAsecDividendValues( + owner.project_dividend_literals(raw), raw, {"descriptive": True} + ) + original = owner.dividend_values_seal(values) + assert owner.dividend_values_seal(copy.deepcopy(values)) == original + for name in ("DIV_VAL_amount", "survivor_property_route_clear"): + changed = copy.deepcopy(values) + array = changed.person[name].array + assert array._mask[0] + array._data[0] = not array._data[0] if name.endswith("clear") else 99.0 + assert owner.dividend_values_seal(changed) != original + + +def dividend_arguments(tmp_path, monkeypatch, *, fraction=Fraction(1)): + """Amend complete invented literals and retained money before any issuance.""" + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch, fraction=fraction, zero=False) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + literals = { + 105: row(I_DIVYN="5"), + 106: row(A_AGE="14", DIV_VAL="0", DIV_YN="0", SUR_YN="0"), + 107: row(A_AGE="70", DIV_VAL="500", SUR_YN="1", SUR_SC2="8", TDIV_VAL="1"), + 108: row(DIV_VAL="0", SUR_YN="1", SUR_SC1="10"), + } + changes = {} + for name in ("DIV_VAL", "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in ("DIV_VAL", "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + *owner.RECEIPT_ENTRIES, + *owner.SURVIVOR_ENTRIES, + *owner.ALLOCATION_ENTRIES, + *owner.TOPCODE_ENTRIES, + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row())[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "dividend-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(tmp_path, monkeypatch, *, fraction=Fraction(1)): + return owner.routing.source.prepare_authenticated_survey_population( + **dividend_arguments(tmp_path, monkeypatch, fraction=fraction) + ) + + +def test_actual_owner_qualified_source_values_and_requalification( + tmp_path, monkeypatch +): + parent = prepared(tmp_path, monkeypatch) + result = owner.qualify_current_asec_dividend(parent) + values = result.person.set_index("native_person_id") + assert values.loc[105, "DIV_VAL_amount"] == 140 + assert values.loc[105, "I_DIVYN_codebook_status"] == "header_reference_conflict" + assert values.loc[106, "DIV_VAL_reporting_status"] == "outside_reporting_universe" + assert values.loc[107, "DIV_VAL_amount"] == 500 + assert not bool(values.loc[107, "survivor_property_route_clear"]) + assert pd.isna(values.loc[108, "survivor_property_route_clear"]) + assert not values.loc[108, "DIV_VAL_amount_known"] + assert { + "DIV_VAL_parent_statuses", + "DIV_VAL_parent_validity", + "DIV_VAL_parent_zero_origin", + } <= set(values) + assert owner.dividend_values_seal( + owner.qualify_current_asec_dividend(parent) + ) == owner.dividend_values_seal(result) + assert ( + not result.evidence["source_admission_issued"] + and not result.evidence["survivor_amounts_read"] + ) + assert result.evidence == json.loads(json.dumps(result.evidence)) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(copy.copy(parent)) + + +@pytest.mark.parametrize( + "coordinate", ["PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "DIV_VAL"] +) +def test_actual_owner_refuses_original_coordinate_or_money_change( + tmp_path, monkeypatch, coordinate +): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if coordinate == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(coordinate)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_compares_unselected_current_year_amounts(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch, fraction=Fraction(1, 2)) + values = owner.qualify_current_asec_dividend(parent) + omitted = set(values.asec_literals.index) - set(values.person.native_person_id) + assert omitted + key = values.asec_literals.loc[next(iter(omitted)), "PERIDNUM"] + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + raw.loc[key, "DIV_VAL"] = "997" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError, match="TOTAL_BITS"): + owner.qualify_current_asec_dividend(parent) + + +def test_actual_owner_requalifies_after_capture_io(tmp_path, monkeypatch): + parent = prepared(tmp_path, monkeypatch) + capture = owner._capture_member + calls = [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_dividend(parent) + assert calls + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "route_mask", "literal", "evidence"] +) +def test_final_physical_seal_covers_complete_returned_values( + tmp_path, monkeypatch, mutation +): + parent = prepared(tmp_path, monkeypatch) + seal, calls = owner.dividend_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + if mutation == "amount_bit": + array = value.person.DIV_VAL_amount.array + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array = value.person.DIV_VAL_amount.array + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "route_mask": + array = value.person.survivor_property_route_clear.array + array._mask[np.flatnonzero(array._mask)[0]] = False + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_SC1") + ] = "8" + else: + value.evidence["survivor_amounts_read"] = True + return before + + monkeypatch.setattr(owner, "dividend_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_dividend(parent) From d20f25cb49a41729ec8ce082653e08fd0caac280 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:20:17 -0400 Subject: [PATCH 167/255] Connect joint property income draws to signed graph reconciliation --- changelog.d/us-property-income-graph.added.md | 1 + docs/us-property-income-graph.md | 58 +++ .../build/us_runtime/graph_property_income.py | 339 ++++++++++++++ .../tests/test_us_graph_property_income.py | 423 ++++++++++++++++++ 4 files changed, 821 insertions(+) create mode 100644 changelog.d/us-property-income-graph.added.md create mode 100644 docs/us-property-income-graph.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py create mode 100644 packages/microcosm-build/tests/test_us_graph_property_income.py diff --git a/changelog.d/us-property-income-graph.added.md b/changelog.d/us-property-income-graph.added.md new file mode 100644 index 000000000..ac92df734 --- /dev/null +++ b/changelog.d/us-property-income-graph.added.md @@ -0,0 +1 @@ +Add a source-blind property-income model graph with original-design QRF fitting, joint draw columns, signed reconciliation and explicit diagnostics. diff --git a/docs/us-property-income-graph.md b/docs/us-property-income-graph.md new file mode 100644 index 000000000..89d2e0c4c --- /dev/null +++ b/docs/us-property-income-graph.md @@ -0,0 +1,58 @@ +# Property income model graph + +`microcosm.build.us_runtime.graph_property_income.property_income_nodes` +declares ten numerical nodes: four conditional QRF fits, four chained draws, +one operation that writes the raw draws as columns, and one signed +reconciliation. It uses the existing QRF and reconciliation implementations. +The fragment consumes separately supplied original donor and recipient +populations; it does not qualify native sources or attach results to clones. + +The shared columns in `property_income_constants.py` describe ordinary +interest, retirement-account interest, dividends and signed broad property +receipts. The first three are nonnegative; broad property receipts may be +negative. `property_reported_total` is a required predictor and the recipient +reconciliation anchor. The source bridge supplies the ASEC reported aggregate +for donors and the adjusted ACS INTP amount for recipients. It must preserve +their different reference periods and account for unresolved income routes. + +The training wrapper requires finite float64 predictors and components, +original DESIGN weights, and a component sum equal to the supplied donor +aggregate. It delegates actual fitting to `LegacyQRFTrainKernel`; it does not +filter, repair or reconcile donor observations. The owning source preparation +must establish original household membership, design-weight mapping, eligible +donor support and source knownness before invoking this fragment. A weight +kind label alone does not establish original source authority. + +Recipient draws preserve the complete target order and earlier raw draws as +conditioning inputs. The column operation binds every raw value to its apply +checkpoint, exact recipient index, sibling producer, and final model history. +The graph executor supplies the authenticated artifact edges. Direct calls or +copied checkpoint bytes do not authorize source or model admission. Person IDs +remain int64 independently of the pandas index used by QRF checkpoints. + +Reconciliation retains raw draws in separate `draw_` columns and exposes each +component's adjustment, active bound, residual and objective. Scales and +tolerances are required parameters. Changing them recomputes reconciliation; +changing recipient anchors recomputes recipient draws while reusing donor fits. +Negative or zero net income never forces all positive components to zero: +positive interest and a property loss can coexist at either total. + +Missing anchors and under-15 source blanks require an explicit upstream +decision; this fragment does not manufacture zeros. Original ASEC observations +are not reconciled to ACS respondents. The next country integration must join +the complete source-qualified basis, draw once per original ACS person, retain +the result on both initial clones, and derive the declared tax splits before +running the dependent PUF successor. Retirement-account earnings are not +ordinary taxable/exempt interest. Broad property receipts are not yet a +rental-income tax leaf. PUF-owned alternatives must not subsequently be forced +back onto the survey anchor. + +The initial invented test uses 44 donor persons and four recipients, including +negative/zero totals, large int64 identities, original membership, metadata and +a zero-weight household. It executes the real twelve-node graph including two +source nodes, compares draws with direct weighted QRF, compares reconciled +values with the pure projection and reopens the store in required-cache mode. +Twenty-one tests also cover dependency reuse, incorrect weight kinds, +unknown/inconsistent donors, altered draw history and invalid declarations. +They establish numerical and graph contracts, not native source acceptance, +statistical adequacy, tax correctness or release readiness. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py new file mode 100644 index 000000000..f829b853f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py @@ -0,0 +1,339 @@ +"""Joint property-income draws and visible reconciliation on original persons. + +This source-blind fragment consumes already qualified donor/recipient branches. +The country host owns source admission, original design-weight mapping, unknown +exclusions and later clone attachment. No source, tax treatment or release +authority follows from executing these numerical operations. +""" + +from __future__ import annotations + +import sys +from dataclasses import replace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import qrf +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_nodes, + legacy_qrf_train_nodes, +) +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.fit.graph_signed_reconciliation import signed_reconciliation_node +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + source_hash, +) +from microcosm.graph.keys import opaque_artifact_key + +from .property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, +) + +PROTOCOL = "microcosm.us.property-income-model.v1" +DRAW_SUMMARY_TYPE = ArtifactType("microcosm.us.property_income_joint_draws", 1) + + +def require(condition, reason): + if not condition: + raise ValueError("PROPERTY_INCOME_GRAPH_" + reason) + + +def _features(features): + codec.names(features, "property predictors") + require( + PROPERTY_REPORTED_TOTAL in features + and not set(features).intersection( + (*PROPERTY_COMPONENTS, *PROPERTY_DRAW_COLUMNS) + ), + "FEATURE_ROSTER", + ) + return features + + +def _draw_node(prefix, population, features, seed, phase): + edges = [] + for i in range(len(PROPERTY_COMPONENTS)): + producer = f"{prefix}.apply.{i:03d}" + edges.extend( + ( + ArtifactInput(f"raw_{i}", producer, "raw_draw", codec.RAW_TARGET_TYPE), + ArtifactInput( + f"state_{i}", producer, "apply_state", codec.APPLY_STATE_TYPE + ), + ) + ) + edges.append( + ArtifactInput( + "training", + f"{prefix}.fit.{len(PROPERTY_COMPONENTS) - 1:03d}", + "training_state", + codec.TRAINING_STATE_TYPE, + ) + ) + return Node( + prefix + ".draws", + PropertyIncomeDrawColumnsKernel.ref, + population=population, + inputs=(Slice("person", _features(features)),), + outputs=tuple(Owned("person", c, "float64") for c in PROPERTY_DRAW_COLUMNS), + params={"prefix": prefix, "features": features, "seed": seed, "phase": phase}, + artifact_inputs=tuple(edges), + artifact_outputs=(ArtifactOutput("summary", DRAW_SUMMARY_TYPE),), + description="Materialize the complete joint draw on original recipient persons; retain model and ordered raw-draw history.", + ) + + +def property_income_nodes( + prefix, + *, + donor_population, + recipient_population, + features, + seed, + n_estimators, + scales, + atol, + rtol, +): + """Declare four fits, four draws, draw placement and signed reconciliation. + + Donor and recipient populations must be separate original-person branches. + The aggregate feature is a qualified ASEC reported total on the donor and + adjusted ACS INTP on the recipient. Unknown anchors must be handled upstream. + Scales and tolerances are required modeling parameters, with no defaults. + """ + require( + type(prefix) is str + and bool(prefix) + and type(seed) is int + and seed >= 0 + and type(n_estimators) is int + and n_estimators > 0 + and donor_population != recipient_population, + "DECLARATION_PARAMETERS", + ) + features = _features(features) + phase = PROTOCOL + ":" + prefix + fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor_population, + entity="person", + predictors=features, + targets=PROPERTY_COMPONENTS, + seed=seed, + n_estimators=n_estimators, + zero_atol=0, + phase=phase, + ) + applies = legacy_qrf_apply_nodes( + prefix + ".apply", + population=recipient_population, + fit_nodes=fits, + seed=seed, + phase=phase, + ) + fits = tuple(replace(node, kernel=PropertyIncomeTrainKernel.ref) for node in fits) + draws = _draw_node(prefix, recipient_population, features, seed, phase) + reconcile = signed_reconciliation_node( + prefix + ".reconcile", + population=recipient_population, + entity="person", + anchor=PROPERTY_REPORTED_TOTAL, + draws=PROPERTY_DRAW_COLUMNS, + components=PROPERTY_COMPONENTS, + nonnegative=(True, True, True, False), + scales=scales, + atol=atol, + rtol=rtol, + diagnostic_prefix=prefix + "_reconciliation", + ) + return (*fits, *applies, draws, reconcile) + + +class PropertyIncomeTrainKernel(LegacyQRFTrainKernel): + """The existing weighted QRF, restricted to the qualified property basis.""" + + ref = "us.property_income.train@1" + + def implementation_hash(self): + return codec.sha( + codec.encode_json( + { + "fragment": source_hash(sys.modules[__name__]), + "fit": super().implementation_hash(), + "components": PROPERTY_COMPONENTS, + "aggregate": PROPERTY_REPORTED_TOTAL, + } + ) + ) + + def run(self, context): + require( + context.node.kernel == self.ref + and context.params == context.node.params + and tuple(context.params["targets"]) == PROPERTY_COMPONENTS, + "TRAINING_DECLARATION", + ) + _features(context.params["predictors"]) + require( + set(context.tables) == {"person"} + and context.weights["person"].kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_WEIGHTS", + ) + table = context.tables["person"] + columns = (*context.params["predictors"], *PROPERTY_COMPONENTS) + require( + all(table[c].dtype == np.dtype("float64") for c in columns) + and np.isfinite(table.loc[:, list(columns)].to_numpy()).all(), + "KNOWN_DONOR_VALUES", + ) + require( + (table.loc[:, list(PROPERTY_COMPONENTS[:3])].to_numpy() >= 0).all() + and np.array_equal( + table.loc[:, list(PROPERTY_COMPONENTS)].sum(axis=1).to_numpy(), + table[PROPERTY_REPORTED_TOTAL].to_numpy(), + ), + "RESOLVED_DONOR_COMPONENTS", + ) + # Delegate the actual fit and its validation to the shared implementation. + # The executable graph retains the stricter country kernel identity. + return LegacyQRFTrainKernel().run( + replace( + context, node=replace(context.node, kernel=LegacyQRFTrainKernel.ref) + ), + ) + + +class PropertyIncomeDrawColumnsKernel(KernelBase): + ref = "us.property_income.draw_columns@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + codec, + qrf, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + expected = _draw_node( + population=context.node.population, **dict(context.params) + ) + require(context.node.normative() == expected.normative(), "DRAW_DECLARATION") + require( + set(context.tables) == {"person"} + and not context.sources + and set(context.artifacts) + == {edge.name for edge in expected.artifact_inputs}, + "DRAW_CONTEXT", + ) + table = context.tables["person"] + require( + table.person_id.dtype == np.dtype("int64") and table.person_id.is_unique, + "PERSON_IDENTITY", + ) + require( + all( + table[c].dtype == np.dtype("float64") + for c in context.params["features"] + ) + and np.isfinite( + table.loc[:, list(context.params["features"])].to_numpy() + ).all(), + "KNOWN_RECIPIENT_FEATURES", + ) + edge_map = {e.name: e for e in expected.artifact_inputs} + + def payload(name, kind): + value = codec.artifact(context, name, kind) + require( + value.key + == opaque_artifact_key(value.producer_key, edge_map[name].artifact), + "ARTIFACT_KEY", + ) + return value.payload + + training, trained = codec.read_training( + payload("training", codec.TRAINING_STATE_TYPE) + ) + state = trained.to_dict() + require( + state["entity"] == "person" + and tuple(state["predictors"]) == context.params["features"] + and tuple(state["targets"]) == PROPERTY_COMPONENTS + and tuple(state["completed_targets"]) == PROPERTY_COMPONENTS, + "COMPLETE_TRAINING_HISTORY", + ) + raw, columns = [], {} + index = pd.Index(table.person_id.to_numpy(copy=True), name="person_id") + for i, (target, column) in enumerate( + zip(PROPERTY_COMPONENTS, PROPERTY_DRAW_COLUMNS, strict=True) + ): + raw_name, state_name = f"raw_{i}", f"state_{i}" + require( + context.artifacts[raw_name].producer_key + == context.artifacts[state_name].producer_key, + "DRAW_SIBLING_PRODUCER", + ) + raw.append(payload(raw_name, codec.RAW_TARGET_TYPE)) + application, chain = codec.read_application( + payload(state_name, codec.APPLY_STATE_TYPE) + ) + require( + chain.entity == "person" + and tuple(chain.predictors) == context.params["features"] + and tuple(chain.targets) == PROPERTY_COMPONENTS + and tuple(chain.completed_targets) == PROPERTY_COMPONENTS[: i + 1] + and chain.recipient_index == qrf._index_identity(table.index) + and application["seed"] == context.params["seed"] + and application["models"] == training["models"][: i + 1] + and application["raw_targets"] + == [ + {"target": name, "sha256": codec.sha(value)} + for name, value in zip( + PROPERTY_COMPONENTS[: i + 1], raw, strict=True + ) + ], + "ORDERED_JOINT_DRAW_HISTORY", + ) + values = codec.read_raw_target(raw[-1], target=target, index=table.index) + columns["person", column] = pd.Series( + values.copy(), index=index, name=column + ) + return KernelResult( + columns=columns, + artifacts={ + "summary": codec.encode_json( + { + "protocol": PROTOCOL, + "rows": len(table), + "components": PROPERTY_COMPONENTS, + "models": training["models"], + "raw_sha256": [codec.sha(value) for value in raw], + "allocation": "one_joint_draw_per_input_person", + } + ) + }, + receipt={"protocol": PROTOCOL, "rows": len(table)}, + ) diff --git a/packages/microcosm-build/tests/test_us_graph_property_income.py b/packages/microcosm-build/tests/test_us_graph_property_income.py new file mode 100644 index 000000000..e0a48c9f6 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_property_income.py @@ -0,0 +1,423 @@ +"""Real weighted QRF and signed reconciliation over invented person branches.""" + +import hashlib +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime.graph_property_income import ( + PROPERTY_COMPONENTS, + PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, + PropertyIncomeDrawColumnsKernel, + PropertyIncomeTrainKernel, + property_income_nodes, +) +from microcosm.fit import QRF +from microcosm.fit.graph_legacy_qrf import LegacyQRFApplyKernel +from microcosm.fit.graph_signed_reconciliation import SignedReconciliationKernel +from microcosm.fit.signed_reconciliation import reconcile_signed_total +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + load_source, + run_graph, +) + +FEATURES = (PROPERTY_REPORTED_TOTAL, "age", "earnings") + + +@pytest.fixture(autouse=True) +def one_worker(monkeypatch): + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + + +def frame(values, *, kind=WeightKind.DESIGN, donor=False): + values = values.copy() + ids = np.arange(len(values), dtype=np.int64) + 2**53 + 21 + groups = np.arange(len(values), dtype=np.int64) // 2 + 301 + values.insert(0, "person_id", ids) + values.insert(1, "person_household_id", groups) + households = np.unique(groups) + weights = np.arange(len(households), dtype=float) + 1 + weights[-1] = 0 + return Frame( + {"person": values, "household": pd.DataFrame({"household_id": households})}, + EntitySchema(group_entities=("household",)), + {"household": Weights(weights, kind)}, + metadata={ + "invented": True, + "branch": "donor" if donor else "recipient", + "keep": {"n": 7}, + }, + ) + + +def frames(): + r = np.arange(44) + donor = pd.DataFrame( + { + "age": 25.0 + r % 20, + "earnings": 1000.0 * (r % 7), + PROPERTY_COMPONENTS[0]: 5.0 * (r % 4), + PROPERTY_COMPONENTS[1]: 10.0 * (r % 3), + PROPERTY_COMPONENTS[2]: 15.0 * (r % 2), + PROPERTY_COMPONENTS[3]: 20.0 * (r % 5 - 2), + } + ) + donor[PROPERTY_REPORTED_TOTAL] = donor.loc[:, list(PROPERTY_COMPONENTS)].sum(axis=1) + recipient = pd.DataFrame( + { + PROPERTY_REPORTED_TOTAL: [-60.0, 0.0, 37.5, 120.0], + "age": [22.0, 55.0, 67.0, 32.0], + "earnings": [2500.0, 0.0, 1200.0, 7600.0], + "retained_note": ["alpha", "beta", "gamma", "delta"], + }, + index=pd.Index([9, 2, 17, 41], name="original_row"), + ) + recipient["retained_note"] = recipient.retained_note.astype("string") + return frame(donor, donor=True), frame(recipient) + + +class SourceKernel(KernelBase): + ref = "test.property_income.source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, ctx): + return KernelResult( + frame=load_source("frame-store", ctx.sources[ctx.node.sources[0]]) + ) + + +class RecordingDrawKernel(PropertyIncomeDrawColumnsKernel): + context = None + + def run(self, context): + self.context = context + return super().run(context) + + +def source_node(name, source): + return Node( + name, + SourceKernel.ref, + sources=(name,), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(entity, c, str(table[c].dtype)) + for entity in source.entities + for table in (source.table(entity),) + for c in table + if not c.endswith("_id") + ), + ) + + +def source_path(store, source): + parts = [ + source.table(entity).to_json(orient="table").encode() + for entity in source.entities + ] + parts.extend( + source.weights_for(entity).values.tobytes() + for entity in source.weighted_entities + ) + return store.put_frame(hashlib.sha256(b"".join(parts)).hexdigest(), source) + + +def setup(tmp_path, *, scales=(1.0, 1.0, 1.0, 1.0), recipient=None): + donor, default_recipient = frames() + recipient = default_recipient if recipient is None else recipient + nodes = property_income_nodes( + "property", + donor_population="donor", + recipient_population="recipient", + features=FEATURES, + seed=811, + n_estimators=2, + scales=scales, + atol=1e-10, + rtol=1e-12, + ) + graph = Graph( + country="us", + sources=( + SourceRef("donor", "frame-store"), + SourceRef("recipient", "frame-store"), + ), + nodes=( + source_node("donor", donor), + source_node("recipient", recipient), + *nodes, + ), + ) + kernels = KernelRegistry() + draw = RecordingDrawKernel() + for kernel in ( + SourceKernel(), + PropertyIncomeTrainKernel(), + LegacyQRFApplyKernel(), + draw, + SignedReconciliationKernel(), + ): + kernels.register(kernel) + store = ContentStore(tmp_path) + sources = { + "donor": source_path(store, donor), + "recipient": source_path(store, recipient), + } + return compile_graph(graph), store, kernels, sources, donor, recipient, draw + + +def execute(setup_values, **kwargs): + compiled, store, kernels, sources, *_ = setup_values + return run_graph(compiled, store=store, kernels=kernels, sources=sources, **kwargs) + + +def assert_preserved(source, result): + for entity in source.entities: + pd.testing.assert_frame_equal( + result.table(entity)[list(source.table(entity))], source.table(entity) + ) + assert result.metadata == source.metadata + assert result.schema == source.schema + for entity in source.weighted_entities: + assert result.weights_for(entity).kind is source.weights_for(entity).kind + np.testing.assert_array_equal( + result.weights_for(entity).values, source.weights_for(entity).values + ) + + +def test_actual_joint_fit_projection_and_required_replay(tmp_path): + values = setup(tmp_path) + compiled, store, kernels, sources, donor, recipient, _ = values + manifest = execute(values) + result = manifest.population("recipient") + assert len(manifest.nodes) == 12 + assert_preserved(donor, manifest.population("donor")) + assert_preserved(recipient, result) + model = QRF(seed=811, n_estimators=2, zero_atol=0) + state = model.start_chain( + donor, list(FEATURES), list(PROPERTY_COMPONENTS), weights="design" + ) + raw = pd.DataFrame(index=recipient.person.index) + for target, draw_name in zip( + PROPERTY_COMPONENTS, PROPERTY_DRAW_COLUMNS, strict=True + ): + next_ = model.fit_draw_next( + donor, recipient, raw, state=state, weights="design" + ) + raw[target] = next_.raw_draw + state = next_.state + np.testing.assert_array_equal(result.person[draw_name], next_.raw_draw) + projected = reconcile_signed_total( + raw.to_numpy(), + recipient.person[PROPERTY_REPORTED_TOTAL].to_numpy(), + components=PROPERTY_COMPONENTS, + nonnegative=np.array([True, True, True, False]), + scales=np.ones(4), + atol=1e-10, + rtol=1e-12, + ) + np.testing.assert_array_equal( + result.person[list(PROPERTY_COMPONENTS)], projected.values + ) + np.testing.assert_allclose( + result.person[list(PROPERTY_COMPONENTS)].sum(axis=1), + recipient.person[PROPERTY_REPORTED_TOTAL], + rtol=1e-12, + atol=1e-10, + ) + assert (result.person[list(PROPERTY_COMPONENTS[:3])] >= 0).all().all() + assert result.person[PROPERTY_COMPONENTS[-1]].iloc[0] < 0 + assert result.person.person_id.dtype == np.dtype("int64") + again = run_graph( + compiled, + store=ContentStore(store.root), + kernels=kernels, + sources=sources, + resume="require", + ) + assert again.key == manifest.key + assert all(node.hit for node in again.nodes.values()) + pd.testing.assert_frame_equal(result.person, again.population("recipient").person) + + +def test_changed_scales_reuse_fit_draws_and_only_recompute_projection(tmp_path): + first = execute(setup(tmp_path)) + second = execute(setup(tmp_path, scales=(4.0, 2.0, 3.0, 1.0))) + assert sum(n.hit for n in second.nodes.values()) == 11 + assert not second.node("property.reconcile").hit + for c in PROPERTY_DRAW_COLUMNS: + np.testing.assert_array_equal( + first.population("recipient").person[c], + second.population("recipient").person[c], + ) + + +def test_changed_recipient_anchor_reuses_all_donor_fits(tmp_path): + execute(setup(tmp_path)) + _, recipient = frames() + data = recipient.person.copy() + data.loc[data.index[0], PROPERTY_REPORTED_TOTAL] = -75.0 + changed = Frame( + {"person": data, "household": recipient.table("household")}, + recipient.schema, + {"household": recipient.weights_for("household")}, + metadata=recipient.metadata, + ) + result = execute(setup(tmp_path, recipient=changed)) + assert sum(n.hit for n in result.nodes.values()) == 5 + assert result.node("donor").hit + assert all(result.node(f"property.fit.{i:03d}").hit for i in range(4)) + assert not result.node("property.reconcile").hit + + +@pytest.mark.parametrize("kind", [WeightKind.IMPORTANCE, WeightKind.CALIBRATED]) +def test_postallocation_weight_kinds_refuse_before_fitting(kind): + donor, _ = frames() + node = property_income_nodes( + "p", + donor_population="d", + recipient_population="r", + features=FEATURES, + seed=2, + n_estimators=2, + scales=(1.0,) * 4, + atol=1e-10, + rtol=1e-12, + )[0] + ctx = KernelContext( + node, + {"person": donor.person}, + {"person": Weights(donor.resolve_weights("person").values, kind)}, + pd.Series(dtype=str), + node.params, + np.random.default_rng(2), + ) + with pytest.raises(ValueError, match="ORIGINAL_DESIGN_WEIGHTS"): + PropertyIncomeTrainKernel().run(ctx) + + +@pytest.mark.parametrize( + "defect", + ["unknown", "nonfinite", "negative_interest", "inconsistent_total", "nullable"], +) +def test_ineligible_donor_basis_refuses_before_fitting(defect): + donor, _ = frames() + table = donor.person.copy() + if defect == "unknown": + table.loc[0, PROPERTY_COMPONENTS[0]] = np.nan + elif defect == "nonfinite": + table.loc[0, PROPERTY_COMPONENTS[3]] = np.inf + elif defect == "negative_interest": + table.loc[0, PROPERTY_COMPONENTS[0]] = -5.0 + table.loc[0, PROPERTY_REPORTED_TOTAL] -= 5.0 + elif defect == "inconsistent_total": + table.loc[0, PROPERTY_REPORTED_TOTAL] += 5.0 + else: + table[PROPERTY_COMPONENTS[0]] = pd.array( + table[PROPERTY_COMPONENTS[0]], dtype="Float64" + ) + node = property_income_nodes( + "p", + donor_population="d", + recipient_population="r", + features=FEATURES, + seed=2, + n_estimators=2, + scales=(1.0,) * 4, + atol=1e-10, + rtol=1e-12, + )[0] + ctx = KernelContext( + node, + {"person": table}, + {"person": donor.resolve_weights("person")}, + pd.Series(dtype=str), + node.params, + np.random.default_rng(2), + ) + with pytest.raises( + ValueError, match="KNOWN_DONOR_VALUES|RESOLVED_DONOR_COMPONENTS" + ): + PropertyIncomeTrainKernel().run(ctx) + + +@pytest.mark.parametrize( + "defect", ["raw_history", "producer", "row_index", "entity_ids", "seed", "roster"] +) +def test_draw_placement_rejects_unbound_history_and_row_identity(tmp_path, defect): + values = setup(tmp_path) + execute(values) + ctx = values[-1].context + if defect in ("raw_history", "producer"): + artifacts = dict(ctx.artifacts) + entry = artifacts["raw_0"] + artifacts["raw_0"] = replace( + entry, + **( + { + "payload": entry.payload[:-8] + + np.array([999.0], dtype=" Date: Sat, 12 Sep 2026 13:27:50 -0400 Subject: [PATCH 168/255] Bind delegated QRF implementation to property fit cache identity --- docs/us-property-income-graph.md | 2 +- .../build/us_runtime/graph_property_income.py | 2 +- .../tests/test_us_graph_property_income.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/us-property-income-graph.md b/docs/us-property-income-graph.md index 89d2e0c4c..d06b4c869 100644 --- a/docs/us-property-income-graph.md +++ b/docs/us-property-income-graph.md @@ -52,7 +52,7 @@ negative/zero totals, large int64 identities, original membership, metadata and a zero-weight household. It executes the real twelve-node graph including two source nodes, compares draws with direct weighted QRF, compares reconciled values with the pure projection and reopens the store in required-cache mode. -Twenty-one tests also cover dependency reuse, incorrect weight kinds, +Twenty-two tests also cover delegated training-code identity, dependency reuse, incorrect weight kinds, unknown/inconsistent donors, altered draw history and invalid declarations. They establish numerical and graph contracts, not native source acceptance, statistical adequacy, tax correctness or release readiness. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py index f829b853f..030132146 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income.py @@ -176,7 +176,7 @@ def implementation_hash(self): codec.encode_json( { "fragment": source_hash(sys.modules[__name__]), - "fit": super().implementation_hash(), + "fit": LegacyQRFTrainKernel().implementation_hash(), "components": PROPERTY_COMPONENTS, "aggregate": PROPERTY_REPORTED_TOTAL, } diff --git a/packages/microcosm-build/tests/test_us_graph_property_income.py b/packages/microcosm-build/tests/test_us_graph_property_income.py index e0a48c9f6..57fac80cc 100644 --- a/packages/microcosm-build/tests/test_us_graph_property_income.py +++ b/packages/microcosm-build/tests/test_us_graph_property_income.py @@ -16,6 +16,7 @@ property_income_nodes, ) from microcosm.fit import QRF +from microcosm.fit import graph_legacy_train from microcosm.fit.graph_legacy_qrf import LegacyQRFApplyKernel from microcosm.fit.graph_signed_reconciliation import SignedReconciliationKernel from microcosm.fit.signed_reconciliation import reconcile_signed_total @@ -41,6 +42,19 @@ FEATURES = (PROPERTY_REPORTED_TOTAL, "age", "earnings") +def test_base_training_implementation_is_part_of_cache_identity(monkeypatch): + seen = [] + original = graph_legacy_train.source_hash + + def observe(*items, **kwargs): + seen.append(items) + return original(*items, **kwargs) + + monkeypatch.setattr(graph_legacy_train, "source_hash", observe) + PropertyIncomeTrainKernel().implementation_hash() + assert any(items[0] is graph_legacy_train.LegacyQRFTrainKernel for items in seen) + + @pytest.fixture(autouse=True) def one_worker(monkeypatch): monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") From 9a369a8f92fbb1a5392c0d0c7ff15f9c4133bb55 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:30:09 -0400 Subject: [PATCH 169/255] Harden ASEC amount metadata and domain agreement --- .../asec-source-domain-contract.fixed.md | 3 + docs/us-asec-source-domain-contract.md | 43 ++++ .../current_asec_child_support_source.py | 45 +++- .../current_asec_income_routing_source.py | 82 +++++-- .../current_asec_interest_source.py | 61 +++-- .../test_us_asec_source_domain_hardening.py | 217 ++++++++++++++++++ ...st_us_current_asec_child_support_source.py | 18 ++ .../test_us_current_asec_interest_source.py | 18 ++ 8 files changed, 453 insertions(+), 34 deletions(-) create mode 100644 changelog.d/asec-source-domain-contract.fixed.md create mode 100644 docs/us-asec-source-domain-contract.md create mode 100644 packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py diff --git a/changelog.d/asec-source-domain-contract.fixed.md b/changelog.d/asec-source-domain-contract.fixed.md new file mode 100644 index 000000000..e6b385f9c --- /dev/null +++ b/changelog.d/asec-source-domain-contract.fixed.md @@ -0,0 +1,3 @@ +Prevent returned ASEC amount metadata from mutating cached definitions. Require +unique current-vintage entries, supported interest and child-support domains, +and agreement with the retained current-money owner before source capture. diff --git a/docs/us-asec-source-domain-contract.md b/docs/us-asec-source-domain-contract.md new file mode 100644 index 000000000..597ec5a7b --- /dev/null +++ b/docs/us-asec-source-domain-contract.md @@ -0,0 +1,43 @@ +# ASEC amount-domain contracts + +The current income-routing, interest and child-support qualifiers read monetary +definitions from the pinned `asec_current_money_domains_v1.json` artifact. +Their public amount-entry functions return detached mappings. Routing and +interest cache immutable tuples privately; child support caches immutable JSON +bytes and reconstructs its nested evidence on each call. Replacing an entry, +clearing a returned mapping or editing nested child-support metadata cannot +change a later caller's definitions. The public mapping and evidence shapes +remain unchanged. + +Each relevant field must occur exactly once and contain exactly one vintage for +the current income year. Its dictionary spelling, URL and digest must match the +qualifier's source pin. Missing or duplicate entries refuse qualification. + +The interest total and child-support parsers accept unsigned integer literals. +They explicitly require the corresponding artifact domains to describe +nonnegative integer dollars, with the supported zero and NIU meanings, no +additional missing-value codes or excluded dollar values, and consistent +encoded, dollar and printed ranges. Bounds remain artifact-derived. Re-pinning +an artifact with an unsupported domain requires a parser change and its own +review; it cannot silently change the interpretation of those literals. + +Before capturing the original source member, each qualifier compares its +relevant amount definitions with the actual retained current-money owner's +`MoneyDomain` entries. Names must be unique; entity, grain, native column, +bounds and zero semantics must agree. Interest checks the retained `INT_VAL` +total. Its additional ordinary and retirement-account components retain their +separate published definitions. Child support checks both `CSP_VAL` and +`CHSP_VAL`; routing checks its existing nine fields. + +These checks preserve the existing source meanings. In particular, a +`CHSP_YN` answer describes an obligation, and neither a no-obligation answer nor +an NIU paid amount establishes zero voluntary payments. Receipt, reporting, +allocation and top-code classifications are unchanged. The changes add no +model, canonical engine input, source issuer or native-data admission. + +Validation used invented mutations of the public domain artifact and actual +invented source-owner fixtures: 73 new domain/cache controls, 88 routing tests, +40 interest tests, 10 allocation tests and 40 child-support tests passed under +the existing bounded source guard. The new controls first reproduced 65 +failures against the prior implementation. These results certify the tested +source contracts, not a native dataset or release. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py index ba1cd9533..88f5d32ab 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_child_support_source.py @@ -65,7 +65,7 @@ def require(condition, reason): @lru_cache(maxsize=1) -def amount_entries(): +def _cached_amount_entries_json(): payload = ( resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() ) @@ -74,16 +74,28 @@ def amount_entries(): for field in json.loads(payload)["fields"]: if field["name"] not in AMOUNT_FIELDS: continue - vintage = next( + require(field["name"] not in entries, "DOMAIN_FIELD_DUPLICATE:" + field["name"]) + vintages = [ v for v in field["vintages"] if v["income_year"] == routing.CURRENT_INCOME_YEAR - ) + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:" + field["name"]) + vintage = vintages[0] require( - vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + vintage["dictionary_spelling"] == field["name"] + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 and vintage["source_url"] == routing.DICTIONARY_URL, "DICTIONARY_PIN", ) + routing._require_nonnegative_amount_domain( + field, + vintage, + zero_semantics="niu" + if field["name"] == "CHSP_VAL" + else "none_or_niu_not_distinguishable_from_amount_alone", + valid_minimum=1 if field["name"] == "CHSP_VAL" else 0, + ) entries[field["name"]] = { "width": vintage["ascii_length_as_printed"], "position": vintage["ascii_position_as_printed"], @@ -95,7 +107,29 @@ def amount_entries(): "temporal_authority": field["temporal_authority"], } require(set(entries) == set(AMOUNT_FIELDS), "AMOUNT_ROSTER") - return entries + return json.dumps(entries, separators=(",", ":")).encode() + + +def amount_entries(): + """Return deeply detached evidence from privately cached immutable bytes.""" + return json.loads(_cached_amount_entries_json()) + + +amount_entries.cache_clear = _cached_amount_entries_json.cache_clear + + +def _domain_agreement(ready): + routing._require_live_amount_domains( + ready, + { + name: ( + entry["domain"]["encoded_range_inclusive"]["minimum"], + entry["domain"]["encoded_range_inclusive"]["maximum"], + entry["domain"]["zero_semantics"], + ) + for name, entry in amount_entries().items() + }, + ) def _amount_pair(token, entry): @@ -321,6 +355,7 @@ def qualify_current_asec_child_support(preparation): ) parent = issued[2].parent ready = parent.ready() + _domain_agreement(ready) header, native_document = json.loads(ready.header), json.loads(issued[1]) require( header["target_year"] == routing.CURRENT_INCOME_YEAR diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py index cef6f66c8..29c406e65 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_income_routing_source.py @@ -86,7 +86,7 @@ class PrintedAmountEntry: @lru_cache(maxsize=1) -def printed_amount_entries(): +def _cached_printed_amount_entries(): """Read the pinned domains artifact once, on demand, never at import.""" payload = resources.files(__package__).joinpath(DOMAINS_RESOURCE).read_bytes() require(_sha(payload) == DOMAINS_SHA256, "DOMAINS_RESOURCE_SHA256") @@ -95,6 +95,7 @@ def printed_amount_entries(): for field in data["fields"]: if field["name"] not in AMOUNT_FIELDS: continue + require(field["name"] not in entries, "DOMAIN_FIELD_DUPLICATE:" + field["name"]) domain = field["domain"] vintages = [ v for v in field["vintages"] if v["income_year"] == CURRENT_INCOME_YEAR @@ -122,7 +123,66 @@ def printed_amount_entries(): zero_semantics=domain["zero_semantics"], ) require(set(entries) == set(AMOUNT_FIELDS), "DOMAIN_FIELD_ROSTER") - return entries + return tuple(entries.items()) + + +def printed_amount_entries(): + """Return a detached mapping over privately cached immutable entries.""" + return dict(_cached_printed_amount_entries()) + + +# Retain the existing test/revalidation cache-control surface. +printed_amount_entries.cache_clear = _cached_printed_amount_entries.cache_clear + + +def _require_nonnegative_amount_domain( + field, vintage, *, zero_semantics, valid_minimum +): + """Reject domains unsupported by the sibling unsigned-literal parsers. + + Amount bounds remain artifact-derived. The checks bind the exact semantics + assumed by those parsers rather than silently adapting a later artifact. + """ + name, domain = field["name"], field["domain"] + maximum = domain["encoded_range_inclusive"]["maximum"] + require( + field["entity"] == field["grain"] == "person" + and field["column"] == name + and type(maximum) is int + and maximum > 0 + and domain["encoded_range_inclusive"] == {"minimum": 0, "maximum": maximum} + and field["minimum"] == 0 + and field["maximum"] == maximum + and vintage["range_header"] == domain["encoded_range_inclusive"] + and domain["negative_dollars_permitted"] is False + and domain["declared_negative_nonmoney_codes"] == [] + and domain["declared_other_missing_codes"] == [] + and domain["declared_niu_codes"] == field["declared_niu_codes"] == [0] + and domain["valid_dollar_range_excludes"] == [] + and domain["valid_dollar_range_inclusive"] + == {"minimum": valid_minimum, "maximum": maximum} + and domain["numeric_type"] == "integer_US_dollars_in_public_use_dictionary" + and domain["zero_semantics"] == field["zero_semantics"] == zero_semantics, + "UNSUPPORTED_AMOUNT_DOMAIN:" + name, + ) + + +def _require_live_amount_domains(ready, entries): + """Bind source domain triples to a unique retained MoneyDomain roster.""" + fields = ready.bindings.spec.fields + domains = {d.name: d for d in fields} + require(len(domains) == len(fields), "MONEY_DOMAIN_DUPLICATE") + for name, (minimum, maximum, zero_semantics) in entries.items(): + domain = domains.get(name) + require(domain is not None, "MONEY_DOMAIN_MISSING:" + name) + require( + domain.entity == domain.grain == "person" + and domain.column == name + and domain.minimum == minimum + and domain.maximum == maximum + and domain.zero_semantics == zero_semantics, + "MONEY_DOMAIN_DISAGREEMENT:" + name, + ) # Printed yes/no entries. The zero label is not shared: OI_YN prints @@ -1108,17 +1168,13 @@ def amount_observations(field, positions, literals): def _domain_agreement(ready): """Bind printed entries to the live current-money domain contract.""" - domains = {d.name: d for d in ready.bindings.spec.fields} - for name, entry in printed_amount_entries().items(): - domain = domains.get(name) - require(domain is not None, "MONEY_DOMAIN_MISSING:" + name) - require( - domain.entity == "person" - and domain.minimum == entry.encoded_minimum - and domain.maximum == entry.encoded_maximum - and domain.zero_semantics == entry.zero_semantics, - "MONEY_DOMAIN_DISAGREEMENT:" + name, - ) + _require_live_amount_domains( + ready, + { + name: (entry.encoded_minimum, entry.encoded_maximum, entry.zero_semantics) + for name, entry in printed_amount_entries().items() + }, + ) @dataclass(frozen=True) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py index 36f3a89af..e1c42112a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_interest_source.py @@ -81,30 +81,58 @@ def require(condition, reason): @lru_cache(maxsize=1) -def amount_entries(): +def _cached_amount_entries(): payload = ( resources.files(__package__).joinpath(routing.DOMAINS_RESOURCE).read_bytes() ) require(routing._sha(payload) == routing.DOMAINS_SHA256, "DOMAINS_HASH") - field = next(f for f in json.loads(payload)["fields"] if f["name"] == "INT_VAL") - vintage = next(v for v in field["vintages"] if v["income_year"] == 2024) + fields = [f for f in json.loads(payload)["fields"] if f["name"] == "INT_VAL"] + require(len(fields) == 1, "DOMAIN_FIELD_ROSTER:INT_VAL") + field = fields[0] + vintages = [ + v for v in field["vintages"] if v["income_year"] == routing.CURRENT_INCOME_YEAR + ] + require(len(vintages) == 1, "DOMAIN_VINTAGE:INT_VAL") + vintage = vintages[0] require( - vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 + vintage["dictionary_spelling"] == "INT_VAL" + and vintage["pdf_sha256"] == routing.DICTIONARY_SHA256 and vintage["source_url"] == routing.DICTIONARY_URL, "DICTIONARY_PIN", ) - return { - "INT_VAL": ( - vintage["ascii_length_as_printed"], - vintage["ascii_position_as_printed"], - vintage["pdf_page_1based"], - vintage["printed_page"], - field["domain"]["encoded_range_inclusive"]["maximum"], - vintage["universe_as_printed"], - field["domain"]["zero_semantics"], - ), - **ADDITIONAL_AMOUNT_ENTRIES, - } + routing._require_nonnegative_amount_domain( + field, + vintage, + zero_semantics="none_or_niu_not_distinguishable_from_amount_alone", + valid_minimum=0, + ) + return tuple( + { + "INT_VAL": ( + vintage["ascii_length_as_printed"], + vintage["ascii_position_as_printed"], + vintage["pdf_page_1based"], + vintage["printed_page"], + field["domain"]["encoded_range_inclusive"]["maximum"], + vintage["universe_as_printed"], + field["domain"]["zero_semantics"], + ), + **ADDITIONAL_AMOUNT_ENTRIES, + }.items() + ) + + +def amount_entries(): + """Return a detached mapping over privately cached immutable tuples.""" + return dict(_cached_amount_entries()) + + +amount_entries.cache_clear = _cached_amount_entries.cache_clear + + +def _domain_agreement(ready): + entry = amount_entries()["INT_VAL"] + routing._require_live_amount_domains(ready, {"INT_VAL": (0, entry[4], entry[6])}) def _slot_status(age, receipt, account, amount): @@ -349,6 +377,7 @@ def qualify_current_asec_interest(preparation): ) parent = issued[2].parent ready = parent.ready() + _domain_agreement(ready) header, native_document = json.loads(ready.header), json.loads(issued[1]) require( header["target_year"] == 2024 diff --git a/packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py b/packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py new file mode 100644 index 000000000..27414a74d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_source_domain_hardening.py @@ -0,0 +1,217 @@ +"""Invented mutation/domain controls for source-only amount qualifiers.""" + +import copy +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import current_asec_child_support_source as child +from microcosm.build.us_runtime import current_asec_income_routing_source as routing +from microcosm.build.us_runtime import current_asec_interest_source as interest + +OWNERS = ((routing, "PNSN_VAL"), (interest, "INT_VAL"), (child, "CSP_VAL")) + + +def getter(owner): + return owner.printed_amount_entries if owner is routing else owner.amount_entries + + +def document(): + return json.loads( + routing.resources.files(routing.__package__) + .joinpath(routing.DOMAINS_RESOURCE) + .read_bytes() + ) + + +def patch_document(monkeypatch, owner, raw): + payload = json.dumps(raw).encode() + resource = SimpleNamespace(read_bytes=lambda: payload) + monkeypatch.setattr( + owner.resources, + "files", + lambda *_: SimpleNamespace(joinpath=lambda *_: resource), + ) + monkeypatch.setattr(routing, "DOMAINS_SHA256", hashlib.sha256(payload).hexdigest()) + getter(owner).cache_clear() + + +@pytest.mark.parametrize("owner,field", OWNERS) +def test_public_cache_mapping_cannot_change_later_calls(owner, field): + get = getter(owner) + get.cache_clear() + expected = copy.deepcopy(get()) + try: + get()[field] = "untrusted replacement" + assert get() == expected + get().clear() + assert get() == expected + finally: + get.cache_clear() + + +def test_child_nested_domain_and_temporal_metadata_are_detached(): + child.amount_entries.cache_clear() + expected = copy.deepcopy(child.amount_entries()) + try: + changed = child.amount_entries() + changed["CSP_VAL"]["width"] = 6 + changed["CSP_VAL"]["domain"]["zero_semantics"] = "valid_zero_dollars" + changed["CHSP_VAL"]["domain"]["declared_negative_nonmoney_codes"].append(-1) + changed["CHSP_VAL"]["temporal_authority"]["pdf_pages_1based"].append(999) + assert child.amount_entries() == expected + finally: + child.amount_entries.cache_clear() + + +@pytest.mark.parametrize("owner,field", OWNERS) +@pytest.mark.parametrize("defect", ["missing", "duplicate"]) +def test_current_vintage_must_exist_exactly_once(monkeypatch, owner, field, defect): + raw = document() + entry = next(f for f in raw["fields"] if f["name"] == field) + current = next(v for v in entry["vintages"] if v["income_year"] == 2024) + if defect == "missing": + entry["vintages"].remove(current) + else: + entry["vintages"].append(copy.deepcopy(current)) + patch_document(monkeypatch, owner, raw) + try: + with pytest.raises(ValueError, match="VINTAGE"): + getter(owner)() + finally: + getter(owner).cache_clear() + + +@pytest.mark.parametrize("owner,field", OWNERS) +@pytest.mark.parametrize("defect", ["missing", "duplicate"]) +def test_amount_field_must_exist_exactly_once(monkeypatch, owner, field, defect): + raw = document() + entry = next(f for f in raw["fields"] if f["name"] == field) + if defect == "missing": + raw["fields"].remove(entry) + else: + raw["fields"].append(copy.deepcopy(entry)) + patch_document(monkeypatch, owner, raw) + try: + with pytest.raises(ValueError, match="(ROSTER|FIELD)"): + getter(owner)() + finally: + getter(owner).cache_clear() + + +@pytest.mark.parametrize( + "owner,field", [(interest, "INT_VAL"), (child, "CSP_VAL"), (child, "CHSP_VAL")] +) +@pytest.mark.parametrize( + "defect", + [ + "negative_bound", + "negative_dollars", + "negative_niu", + "other_missing", + "excluded_dollars", + "extra_niu", + "zero_semantics", + "range_header", + "numeric_type", + "entity", + "grain", + "column", + ], +) +def test_repinned_unsupported_amount_domains_refuse(monkeypatch, owner, field, defect): + raw = document() + entry = next(f for f in raw["fields"] if f["name"] == field) + domain = entry["domain"] + if defect == "negative_bound": + domain["encoded_range_inclusive"]["minimum"] = -1 + elif defect == "negative_dollars": + domain["negative_dollars_permitted"] = True + elif defect == "negative_niu": + domain["declared_negative_nonmoney_codes"] = [-1] + elif defect == "other_missing": + domain["declared_other_missing_codes"] = [-2] + elif defect == "excluded_dollars": + domain["valid_dollar_range_excludes"] = [2] + elif defect == "extra_niu": + domain["declared_niu_codes"] = [0, 17] + elif defect == "zero_semantics": + domain["zero_semantics"] = "valid_zero_dollars" + elif defect == "range_header": + next(v for v in entry["vintages"] if v["income_year"] == 2024)["range_header"][ + "maximum" + ] += 1 + elif defect == "numeric_type": + domain["numeric_type"] = "decimal_US_dollars" + elif defect in ("entity", "grain"): + entry[defect] = "household" + else: + entry["column"] = "different_native_amount" + patch_document(monkeypatch, owner, raw) + try: + with pytest.raises(ValueError, match="DOMAIN"): + getter(owner)() + finally: + getter(owner).cache_clear() + + +def ready_domain(): + return SimpleNamespace( + bindings=SimpleNamespace( + spec=SimpleNamespace( + fields=tuple( + routing.money.MoneyDomain( + **{ + k: f[k] + for k in ( + "name", + "entity", + "column", + "grain", + "minimum", + "maximum", + "zero_semantics", + ) + } + ) + for f in document()["fields"] + ) + ) + ) + ) + + +@pytest.mark.parametrize("owner,field", OWNERS) +@pytest.mark.parametrize( + "attribute", ["entity", "column", "grain", "minimum", "maximum", "zero_semantics"] +) +def test_live_ready_domain_must_agree(owner, field, attribute): + ready = ready_domain() + owner._domain_agreement(ready) + value = { + "entity": "household", + "column": "different_native_amount", + "grain": "household", + "minimum": -5, + "maximum": 999999999, + "zero_semantics": "changed_zero_meaning", + }[attribute] + ready.bindings.spec.fields = tuple( + replace(d, **{attribute: value}) if d.name == field else d + for d in ready.bindings.spec.fields + ) + with pytest.raises(ValueError, match="DOMAIN_DISAGREEMENT"): + owner._domain_agreement(ready) + + +@pytest.mark.parametrize("owner,field", OWNERS) +def test_live_ready_domain_duplicates_are_not_overwritten(owner, field): + ready = ready_domain() + ready.bindings.spec.fields += tuple( + d for d in ready.bindings.spec.fields if d.name == field + ) + with pytest.raises(ValueError, match="DOMAIN"): + owner._domain_agreement(ready) diff --git a/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py index a3b4f0121..d3c820736 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py +++ b/packages/microcosm-build/tests/test_us_current_asec_child_support_source.py @@ -249,6 +249,24 @@ def test_actual_owner_keeps_obligation_distinct_and_refuses_copied_owner( with pytest.raises(ValueError): owner.qualify_current_asec_child_support(copy.copy(parent)) + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_child_support(parent) + assert checked == [True] + @pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "CSP_VAL", "CHSP_VAL"]) def test_actual_owner_requires_original_source_identity_and_amounts( diff --git a/packages/microcosm-build/tests/test_us_current_asec_interest_source.py b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py index f69dc380c..bf8e34334 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_interest_source.py +++ b/packages/microcosm-build/tests/test_us_current_asec_interest_source.py @@ -302,6 +302,24 @@ def test_actual_owner_join_keeps_source_total_and_requalifies(tmp_path, monkeypa with pytest.raises(ValueError): owner.qualify_current_asec_interest(copy.copy(parent)) + # The actual qualifier checks the retained MoneyDomain before source capture. + check, checked = owner._domain_agreement, [] + + def rejected_domain(ready): + check(ready) + checked.append(True) + raise ValueError("INVENTED_DOMAIN_REFUSAL") + + def forbidden_capture(*args): + pytest.fail("source capture preceded live domain agreement") + + with monkeypatch.context() as patch: + patch.setattr(owner, "_domain_agreement", rejected_domain) + patch.setattr(owner, "_capture_member", forbidden_capture) + with pytest.raises(ValueError, match="INVENTED_DOMAIN_REFUSAL"): + owner.qualify_current_asec_interest(parent) + assert checked == [True] + @pytest.mark.parametrize("coordinate", ["PH_SEQ", "A_LINENO", "A_AGE", "PERIDNUM"]) def test_actual_owner_refuses_changed_source_coordinates( From e2e38a3c75175abfeea3b8f25f0412afb6c100fb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:31:20 -0400 Subject: [PATCH 170/255] Format the property graph regression imports --- .../microcosm-build/tests/test_us_graph_property_income.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_graph_property_income.py b/packages/microcosm-build/tests/test_us_graph_property_income.py index 57fac80cc..33b4df968 100644 --- a/packages/microcosm-build/tests/test_us_graph_property_income.py +++ b/packages/microcosm-build/tests/test_us_graph_property_income.py @@ -15,8 +15,7 @@ PropertyIncomeTrainKernel, property_income_nodes, ) -from microcosm.fit import QRF -from microcosm.fit import graph_legacy_train +from microcosm.fit import QRF, graph_legacy_train from microcosm.fit.graph_legacy_qrf import LegacyQRFApplyKernel from microcosm.fit.graph_signed_reconciliation import SignedReconciliationKernel from microcosm.fit.signed_reconciliation import reconcile_signed_total From 80fcc7deedd020ba91208e5646fc2ea438f43a31 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:22:11 -0400 Subject: [PATCH 171/255] Build pure ASEC property donor basis and exclusion diagnostics --- .../us-current-asec-property-basis.added.md | 1 + docs/us-current-asec-property-basis.md | 123 +++++ .../us_runtime/current_asec_property_basis.py | 415 ++++++++++++++ .../test_us_current_asec_property_basis.py | 520 ++++++++++++++++++ 4 files changed, 1059 insertions(+) create mode 100644 changelog.d/us-current-asec-property-basis.added.md create mode 100644 docs/us-current-asec-property-basis.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_property_basis.py diff --git a/changelog.d/us-current-asec-property-basis.added.md b/changelog.d/us-current-asec-property-basis.added.md new file mode 100644 index 000000000..908dff17d --- /dev/null +++ b/changelog.d/us-current-asec-property-basis.added.md @@ -0,0 +1 @@ +Add a pure ASEC property donor basis with separate reported and component totals, explicit retirement-slot routing derivations, exact identity alignment and original-design-weight exclusion diagnostics. diff --git a/docs/us-current-asec-property-basis.md b/docs/us-current-asec-property-basis.md new file mode 100644 index 000000000..b4db0571d --- /dev/null +++ b/docs/us-current-asec-property-basis.md @@ -0,0 +1,123 @@ +# ASEC property donor basis + +`build_asec_property_basis` is a pure operation over the selected, qualified ASEC +person descriptions. It does not qualify a source, issue a handle, read files, +fit a model or assign tax leaves. The owning country host retains the actual +preparation and source owners, checks them before consumption and requalifies +and checks their complete seals after its last relevant I/O before return or +export. Reconstructing the descriptive input or output dataclasses grants no +source authority. + +## Inputs and outputs + +The keyword-only inputs are the `person` tables from the current interest, +income-routing and dividend/survivor qualifiers, named `interest`, +`income_routing` and `dividend`. Every table must have the same exact ordered +`person_id` index, matching unique `native_person_id` values and matching source +ages. Both ID axes are int64. Source ages may use different numeric storage +but must be finite, integer-valued ages 0–99 and equal without tolerance. No +sorting or inner join repairs a mismatch. A common row permutation is valid; +permuting one source alone refuses. + +`original_household_membership` is an int64 Series on that exact person index. +`original_household_design_weights` is a finite, nonnegative Series indexed by +unique int64 `household_id`, with exactly the membership's household set. +Extra, missing or duplicated household keys refuse. These inputs must be the +original household design mapping, before importance allocation or cloning; +this pure operation cannot authenticate that origin. Zero-weight persons are +retained. No engine or microsimulation aggregation occurs. + +The descriptive `AsecPropertyBasis` contains: + +- `person`: the component values, reported total, discrepancies, named retirement + derivations, original household mapping and the two eligibility masks. +- `provenance`: detached copies of all three supplied qualified tables, with + `interest.`, `income_routing.` and `dividend.` prefixes. Published amounts, + unknownness, allocation and disclosure indicators are preserved independently. +- `exclusions`: independent reasons for exclusion on the complete person axis. + A person can have multiple reasons; reason totals must not be summed as a + disjoint partition. +- `summary`: counts and original design-weight mass for the whole cohort, each + eligibility mask, the joint-fit complement and each exclusion reason. + +The shared names live in `property_income_constants.py`. `PROPERTY_COMPONENTS` +is, in order, `property_ordinary_interest`, `property_retirement_interest`, +`property_dividends`, `property_broad_receipts`. `PROPERTY_REPORTED_TOTAL` names +`property_reported_total`. They are donor observations or named routing +contributions, never already-reconciled model draws. + +## Measurement bridge + +Ordinary interest uses qualified `TRDINT_VAL`; dividends use qualified +`DIV_VAL`; broad signed property receipts use qualified `RNT_VAL`. The latter +is not a pure rental-income tax leaf. Dividends need a qualified yes/positive +or no/zero receipt. A receipt-yes zero for dividends or signed property stays +unknown under the published none-or-NIU coding. + +Retirement-account interest uses the two qualified account slots. Known no +with both slots NIU and zero supplies the already-qualified zero contributions. +Known yes requires at least one declared account with a known positive amount. +Every other slot must be another known positive account or an explicitly unused +slot: readable account code 0, published amount 0, in-range amount literal and +`unreported_account_slot` source status. The latter contributes a **derived** +zero and gets its own boolean, slot count and derivation label. Its original +canonical amount stays unknown in provenance. An amount zero by itself, +a declared account with unknown/zero amount, yes without any active account, +a missing code, or contradictory receipt/account evidence does not resolve the +retirement total. The derivation applies only when the complete route resolves. + +The reported donor total is separately retained as `INT_VAL + DIV_VAL + RNT_VAL`. +The component sum uses ordinary plus retirement-account interest, dividends and +broad property receipts. Both `interest_component_discrepancy` and +`reported_minus_component_total` remain visible, with signed magnitudes. No +balancing rewrites an ASEC observation. Allocation never filters the donor fit. +Disclosure flags do not prove the cause of a discrepancy. + +Other-income clearance requires known nonreceipt with a NIU category, or a +readable reported category outside property codes 5–8 and unspecified code 19. +A reported outside category resolves its route even if its own dollar amount +is ambiguous; no other-income amount is included in this basis. Missing, +unreadable or contradictory routing stays unresolved. Survivor clearance uses +the dividend qualifier's nullable clearance and checks agreement with its +readable codes: known no with NIU slots, or known yes with at least one active +source and every slot outside estate/trust code 8 and unspecified code 10. +Possible property overlaps and unknown routes have different exclusion reasons. +Neither other income nor survivor amounts enter the component or reported sum. + +`reported_total_eligible` requires age 15+, a known finite reported total, and +clear other-income and survivor routes. It is a diagnostic sample, not a second +aggregate model. `joint_component_fit_eligible` additionally requires all four +components and an exactly zero interest discrepancy. Missing or under-15 values +are never turned into analytic zeros. Negative and zero net totals remain valid: +`(100, 20, 0, -120)` has total zero and keeps both positive interest components. + +The summary reports `design_weighted_person_mass` (one mapped original household +design weight per selected person) separately from `union_household_design_mass` +(each household with at least one selected person counted once). Overlapping +selection groups can share household mass. Stable finite summation avoids +row-order-dependent loss of small weights. These are coverage diagnostics, not +calibrated population claims. Conditional donor support, bias from exclusions +and held-out model quality still need the model owner's diagnostics. + +## Source and acceptance boundaries + +The source contracts use the pinned +[2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +(SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`). +The bridge targets the broad property concept in the +[2024 ACS questionnaire](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18). +The live ACS guide follows the 2025 form; its retirement-interest wording and +the historical 2016 Census methods discussion corroborate the concept without +establishing vintage or respondent identity. ASEC calendar-year and ACS +rolling-year measurement remain different. Broad property routing and +retirement-account earnings remain distinct from taxability. + +The bounded tests use invented tables and a composition test through the real +pure source projectors using invented literals. They cover both identity axes, +permutations, missing/contradictory routes, unknown account slots, signed and +zero-net examples, separate masks, allocation preservation, weight mapping, +empty/zero-weight records and refusal of malformed/nonfinite descriptions. +They do not run source qualification, native data, the country model, fitting, +cloning or replay. The later graph owns joint draws and signed reconciliation; +the later host must preserve original ASEC observations and share survey draws +across original/PUF clone pairs before the PUF alternative takes ownership. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py new file mode 100644 index 000000000..e46f158d4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py @@ -0,0 +1,415 @@ +"""Pure, descriptive ASEC property donor basis; never a source authority. + +The caller retains the preparation and qualified-source owners around I/O. +These tables establish neither a source seal nor a tax treatment. Original +household design weights describe coverage; they do not alter observations. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import fsum + +import numpy as np +import pandas as pd + +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + +# Modeling bridge exclusions, not alternative source codebooks. +OTHER_PROPERTY_CATEGORIES = frozenset({5, 6, 7, 8}) +OTHER_UNSPECIFIED_CATEGORY = 19 + + +@dataclass(frozen=True) +class AsecPropertyBasis: + """Detached descriptive outputs. Constructing or copying grants no authority.""" + + person: pd.DataFrame + provenance: pd.DataFrame + exclusions: pd.DataFrame + summary: pd.DataFrame + + +def _require(condition, reason): + if not condition: + raise ValueError("ASEC_PROPERTY_BASIS_" + reason) + + +def _axis(frame): + _require(type(frame) is pd.DataFrame, "TABLE") + _require( + frame.index.name == "person_id" + and frame.index.dtype == np.dtype("int64") + and frame.index.is_unique + and frame.columns.is_unique, + "PERSON_AXIS", + ) + _require({"native_person_id", "source_age"} <= set(frame), "IDENTITY_COLUMNS") + native = frame.native_person_id + _require( + native.dtype == np.dtype("int64") and native.is_unique, + "NATIVE_AXIS", + ) + age = _numeric(frame.source_age, "AGE") + _require( + np.isfinite(age).all() + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "AGE", + ) + return age + + +def _numeric(series, label): + _require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "NUMERIC:" + label, + ) + values = series.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + _require(not np.isinf(values).any(), "NONFINITE:" + label) + return values + + +def _text(frame, name): + _require(name in frame, "COLUMN:" + name) + series = frame[name] + _require( + series.notna().all() and all(type(v) is str for v in series), + "STATUS:" + name, + ) + return series.to_numpy(copy=True) + + +def _code(frame, name): + _require(name + "_code" in frame, "COLUMN:" + name) + code = _numeric(frame[name + "_code"], name) + status = _text(frame, name + "_literal_status") + _require(((np.isnan(code)) | (code == np.floor(code))).all(), "CODE:" + name) + return code, status == "in_printed_range" + + +def _amount(frame, name, *, signed=False, ordinary=False): + for suffix in ("_amount", "_amount_known", "_reporting_status"): + _require(name + suffix in frame, "COLUMN:" + name + suffix) + amount = _numeric(frame[name + "_amount"], name) + known = frame[name + "_amount_known"] + _require( + pd.api.types.is_bool_dtype(known.dtype) and known.notna().all(), + "KNOWNNESS:" + name, + ) + known = known.to_numpy(dtype=bool, copy=True) + status = _text(frame, name + "_reporting_status") + allowed = np.isin(status, ["known_receipt", "known_nonreceipt"]) + if ordinary: + allowed |= status == "observed_zero_component" + _require( + np.array_equal(known, np.isfinite(amount)) + and np.array_equal(known, allowed) + and (signed or (amount[known] >= 0).all()) + and (amount[status == "known_nonreceipt"] == 0).all(), + "AMOUNT_CONTRACT:" + name, + ) + return amount + + +def _retirement(interest): + receipt, readable = _code(interest, "RINT_YN") + slots = [] + unused = [] + nonreceipt = [] + for n in (1, 2): + name = f"RINT_VAL{n}" + amount = _amount(interest, name) + code, valid = _code(interest, f"RINT_SC{n}") + published = _numeric(interest[name + "_published_amount"], name) + literal = _text(interest, name + "_literal_status") + status = _text(interest, name + "_reporting_status") + active = ( + valid + & (code >= 1) + & (code <= 7) + & (amount > 0) + & (status == "known_receipt") + ) + structural = ( + valid + & (code == 0) + & (published == 0) + & (literal == "in_printed_range") + & (status == "unreported_account_slot") + & np.isnan(amount) + ) + no = ( + valid + & (code == 0) + & (published == 0) + & (amount == 0) + & (literal == "in_printed_range") + & (status == "known_nonreceipt") + ) + slots.append((amount, active)) + unused.append(structural) + nonreceipt.append(no) + yes = readable & (receipt == 1) + resolved_yes = ( + yes + & (slots[0][1] | slots[1][1]) + & (slots[0][1] | unused[0]) + & (slots[1][1] | unused[1]) + ) + resolved_no = readable & (receipt == 2) & nonreceipt[0] & nonreceipt[1] + total = np.full(len(interest), np.nan) + total[resolved_no] = 0 + with np.errstate(over="raise", invalid="raise"): + total[resolved_yes] = ( + np.where(unused[0], 0, slots[0][0])[resolved_yes] + + np.where(unused[1], 0, slots[1][0])[resolved_yes] + ) + derived = [u & resolved_yes for u in unused] + count = derived[0].astype(np.int8) + derived[1].astype(np.int8) + label = np.full(len(interest), "unresolved_retirement_interest", dtype=object) + label[resolved_no] = "known_nonreceipt_sum" + label[resolved_yes] = "declared_account_sum" + label[count > 0] = "declared_account_sum_with_unused_slot_zero" + return total, count, label, derived + + +def _other_route(routing): + receipt, receipt_ok = _code(routing, "other_income_receipt") + category, category_ok = _code(routing, "other_income_category") + status = _text(routing, "other_income_reporting_status") + route = _text(routing, "other_income_routing_status") + reported = ( + receipt_ok & category_ok & (receipt == 1) & (route == "reported_category") + ) + possible = reported & np.isin(category, tuple(OTHER_PROPERTY_CATEGORIES)) + clear = ( + receipt_ok + & category_ok + & (receipt == 2) + & (category == 0) + & (status == "known_nonreceipt") + & (route == "niu_category") + ) | ( + reported + & (category >= 1) + & (category <= 20) + & ~np.isin(category, (*OTHER_PROPERTY_CATEGORIES, OTHER_UNSPECIFIED_CATEGORY)) + ) + return clear, possible + + +def _survivor_route(dividend, age): + # The qualifier already owns this route's source interpretation. Check the + # lightweight declared code/status agreement, never reread its source here. + receipt, receipt_ok = _code(dividend, "SUR_YN") + first, first_ok = _code(dividend, "SUR_SC1") + second, second_ok = _code(dividend, "SUR_SC2") + readable = first_ok & second_ok + yes = receipt_ok & (receipt == 1) + possible = yes & ((first == 8) | (second == 8)) + clear = (receipt_ok & (receipt == 2) & readable & (first == 0) & (second == 0)) | ( + yes + & readable + & ((first > 0) | (second > 0)) + & (first >= 0) + & (first <= 9) + & (first != 8) + & (second >= 0) + & (second <= 9) + & (second != 8) + ) + clear &= age >= 15 + possible &= age >= 15 + declared = dividend.survivor_property_route_clear + _require( + pd.api.types.is_bool_dtype(declared.dtype) + and np.array_equal(clear, declared.fillna(False).to_numpy(dtype=bool)) + and np.array_equal( + possible, declared.eq(False).fillna(False).to_numpy(dtype=bool) + ), + "SURVIVOR_ROUTE_AGREEMENT", + ) + return clear, possible + + +def _weights(index, membership, weights): + _require( + type(membership) is pd.Series + and membership.index.equals(index) + and membership.index.name == index.name + and membership.dtype == np.dtype("int64"), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + type(weights) is pd.Series + and weights.index.dtype == np.dtype("int64") + and weights.index.name == "household_id" + and weights.index.is_unique + and set(membership) == set(weights.index), + "DESIGN_WEIGHT_MEMBERSHIP", + ) + values = _numeric(weights, "DESIGN_WEIGHTS") + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_WEIGHTS") + return weights.loc[membership].to_numpy(dtype=np.float64, copy=True) + + +def build_asec_property_basis( + *, + interest: pd.DataFrame, + income_routing: pd.DataFrame, + dividend: pd.DataFrame, + original_household_membership: pd.Series, + original_household_design_weights: pd.Series, +) -> AsecPropertyBasis: + """Compose exactly aligned *original*, qualified ASEC person descriptions. + + No sorting, inner joins, dropping, imputation or source qualification occurs. + A jointly permuted input is valid; a permutation of one source alone refuses. + The two masks distinguish reported-total eligibility from complete, balanced + joint-component fit eligibility. Neither requests a second aggregate model. + All mass summaries use explicitly supplied original household design weights. + """ + age = _axis(interest) + for frame in (income_routing, dividend): + other_age = _axis(frame) + _require(frame.index.equals(interest.index), "PERSON_ALIGNMENT") + _require( + frame.native_person_id.equals(interest.native_person_id), "NATIVE_ALIGNMENT" + ) + _require(np.array_equal(age, other_age), "AGE_ALIGNMENT") + design = _weights( + interest.index, original_household_membership, original_household_design_weights + ) + ordinary = _amount(interest, "TRDINT_VAL", ordinary=True) + reported_interest = _amount(interest, "INT_VAL") + dividends = _amount(dividend, "DIV_VAL") + div_receipt, div_readable = _code(dividend, "DIV_YN") + _require( + ( + ~np.isfinite(dividends) + | ( + div_readable + & ( + ((div_receipt == 1) & (dividends > 0)) + | ((div_receipt == 2) & (dividends == 0)) + ) + ) + ).all(), + "DIVIDEND_RECEIPT_AGREEMENT", + ) + property_amount = _numeric(income_routing.net_property_known_amount, "RNT_VAL") + property_status = _text(income_routing, "net_property_reporting_status") + _require( + np.array_equal( + np.isfinite(property_amount), + np.isin(property_status, ["known_receipt", "known_nonreceipt"]), + ) + and (property_amount[property_status == "known_nonreceipt"] == 0).all() + and (property_amount[property_status == "known_receipt"] != 0).all(), + "PROPERTY_KNOWNNESS", + ) + retirement, count, derivation, derived_slots = _retirement(interest) + other_clear, other_possible = _other_route(income_routing) + survivor_clear, survivor_possible = _survivor_route(dividend, age) + person = interest[["native_person_id", "source_age"]].copy(deep=True) + person["original_household_id"] = original_household_membership.to_numpy(copy=True) + person["original_household_design_weight"] = design + for name, amount in zip( + PROPERTY_COMPONENTS, + (ordinary, retirement, dividends, property_amount), + strict=True, + ): + person[name] = amount + with np.errstate(over="raise", invalid="raise"): + person[PROPERTY_REPORTED_TOTAL] = ( + reported_interest + dividends + property_amount + ) + person["property_component_sum"] = ( + ordinary + retirement + dividends + property_amount + ) + person["interest_component_discrepancy"] = reported_interest - ( + ordinary + retirement + ) + person["reported_minus_component_total"] = ( + person[PROPERTY_REPORTED_TOTAL].to_numpy() + - person.property_component_sum.to_numpy() + ) + person["retirement_interest_derivation"] = pd.array(derivation, dtype="string") + person["retirement_structural_zero_slot_count"] = count + for n, values in enumerate(derived_slots, 1): + person[f"retirement_slot{n}_derived_unused_zero"] = values + exclusions = pd.DataFrame( + { + "under15": age < 15, + "reported_total_unknown": person[PROPERTY_REPORTED_TOTAL].isna(), + "ordinary_interest_unknown": np.isnan(ordinary), + "retirement_interest_unknown": np.isnan(retirement), + "dividends_unknown": np.isnan(dividends), + "property_receipts_unknown": np.isnan(property_amount), + "interest_discrepancy_unknown": person.interest_component_discrepancy.isna(), + "interest_discrepancy_nonzero": person.interest_component_discrepancy.notna() + & person.interest_component_discrepancy.ne(0), + "other_income_possible_property": other_possible, + "other_income_route_unresolved": ~other_clear & ~other_possible, + "survivor_possible_property": survivor_possible, + "survivor_route_unresolved": ~survivor_clear & ~survivor_possible, + }, + index=interest.index, + ) + person["reported_total_eligible"] = ( + (age >= 15) + & np.isfinite(person[PROPERTY_REPORTED_TOTAL]) + & other_clear + & survivor_clear + ) + person["joint_component_fit_eligible"] = ~exclusions.any(axis=1) + # Detached qualified columns preserve source statuses, amounts, allocation + # and disclosure flags. Prefixes make disagreements visible without choosing + # one qualifier's value or attributing a discrepancy to disclosure treatment. + provenance = pd.concat( + [ + frame.copy(deep=True).add_prefix(prefix + ".") + for prefix, frame in ( + ("interest", interest), + ("income_routing", income_routing), + ("dividend", dividend), + ) + ], + axis=1, + ) + masks = {"all": np.ones(len(person), dtype=bool)} + masks.update( + { + name: person[name].to_numpy() + for name in ("reported_total_eligible", "joint_component_fit_eligible") + } + ) + masks["excluded_joint_component_fit"] = ~masks["joint_component_fit_eligible"] + masks.update( + {"excluded:" + name: exclusions[name].to_numpy() for name in exclusions} + ) + rows = [] + for name, mask in masks.items(): + households = original_household_membership.iloc[np.flatnonzero(mask)].unique() + with np.errstate(over="raise", invalid="raise"): + mass = fsum(design[mask]) + household_mass = fsum( + original_household_design_weights.loc[households].to_numpy( + dtype=np.float64 + ) + ) + _require(np.isfinite(mass) and np.isfinite(household_mass), "MASS_OVERFLOW") + rows.append((name, int(mask.sum()), len(households), mass, household_mass)) + summary = pd.DataFrame( + rows, + columns=[ + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ], + ).set_index("selection") + return AsecPropertyBasis(person, provenance, exclusions, summary) diff --git a/packages/microcosm-build/tests/test_us_current_asec_property_basis.py b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py new file mode 100644 index 000000000..1d9dced98 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py @@ -0,0 +1,520 @@ +"""Pure invented descriptions; no source files, qualification, models or engines.""" + +import copy + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from microcosm.build.us_runtime.current_asec_property_basis import ( + build_asec_property_basis, +) +from microcosm.build.us_runtime.property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_REPORTED_TOTAL, +) + + +def code(frame, name, values, statuses=None): + frame[name + "_code"] = pd.array(values, dtype="Int16") + frame[name + "_literal_status"] = pd.array( + statuses or ["in_printed_range"] * len(values), dtype="string" + ) + + +def amount(frame, name, values, statuses=None): + if statuses is None: + statuses = ["known_nonreceipt" if v == 0 else "known_receipt" for v in values] + frame[name + "_published_amount"] = pd.array(values, dtype="Float64") + frame[name + "_literal_status"] = pd.array( + ["in_printed_range"] * len(values), dtype="string" + ) + frame[name + "_reporting_status"] = pd.array(statuses, dtype="string") + known = np.isin( + statuses, ["known_receipt", "known_nonreceipt", "observed_zero_component"] + ) + frame[name + "_amount_known"] = known + frame[name + "_amount"] = frame[name + "_published_amount"].where(known) + + +def inputs(n=3): + index = pd.Index(np.arange(n, dtype=np.int64) + 101, name="person_id") + base = pd.DataFrame( + { + "native_person_id": np.arange(n, dtype=np.int64) + 901, + "source_age": np.full(n, 40.0), + }, + index=index, + ) + interest, routing, dividend = (base.copy() for _ in range(3)) + amount(interest, "INT_VAL", [100.0] * n) + amount(interest, "TRDINT_VAL", [100.0] * n) + code(interest, "RINT_YN", [2] * n) + for k in (1, 2): + amount(interest, f"RINT_VAL{k}", [0.0] * n) + code(interest, f"RINT_SC{k}", [0] * n) + interest["TRINT_VAL1_literal"] = pd.array(["0"] * n, dtype="string") + interest["allocation_origin"] = pd.array( + ["publisher_allocated"] * n, dtype="string" + ) + routing["net_property_known_amount"] = np.full(n, -110.0) + routing["net_property_reporting_status"] = pd.array( + ["known_receipt"] * n, dtype="string" + ) + code(routing, "other_income_receipt", [2] * n) + code(routing, "other_income_category", [0] * n) + routing["other_income_reporting_status"] = pd.array( + ["known_nonreceipt"] * n, dtype="string" + ) + routing["other_income_routing_status"] = pd.array( + ["niu_category"] * n, dtype="string" + ) + amount(dividend, "DIV_VAL", [0.0] * n) + code(dividend, "DIV_YN", [2] * n) + code(dividend, "SUR_YN", [2] * n) + code(dividend, "SUR_SC1", [0] * n) + code(dividend, "SUR_SC2", [0] * n) + dividend["survivor_property_route_clear"] = pd.array([True] * n, dtype="boolean") + membership = pd.Series( + np.arange(n, dtype=np.int64) // 2 + 41, index=index, name="household_id" + ) + households = np.unique(membership) + weights = pd.Series( + np.arange(len(households), dtype=np.float64) * 3 + 2, + index=pd.Index(households, name="household_id"), + ) + return dict( + interest=interest, + income_routing=routing, + dividend=dividend, + original_household_membership=membership, + original_household_design_weights=weights, + ) + + +@pytest.mark.parametrize("rental,total", [(-110, -10), (-100, 0), (-40, 60), (50, 150)]) +def test_signed_anchors_preserve_positive_interest_and_losses(rental, total): + source = inputs() + source["income_routing"]["net_property_known_amount"] = float(rental) + before = copy.deepcopy(source) + result = build_asec_property_basis(**source) + assert result.person[PROPERTY_COMPONENTS[0]].eq(100).all() + assert result.person[PROPERTY_COMPONENTS[3]].eq(rental).all() + assert result.person[PROPERTY_REPORTED_TOTAL].eq(total).all() + assert result.person.joint_component_fit_eligible.all() + assert result.person.retirement_structural_zero_slot_count.eq(0).all() + assert ( + result.provenance["interest.allocation_origin"].eq("publisher_allocated").all() + ) + for name in ("interest", "income_routing", "dividend"): + assert_frame_equal(source[name], before[name]) + result.provenance.iloc[0, 0] = -9 + assert source["interest"].iloc[0, 0] == 901 + + +def test_named_unused_slot_derivation_keeps_published_niu_unknown(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + amount(i, "INT_VAL", [120]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_retirement_interest == 20 + assert row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 1 + assert row.retirement_slot2_derived_unused_zero + assert ( + row.retirement_interest_derivation + == "declared_account_sum_with_unused_slot_zero" + ) + assert pd.isna(result.provenance.iloc[0]["interest.RINT_VAL2_amount"]) + assert result.provenance.iloc[0]["interest.RINT_VAL2_published_amount"] == 0 + + +@pytest.mark.parametrize( + "case", + [ + "zero_declared", + "missing_code", + "missing_amount", + "all_unused", + "niu_receipt", + "no_with_account", + ], +) +def test_unresolved_retirement_never_completes_unknown_slots(case): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + code(i, "RINT_SC1", [4]) + amount(i, "RINT_VAL1", [20]) + amount(i, "RINT_VAL2", [0], ["unreported_account_slot"]) + if case == "zero_declared": + code(i, "RINT_SC2", [4]) + amount(i, "RINT_VAL2", [0], ["ambiguous_recipient_zero"]) + elif case == "missing_code": + code(i, "RINT_SC2", [None], ["missing"]) + elif case == "missing_amount": + amount(i, "RINT_VAL2", [np.nan], ["missing_amount"]) + elif case == "all_unused": + code(i, "RINT_SC1", [0]) + amount(i, "RINT_VAL1", [0], ["unreported_account_slot"]) + elif case == "niu_receipt": + code(i, "RINT_YN", [0]) + else: + code(i, "RINT_YN", [2]) + row = build_asec_property_basis(**source).person.iloc[0] + assert np.isnan(row.property_retirement_interest) + assert row.reported_total_eligible + assert not row.joint_component_fit_eligible + assert row.retirement_structural_zero_slot_count == 0 + + +def test_two_active_accounts_sum_without_a_tax_label(): + source = inputs(1) + i = source["interest"] + code(i, "RINT_YN", [1]) + for n, value in ((1, 20), (2, 30)): + code(i, f"RINT_SC{n}", [n]) + amount(i, f"RINT_VAL{n}", [value]) + amount(i, "INT_VAL", [150]) + result = build_asec_property_basis(**source) + assert result.person.property_retirement_interest.iloc[0] == 50 + assert result.person.joint_component_fit_eligible.iloc[0] + assert ( + result.person.retirement_interest_derivation.iloc[0] == "declared_account_sum" + ) + + +def test_reported_aggregate_discrepancy_is_unprojected_and_excluded(): + source = inputs(1) + amount(source["interest"], "INT_VAL", [110]) + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row.property_component_sum == -10 + assert row.interest_component_discrepancy == 10 + assert row.reported_minus_component_total == 10 + assert row.reported_total_eligible and not row.joint_component_fit_eligible + assert result.exclusions.interest_discrepancy_nonzero.iloc[0] + assert not any("cause" in column for column in result.person) + + +@pytest.mark.parametrize( + "receipt,category,status,route,clear,overlap", + [ + (1, 20, "known_receipt", "reported_category", True, False), + (1, 2, "ambiguous_recipient_zero", "reported_category", True, False), + (1, 5, "known_receipt", "reported_category", False, True), + (1, 8, "known_receipt", "reported_category", False, True), + (1, 19, "known_receipt", "reported_category", False, False), + (1, 0, "known_receipt", "receipt_without_category", False, False), + (2, 5, "known_nonreceipt", "category_without_receipt", False, False), + (0, 0, "niu", "niu_category", False, False), + ( + None, + None, + "missing_receipt_literal", + "missing_category_literal", + False, + False, + ), + ], +) +def test_other_income_routes_are_strict_and_never_added( + receipt, category, status, route, clear, overlap +): + source = inputs(1) + r = source["income_routing"] + code( + r, + "other_income_receipt", + [receipt], + ["missing" if receipt is None else "in_printed_range"], + ) + code( + r, + "other_income_category", + [category], + ["missing" if category is None else "in_printed_range"], + ) + r["other_income_reporting_status"] = status + r["other_income_routing_status"] = route + result = build_asec_property_basis(**source) + assert bool(result.person.reported_total_eligible.iloc[0]) == clear + assert bool(result.exclusions.other_income_possible_property.iloc[0]) == overlap + assert result.person.property_reported_total.iloc[0] == -10 + + +@pytest.mark.parametrize( + "receipt,first,second,declared", + [ + (1, 1, 0, True), + (1, 0, 9, True), + (1, 8, 0, False), + (1, 8, None, False), + (1, 10, 0, None), + (1, 1, None, None), + (1, 0, 0, None), + (2, 8, 0, None), + (0, 0, 0, None), + ], +) +def test_survivor_routes_keep_overlap_and_unknown_exclusions( + receipt, first, second, declared +): + source = inputs(1) + d = source["dividend"] + for name, value in (("SUR_YN", receipt), ("SUR_SC1", first), ("SUR_SC2", second)): + code(d, name, [value], ["missing" if value is None else "in_printed_range"]) + d["survivor_property_route_clear"] = pd.array([declared], dtype="boolean") + result = build_asec_property_basis(**source) + assert bool(result.person.reported_total_eligible.iloc[0]) is (declared is True) + assert bool(result.exclusions.survivor_possible_property.iloc[0]) is ( + declared is False + ) + + +@pytest.mark.parametrize( + "axis", ["person", "native", "age", "duplicate_native", "duplicate_person"] +) +def test_both_identity_axes_and_age_must_match_exactly(axis): + source = inputs() + d = source["dividend"] + if axis == "person": + source["dividend"] = d.iloc[::-1] + elif axis == "native": + d["native_person_id"] = d.native_person_id.to_numpy()[::-1] + elif axis == "age": + d.iloc[0, d.columns.get_loc("source_age")] = 41 + elif axis == "duplicate_native": + d["native_person_id"] = 901 + else: + d.index = pd.Index([101, 101, 103], name="person_id") + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_joint_permutation_and_household_weight_order_preserve_result(): + source = inputs() + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[[2, 0, 1]] + source["original_household_design_weights"] = source[ + "original_household_design_weights" + ].iloc[::-1] + permuted = build_asec_property_basis(**source) + assert_frame_equal(original.person, permuted.person.loc[original.person.index]) + assert_frame_equal(original.summary, permuted.summary) + + +def test_weighted_exclusions_report_person_and_union_household_mass(): + source = inputs() + source["interest"].loc[101, "INT_VAL_amount"] = 110 + result = build_asec_property_basis(**source) + all_rows = result.summary.loc["all"] + excluded = result.summary.loc["excluded_joint_component_fit"] + assert all_rows.design_weighted_person_mass == 9 + assert all_rows.union_household_design_mass == 7 + assert excluded.person_count == 1 + assert excluded.design_weighted_person_mass == 2 + assert excluded.union_household_design_mass == 2 + + +@pytest.mark.parametrize( + "case", + [ + "extra_household", + "missing_household", + "duplicate_household", + "bad_person_order", + "negative_weight", + "infinite_weight", + ], +) +def test_only_exact_original_household_design_mapping_is_admitted(case): + source = inputs() + weights = source["original_household_design_weights"] + if case == "extra_household": + weights.loc[999] = 4 + elif case == "missing_household": + source["original_household_design_weights"] = weights.iloc[:1] + elif case == "duplicate_household": + weights.index = pd.Index([41, 41], name="household_id") + elif case == "bad_person_order": + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + else: + weights.iloc[0] = -1 if case == "negative_weight" else np.inf + with pytest.raises(ValueError): + build_asec_property_basis(**source) + + +def test_empty_and_zero_weight_records_are_retained(): + empty = build_asec_property_basis(**inputs(0)) + assert len(empty.person) == 0 + assert empty.summary.person_count.eq(0).all() + source = inputs() + source["original_household_design_weights"].iloc[:] = 0 + result = build_asec_property_basis(**source) + assert len(result.person) == 3 and result.person.joint_component_fit_eligible.all() + assert result.summary.design_weighted_person_mass.eq(0).all() + + +def test_under15_unknowns_are_excluded_without_analytic_zero(): + source = inputs(1) + for key in ("interest", "income_routing", "dividend"): + source[key]["source_age"] = 14.0 + i = source["interest"] + for name in ("INT_VAL", "TRDINT_VAL", "RINT_VAL1", "RINT_VAL2"): + amount(i, name, [0], ["outside_reporting_universe"]) + code(i, "RINT_YN", [0]) + d = source["dividend"] + amount(d, "DIV_VAL", [0], ["outside_reporting_universe"]) + code(d, "DIV_YN", [0]) + code(d, "SUR_YN", [0]) + d["survivor_property_route_clear"] = pd.array([None], dtype="boolean") + r = source["income_routing"] + r["net_property_known_amount"] = np.nan + r["net_property_reporting_status"] = "outside_reporting_universe" + result = build_asec_property_basis(**source) + assert result.person[list(PROPERTY_COMPONENTS)].isna().all().all() + assert result.exclusions.under15.iloc[0] + assert not result.person.reported_total_eligible.iloc[0] + + +@pytest.mark.parametrize( + "case", + [ + "nonfinite", + "negative_interest", + "false_knownness", + "dividend_no_positive", + "rent_receipt_zero", + "survivor_false_clear", + "overflow", + ], +) +def test_invalid_descriptions_and_overflow_refuse(case): + source = inputs(1) + if case in ("nonfinite", "negative_interest"): + source["interest"].loc[101, "TRDINT_VAL_amount"] = ( + np.inf if case == "nonfinite" else -1 + ) + elif case == "false_knownness": + source["interest"]["INT_VAL_amount_known"] = False + elif case == "dividend_no_positive": + amount(source["dividend"], "DIV_VAL", [10]) + elif case == "rent_receipt_zero": + source["income_routing"]["net_property_known_amount"] = 0.0 + elif case == "survivor_false_clear": + source["dividend"]["survivor_property_route_clear"] = pd.array( + [False], dtype="boolean" + ) + else: + amount(source["interest"], "INT_VAL", [1.7e308]) + source["income_routing"]["net_property_known_amount"] = 1.7e308 + with pytest.raises((ValueError, FloatingPointError)): + build_asec_property_basis(**source) + + +def test_weight_summaries_are_stable_under_widely_different_weight_permutation(): + source = inputs() + source["original_household_membership"] = pd.Series( + [41, 42, 43], index=source["interest"].index, dtype="int64" + ) + source["original_household_design_weights"] = pd.Series( + [1e16, 1.0, 1.0], index=pd.Index([41, 42, 43], name="household_id") + ) + original = build_asec_property_basis(**source) + for key in ( + "interest", + "income_routing", + "dividend", + "original_household_membership", + ): + source[key] = source[key].iloc[::-1] + reordered = build_asec_property_basis(**source) + assert_frame_equal(original.summary, reordered.summary) + assert original.summary.loc["all", "design_weighted_person_mass"] == 1e16 + 2 + + +def test_fully_qualified_nonreceipt_zeros_remain_eligible(): + source = inputs(1) + for name in ("INT_VAL", "TRDINT_VAL"): + amount(source["interest"], name, [0]) + source["income_routing"]["net_property_known_amount"] = 0.0 + source["income_routing"]["net_property_reporting_status"] = "known_nonreceipt" + row = build_asec_property_basis(**source).person.iloc[0] + assert row[list(PROPERTY_COMPONENTS)].eq(0).all() + assert row.property_reported_total == 0 and row.joint_component_fit_eligible + + +@pytest.mark.parametrize("field", ["DIV_VAL", "RNT_VAL"]) +def test_receipt_yes_zero_stays_unknown_instead_of_eligible(field): + source = inputs(1) + if field == "DIV_VAL": + amount(source["dividend"], field, [0], ["ambiguous_recipient_zero"]) + code(source["dividend"], "DIV_YN", [1]) + else: + source["income_routing"]["net_property_known_amount"] = np.nan + source["income_routing"]["net_property_reporting_status"] = ( + "receipt_with_net_zero" + ) + result = build_asec_property_basis(**source) + assert np.isnan(result.person.property_reported_total.iloc[0]) + assert not result.person.reported_total_eligible.iloc[0] + assert not result.person.joint_component_fit_eligible.iloc[0] + + +def test_actual_pure_projector_column_composition_with_invented_literals(): + from test_us_current_asec_income_routing import _pure_rows, _set_amount + + from microcosm.build.us_runtime import current_asec_dividend_source as dividend + from microcosm.build.us_runtime import current_asec_income_routing_source as routing + from microcosm.build.us_runtime import current_asec_interest_source as interest + + source = inputs(1) + raw_interest = {name: "0" for name in interest.READ_COLUMNS} + raw_interest.update( + A_AGE="40", + INT_YN="1", + INT_VAL="120", + TRDINT_VAL="100", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="20", + ) + raw_dividend = {name: "0" for name in dividend.READ_COLUMNS} + raw_dividend.update(A_AGE="40", DIV_YN="2", SUR_YN="2") + raw_routing, ages = _pure_rows(1, [40], RNT_YN=["1"], OI_YN=["2"]) + _set_amount(raw_routing, "RNT_VAL", [-120]) + projected = { + "interest": interest.project_interest_literals(pd.DataFrame([raw_interest])), + "income_routing": routing.project_income_routing(raw_routing, ages), + "dividend": dividend.project_dividend_literals(pd.DataFrame([raw_dividend])), + } + for name, frame in projected.items(): + frame.index = source[name].index + frame["native_person_id"] = source[name].native_person_id + source[name] = frame + result = build_asec_property_basis(**source) + row = result.person.iloc[0] + assert row.property_reported_total == 0 + assert row[list(PROPERTY_COMPONENTS)].tolist() == [100, 20, 0, -120] + assert row.joint_component_fit_eligible + assert row.retirement_slot2_derived_unused_zero + assert ( + result.provenance.iloc[0]["interest.RINT_VAL2_reporting_status"] + == "unreported_account_slot" + ) From bc278d9808e41820b175da4c1b036c283929d6d7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:47:12 -0400 Subject: [PATCH 172/255] Exclude unqualified extra survivor sources from property donors --- changelog.d/property-survivor-scope.fixed.md | 1 + docs/us-current-asec-dividend-source.md | 7 +++- docs/us-current-asec-property-basis.md | 21 ++++++++---- .../us_runtime/current_asec_property_basis.py | 21 ++++++++++-- .../test_us_current_asec_property_basis.py | 34 ++++++++++++++++++- 5 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 changelog.d/property-survivor-scope.fixed.md diff --git a/changelog.d/property-survivor-scope.fixed.md b/changelog.d/property-survivor-scope.fixed.md new file mode 100644 index 000000000..4b9258e1c --- /dev/null +++ b/changelog.d/property-survivor-scope.fixed.md @@ -0,0 +1 @@ +Exclude positive survivor receipts from the initial property donor bridge because its two qualified source slots do not cover the additional sources included in the published aggregate. Preserve visible route diagnostics and report the excluded design-weight mass. diff --git a/docs/us-current-asec-dividend-source.md b/docs/us-current-asec-dividend-source.md index 5439e50c7..de977cd18 100644 --- a/docs/us-current-asec-dividend-source.md +++ b/docs/us-current-asec-dividend-source.md @@ -70,7 +70,12 @@ leave it unknown. Under-15 rows remain unknown; incomplete under-15 literals are labeled unresolved rather than contradictory. Independent booleans retain the presence of codes 8 and 10 even when the receipt is unresolved. -This clearance describes the recorded source routes. It does not establish +This clearance describes only the two recorded source slots. The dictionary's +`SRVS_VAL` entry (PDF page 49, printed page 6C-28) also includes unedited third +and fourth sources. Their types are not qualified here, so two clear visible +slots cannot establish complete absence of survivor estate/trust income. +The property donor bridge conservatively excludes positive survivor receipts +until that additional scope is qualified. This clearance does not establish that the person has no property income. No survivor amount is captured, no donor exclusion is applied, and a missing slot is never an observed zero amount. A later measurement bridge must explicitly choose and validate how diff --git a/docs/us-current-asec-property-basis.md b/docs/us-current-asec-property-basis.md index b4db0571d..40a4be5ea 100644 --- a/docs/us-current-asec-property-basis.md +++ b/docs/us-current-asec-property-basis.md @@ -77,15 +77,24 @@ Other-income clearance requires known nonreceipt with a NIU category, or a readable reported category outside property codes 5–8 and unspecified code 19. A reported outside category resolves its route even if its own dollar amount is ambiguous; no other-income amount is included in this basis. Missing, -unreadable or contradictory routing stays unresolved. Survivor clearance uses -the dividend qualifier's nullable clearance and checks agreement with its -readable codes: known no with NIU slots, or known yes with at least one active -source and every slot outside estate/trust code 8 and unspecified code 10. -Possible property overlaps and unknown routes have different exclusion reasons. +unreadable or contradictory routing stays unresolved. The dividend qualifier's +nullable survivor clearance is checked against its readable codes and retained +as `survivor_visible_routes_clear`. It covers only `SUR_SC1` and `SUR_SC2`. +The 2025 dictionary describes `SRVS_VAL` as including edited sources 1/2 plus +unedited sources 3/4 (PDF page 49, printed page 6C-28); those additional source +types are not qualified here. A positive survivor receipt therefore cannot +establish complete absence of estate/trust income even when both visible +slots are outside codes 8 and 10. + +The first bridge accepts only known survivor nonreceipt with readable NIU +slots, exposed as `survivor_full_scope_clear`. Positive receipts have a separate +`survivor_additional_sources_unresolved` exclusion and weighted coverage row. +Possible visible property overlaps and unknown visible routes retain their own +exclusion reasons. Overlapping exclusion rows must not be added together. Neither other income nor survivor amounts enter the component or reported sum. `reported_total_eligible` requires age 15+, a known finite reported total, and -clear other-income and survivor routes. It is a diagnostic sample, not a second +clear other-income routes and complete survivor scope. It is a diagnostic sample, not a second aggregate model. `joint_component_fit_eligible` additionally requires all four components and an exactly zero interest discrepancy. Missing or under-15 values are never turned into analytic zeros. Negative and zero net totals remain valid: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py index e46f158d4..b04d0c95f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_property_basis.py @@ -231,7 +231,14 @@ def _survivor_route(dividend, age): ), "SURVIVOR_ROUTE_AGREEMENT", ) - return clear, possible + # SRVS_VAL includes unedited third/fourth sources absent from SUR_SC1/2 + # (2025 dictionary, PDF49 / printed6C-28). Visible-slot clearance cannot + # establish full property scope for a person reporting survivor income. + # Retain that descriptive clearance, but only known nonreceipt can clear + # the first donor bridge until the additional-source scope is qualified. + full_clear = clear & receipt_ok & (receipt == 2) + extra_sources_unresolved = yes & (age >= 15) + return clear, possible, full_clear, extra_sources_unresolved def _weights(index, membership, weights): @@ -312,10 +319,17 @@ def build_asec_property_basis( ) retirement, count, derivation, derived_slots = _retirement(interest) other_clear, other_possible = _other_route(income_routing) - survivor_clear, survivor_possible = _survivor_route(dividend, age) + ( + survivor_visible_clear, + survivor_possible, + survivor_clear, + survivor_extra_unresolved, + ) = _survivor_route(dividend, age) person = interest[["native_person_id", "source_age"]].copy(deep=True) person["original_household_id"] = original_household_membership.to_numpy(copy=True) person["original_household_design_weight"] = design + person["survivor_visible_routes_clear"] = survivor_visible_clear + person["survivor_full_scope_clear"] = survivor_clear for name, amount in zip( PROPERTY_COMPONENTS, (ordinary, retirement, dividends, property_amount), @@ -354,7 +368,8 @@ def build_asec_property_basis( "other_income_possible_property": other_possible, "other_income_route_unresolved": ~other_clear & ~other_possible, "survivor_possible_property": survivor_possible, - "survivor_route_unresolved": ~survivor_clear & ~survivor_possible, + "survivor_route_unresolved": ~survivor_visible_clear & ~survivor_possible, + "survivor_additional_sources_unresolved": survivor_extra_unresolved, }, index=interest.index, ) diff --git a/packages/microcosm-build/tests/test_us_current_asec_property_basis.py b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py index 1d9dced98..db4500db2 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_property_basis.py +++ b/packages/microcosm-build/tests/test_us_current_asec_property_basis.py @@ -260,6 +260,7 @@ def test_other_income_routes_are_strict_and_never_added( (1, 10, 0, None), (1, 1, None, None), (1, 0, 0, None), + (2, 0, 0, True), (2, 8, 0, None), (0, 0, 0, None), ], @@ -273,12 +274,43 @@ def test_survivor_routes_keep_overlap_and_unknown_exclusions( code(d, name, [value], ["missing" if value is None else "in_printed_range"]) d["survivor_property_route_clear"] = pd.array([declared], dtype="boolean") result = build_asec_property_basis(**source) - assert bool(result.person.reported_total_eligible.iloc[0]) is (declared is True) + full_clear = receipt == 2 and declared is True + assert bool(result.person.reported_total_eligible.iloc[0]) is full_clear + assert bool(result.person.joint_component_fit_eligible.iloc[0]) is full_clear + assert bool(result.person.survivor_visible_routes_clear.iloc[0]) is ( + declared is True + ) + assert bool(result.exclusions.survivor_additional_sources_unresolved.iloc[0]) is ( + receipt == 1 + ) assert bool(result.exclusions.survivor_possible_property.iloc[0]) is ( declared is False ) +def test_visible_survivor_clearance_does_not_clear_unobserved_extra_sources(): + source = inputs(3) + d = source["dividend"] + code(d, "SUR_YN", [1, 1, 2]) + code(d, "SUR_SC1", [1, 8, 0]) + d["survivor_property_route_clear"] = pd.array([True, False, True], dtype="boolean") + before = d.copy(deep=True) + result = build_asec_property_basis(**source) + assert result.person.survivor_visible_routes_clear.tolist() == [True, False, True] + assert result.person.survivor_full_scope_clear.tolist() == [False, False, True] + assert result.person.joint_component_fit_eligible.tolist() == [False, False, True] + assert result.person.property_reported_total.eq(-10).all() + reason = result.summary.loc["excluded:survivor_additional_sources_unresolved"] + assert reason.person_count == 2 and reason.household_count == 1 + assert reason.design_weighted_person_mass == 4 + assert reason.union_household_design_mass == 2 + assert result.exclusions.survivor_possible_property.tolist() == [False, True, False] + assert_frame_equal(source["dividend"], before) + assert_frame_equal( + result.provenance.filter(like="dividend."), before.add_prefix("dividend.") + ) + + @pytest.mark.parametrize( "axis", ["person", "native", "age", "duplicate_native", "duplicate_person"] ) From 48507e84f94f177a88dbccb1c0f5770cf2227750 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:45:05 -0400 Subject: [PATCH 173/255] Classify reviewed property and fiscal source boundaries --- .../tests/test_us_spine_blindness.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 3ed1f83e6..d822f2560 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -137,6 +137,10 @@ # Source-qualified graph additions reviewed 2026-09-12. These are # original-record joins or declarations, not source-routed PUF models. "current_acs_income_anchor_source.py", # ACS anchor -> exact original ids. + # Exact original ASEC joins, preserving reporting/routing unknowns; + # neither qualifier selects modeled donors or attaches tax leaves. + "current_asec_child_support_source.py", # Paid/received support observations. + "current_asec_dividend_source.py", # Dividend observations/survivor routes. "current_asec_interest_source.py", # ASEC interest -> original money owner. "current_asec_unemployment_source.py", # UC literal -> original ASEC ids. "current_survey_amounts.py", # Validate origin join; fan out to both clones. @@ -299,6 +303,8 @@ "org_wages.py", "parity_reference.py", "pregnancy.py", + # Shared numeric property component names; no source access or treatment. + "property_income_constants.py", # Pinned-archive sidecar restore (PAW_TYP); no population treatment. "public_assistance_type_source.py", "puf_aggregate_records.py", @@ -368,8 +374,15 @@ { # Qualified raw ACS INTP/RETP anchors, preserving literal knownness "current_acs_income_anchor_source.py", + # Exact received/paid child-support source observations; no tax treatment + "current_asec_child_support_source.py", + # Dividend receipt/amount and survivor-route source qualification + "current_asec_dividend_source.py", # Qualified ASEC interest components and unreconciled total diagnostics "current_asec_interest_source.py", + # Pure supplied-table property donor basis and exclusion diagnostics; + # no source owner or source-spine access exemption + "current_asec_property_basis.py", # Qualified UC receipt/amount observations, preserving unknown zeros "current_asec_unemployment_source.py", # Current UC/health amount qualification and exact two-clone attachment @@ -378,12 +391,16 @@ "current_survey_health_coverage.py", # Authenticated original ACS/ASEC health-coverage observations "current_survey_health_source.py", + # Explicit fiscal input declarations; assumptions/inactivity disabled + "fiscal_leaf_policy.py", # Typed source, recode and attachment health graph fragment "graph_current_survey_health.py", # Grouped dense calibration over retained fiscal measurements "graph_fiscal_dense_calibration.py", # Measured fiscal inputs and exact retained population/target axes "graph_fiscal_measurement.py", + # Source-blind weighted QRF chain, raw draws and signed reconciliation + "graph_property_income.py", # Source-blind PUF clone placement and original-channel preservation "graph_puf55_route_attachment.py", # Complete PUF composition and retained checked-output lifetime @@ -3446,6 +3463,12 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: # accepted, and only for the listed modules. _REVIEWED_DYNAMIC_SELECTOR_MODULES = frozenset( { + # Reviewed 2026-09-12: fixed dividend/interest/RINT-slot amount, status + # and code families passed to private helpers; no source-channel reads. + "current_asec_property_basis.py", + # Static leaf/entity closure and engine metadata/default dictionaries, + # not population provenance. Assumption execution remains disabled. + "fiscal_leaf_policy.py", # Reviewed 2026-09-12: exact geography fields/CSR masks, typed artifact # names and producer keys, retained registry entries and node histories. # These four remain scanned for provenance columns and accessor calls. From a0895eca067b3127b4a80d31f9fe2f9860409e11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 13:49:45 -0400 Subject: [PATCH 174/255] Compose qualified original survey property income branches --- ...s-current-property-income-sources.added.md | 1 + docs/us-current-property-income-sources.md | 109 +++++ .../current_property_income_sources.py | 396 ++++++++++++++++ ...test_us_current_property_income_sources.py | 439 ++++++++++++++++++ 4 files changed, 945 insertions(+) create mode 100644 changelog.d/us-current-property-income-sources.added.md create mode 100644 docs/us-current-property-income-sources.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py create mode 100644 packages/microcosm-build/tests/test_us_current_property_income_sources.py diff --git a/changelog.d/us-current-property-income-sources.added.md b/changelog.d/us-current-property-income-sources.added.md new file mode 100644 index 000000000..ca64a9350 --- /dev/null +++ b/changelog.d/us-current-property-income-sources.added.md @@ -0,0 +1 @@ +Compose source-qualified original ASEC property donor and ACS adult-anchor recipient branches, preserving DESIGN weights, full origin axes, knownness diagnostics and final source-lifetime checks without fitting or attaching tax leaves. diff --git a/docs/us-current-property-income-sources.md b/docs/us-current-property-income-sources.md new file mode 100644 index 000000000..b722a1218 --- /dev/null +++ b/docs/us-current-property-income-sources.md @@ -0,0 +1,109 @@ +# Original property-income source composition + +`qualify_current_property_income_sources` composes the source branches for the +property-income model. It borrows the existing shared-predictor preparation, +qualifies the original ACS anchors and ASEC interest, income-routing and +dividend/survivor observations, and applies the reviewed pure ASEC donor basis. +It fits no model, supplies no tax split and performs no clone attachment. + +The arguments match the existing shared predictor boundary: the actual +preparation, allocated population and clone population, with optional shared +demographic conditioning and geography configuration. The allocated/clone +objects satisfy that existing ownership contract. Their weights never supply +the donor weights. All donor membership and DESIGN weights come from the +original source Frame selected by the shared qualifier. + +The result is descriptive `QualifiedPropertyIncomeSources`. Its source values, +source Frame and full origin document are retained for inspection and later +source/clone joins. The object has no issuer registry and cannot replace the +actual retained preparation. The complete descriptive seal is a mutation +check, not a credential. Downstream owners must qualify before consumption +and after their last relevant I/O before returning or exporting a successor. + +## Selected branches + +`donor_frame` is the original ASEC DESIGN Frame restricted to jointly eligible +persons, with referenced groups pruned by the existing Frame selector. +`donor_columns` contains the qualified shared features, reported property total +feature and four original components. It uses the same exact ordered person +IDs as that branch. The four component columns and reported total import their +canonical names from `property_income_constants.py`. + +`recipient_frame` selects original ACS persons with a finite, known adult INTP +anchor. `recipient_columns` contains the same shared features plus the qualified +ADJINC-adjusted anchor under the reported-total feature name. The existing +recipient matrix codec preserves their ordered identities and float64 bits. +`recipient_matrix` is the encoded matrix. Source AGEP and canonical shared +feature age remain distinguishable; no raw age is overwritten by the feature. +Shared earnings features preserve the existing qualifier's declared NIU operator; +this composition adds no new zero completion for earnings or property anchors. + +Adding these model columns to a graph population is a later visible graph +operation. The selected Frames themselves keep all original source columns, +weights, memberships, strata and metadata. The source composition does not +quietly change a Frame's tax leaves or model columns. + +The complete original selected source axis is never reduced in `origins` or +`source_frame`. `origin_document` preserves all original entity origin records; +`donor_basis` contains every original ASEC row's component knownness, provenance, +exclusions and design-weight summaries. `recipient_diagnostics` preserves every +selected ACS row's raw literals and parsed anchor metadata, plus separate +under-15 and unknown-anchor exclusions and the eligible-recipient mask. + +An empty eligible branch has an empty indexed columns table and `None` for +its Frame, and no recipient matrix when no ACS recipient is eligible. Complete +diagnostics remain available. The existing Frame selector and matrix codec +require nonempty inputs; the composition does not manufacture a record or zero +to satisfy them. A later model host must refuse fitting an empty branch or make +an explicitly reviewed completion decision. + +## Exact identity and lifetime + +The complete shared source Frame, its origins and the original preparation's +person origin records must agree in order, source channel and native identity. +Each ASEC qualifier must exactly match the selected ASEC person and native-ID +axes; the pure basis also checks their source ages. ACS anchors must match the +selected original ACS axis. Donor feature and recipient matrix axes are checked +before selection. No sorting, inner join or missing-key fill hides a mismatch. + +The donor household design weights are mapped by exact original membership. +Their person mapping must match the original Frame's resolved DESIGN weights; +importance or clone weights are never substituted. This is source construction, +not engine microsimulation aggregation. + +Each initial qualifier's complete physical description is sealed as soon as it +returns, and checked while subsequent qualifiers do I/O. After pure composition, +all five original source qualifications are repeated and compared to those +seals. Every returned value, including nullable data beneath missing masks, +is checked throughout. The actual preparation is then rechecked after the last +qualifier's I/O. The final checks are pure: retained issuer identity, nested +original source Frames, allocated/clone support, the exact optional geography +configuration payload and all complete derived seals. + +The income-routing qualifier has no standalone physical-seal API, so this +composition uses the existing full table-stamp helper over its complete person +and original-literal tables plus evidence. Existing shared/ACS/interest/dividend +seal APIs are reused directly. No source-reader implementation or new authority +framework is introduced. + +## Verification scope + +The invented fixture creates closed source bytes before issuance, then executes +the actual survey source graph and source composition. It includes a negative +ACS anchor, malformed adult anchor, under-15 blank anchor, a resolved ASEC +ordinary/retirement-interest route, and excluded ASEC overlap/unknown cases. +Controls check ordered axes, weight kind, adjusted bits, all returned seal +surfaces and copied preparation refusal. A separate injected mutation isolates +the final original-source I/O fence using previously checked descriptive values; +it does not claim another complete source replay. + +These checks run without native microdata, QRF, PUF or a country engine. Model +quality, donor conditional support, country financial integration, both-clone +fanout, replay and release acceptance remain later boundaries. + +The initial composition acceptance used donor basis `2c959096eec0441e2acd975e63522aac7f1b7016`. +A subsequent primary-source review found that survivor slots 1/2 do not establish +complete absence of estate/trust overlap: the aggregate can include sources 3/4. +The country host must adopt the separate conservative donor-basis correction +before using this branch for fitting. The initial test's eligible donor reports +no survivor income, so its observed values do not rely on a yes-survivor clearance. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py new file mode 100644 index 000000000..5eccce15c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_income_sources.py @@ -0,0 +1,396 @@ +"""Qualify original survey property branches without fitting or issuing authority.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.frame import WeightKind +from microcosm.graph import Population + +from . import current_acs_income_anchor_source as acs +from . import current_asec_dividend_source as dividend +from . import current_asec_income_routing_source as routing +from . import current_asec_interest_source as interest +from . import current_survey_predictors as shared +from . import graph_full_puf_enrichment as physical +from .current_asec_property_basis import AsecPropertyBasis, build_asec_property_basis +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + +PROTOCOL = "microcosm.us.current-property-income-sources.v1" + + +def require(condition, reason): + if not condition: + raise ValueError("PROPERTY_INCOME_SOURCES_" + reason) + + +@dataclass(frozen=True) +class QualifiedPropertyIncomeSources: + """Descriptive values only; the caller must retain and requalify actual owners.""" + + shared_predictors: shared.QualifiedSurveyPredictors + acs_anchor_values: acs.QualifiedAcsIncomeAnchors + asec_interest_values: interest.CurrentAsecInterestValues + asec_routing_values: routing.CurrentAsecIncomeRoutingValues + asec_dividend_values: dividend.CurrentAsecDividendValues + donor_basis: AsecPropertyBasis + source_frame: object + donor_frame: object | None + donor_columns: pd.DataFrame + recipient_frame: object | None + recipient_columns: pd.DataFrame + recipient_matrix: bytes | None + origins: pd.DataFrame + origin_document: bytes + recipient_diagnostics: pd.DataFrame + projection: bytes + evidence: dict + + +def _routing_seal(value): + require(type(value) is routing.CurrentAsecIncomeRoutingValues, "ROUTING_TYPE") + # The legacy routing qualifier exposes no physical-seal helper. Preserve + # its complete detached descriptions, including nullable backing storage. + return ( + physical._table_stamp(value.person), + physical._table_stamp(value.asec_literals), + shared.codec.encode_json(value.evidence), + ) + + +def _source_seals(values): + predictor, anchors, interest_values, routing_values, dividend_values = values + return ( + shared._qualified_seal(predictor), + acs.income_anchor_seal(anchors), + interest.interest_values_seal(interest_values), + _routing_seal(routing_values), + dividend.dividend_values_seal(dividend_values), + ) + + +def _frame_seal(frame): + return ( + None + if frame is None + else physical._population_stamp( + Population.from_frame(frame, "property_income_sources.description") + ) + ) + + +def property_income_sources_seal(value): + """Complete descriptive physical seal, not source authentication or a token.""" + require(type(value) is QualifiedPropertyIncomeSources, "VALUES_TYPE") + return ( + _source_seals( + ( + value.shared_predictors, + value.acs_anchor_values, + value.asec_interest_values, + value.asec_routing_values, + value.asec_dividend_values, + ) + ), + tuple( + _frame_seal(frame) + for frame in (value.source_frame, value.donor_frame, value.recipient_frame) + ), + tuple( + physical._table_stamp(table) + for table in ( + value.donor_basis.person, + value.donor_basis.provenance, + value.donor_basis.exclusions, + value.donor_basis.summary, + value.donor_columns, + value.recipient_columns, + value.origins, + value.recipient_diagnostics, + ) + ), + value.recipient_matrix, + value.origin_document, + value.projection, + shared.codec.encode_json(value.evidence), + ) + + +def _same_axis(frame, origins, reason): + require( + frame.index.equals(origins.index) + and frame.index.name == origins.index.name + and frame.native_person_id.equals(origins.native_person_id), + reason, + ) + + +def _compose(values, origin_document): + """Pure assembly of already-qualified values; no source authority from args.""" + predictor, anchors, interest_values, routing_values, dividend_values = values + frame = predictor.source_frame + people = frame.person + origins = predictor.origins.copy(deep=True) + ids = pd.Index(people.person_id.to_numpy(copy=True), name="person_id") + require( + ids.equals(origins.index) + and np.array_equal(origins.person_id.to_numpy(), ids.to_numpy()) + and set(origins.source) == {"asec", "acs"}, + "ORIGIN_AXIS", + ) + raw_origins = json.loads(origin_document)["persons"] + full_origins = pd.DataFrame(raw_origins["rows"], columns=raw_origins["columns"]) + require( + np.array_equal(full_origins.person_id.to_numpy(), ids.to_numpy()) + and np.array_equal(full_origins.source.to_numpy(), origins.source.to_numpy()) + and np.array_equal( + full_origins.selected_receiving_person_id.to_numpy(), + origins.native_person_id.to_numpy(), + ), + "FULL_ORIGIN_AXIS", + ) + donor_origins = origins.loc[origins.source.eq("asec")] + recipient_origins = origins.loc[origins.source.eq("acs")] + for selected in ( + interest_values.person, + routing_values.person, + dividend_values.person, + ): + _same_axis(selected, donor_origins, "ASEC_SOURCE_AXIS") + _same_axis(anchors.anchors, recipient_origins, "ACS_SOURCE_AXIS") + require( + predictor.donor_columns.index.equals(donor_origins.index) + and np.array_equal( + predictor.donor_frame.person.person_id.to_numpy(), + donor_origins.index.to_numpy(), + ), + "DONOR_FEATURE_AXIS", + ) + donor_source = predictor.donor_frame + require( + donor_source.weights_for("household").kind is WeightKind.DESIGN + and donor_source.resolve_weights("person").kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_WEIGHTS", + ) + membership = pd.Series( + donor_source.person.person_household_id.to_numpy(copy=True), + index=donor_origins.index, + name="household_id", + ) + household_ids = pd.Index( + donor_source.table("household").household_id.to_numpy(copy=True), + name="household_id", + ) + design_weights = pd.Series( + donor_source.weights_for("household").values.copy(), index=household_ids + ) + basis = build_asec_property_basis( + interest=interest_values.person, + income_routing=routing_values.person, + dividend=dividend_values.person, + original_household_membership=membership, + original_household_design_weights=design_weights, + ) + require( + np.array_equal( + basis.person.original_household_design_weight.to_numpy(), + donor_source.resolve_weights("person").values, + ), + "PERSON_DESIGN_WEIGHT_MAPPING", + ) + features = shared.feature_columns(predictor.demographic_conditioning) + donor_columns = predictor.donor_columns.loc[:, list(features)].copy(deep=True) + for name in (PROPERTY_REPORTED_TOTAL, *PROPERTY_COMPONENTS): + donor_columns[name] = basis.person[name].to_numpy(copy=True) + donor_mask = basis.person.joint_component_fit_eligible.to_numpy(dtype=bool) + donor_columns = donor_columns.loc[donor_mask].copy(deep=True) + donor_frame = donor_source.select(donor_mask) if donor_mask.any() else None + decoded = model_input.decode_recipient_matrix(predictor.matrix) + require( + decoded.entity == "person" + and tuple(decoded.features.columns) == features + and decoded.features.index.equals(recipient_origins.index), + "RECIPIENT_FEATURE_AXIS", + ) + original_anchors = anchors.anchors + diagnostics = original_anchors.copy(deep=True) + adult = original_anchors.property_income_in_income_universe.to_numpy(dtype=bool) + known = original_anchors.property_income_known.to_numpy(dtype=bool) + amounts = original_anchors.property_income_amount.to_numpy( + dtype=np.float64, na_value=np.nan + ) + require(np.array_equal(known, np.isfinite(amounts)), "ACS_ANCHOR_KNOWNNESS") + diagnostics["excluded_under15"] = ~adult + diagnostics["excluded_unknown_anchor"] = ~known + diagnostics["eligible_recipient"] = adult & known + recipient_mask = adult & known + recipient_columns = decoded.features.copy(deep=True) + recipient_columns[PROPERTY_REPORTED_TOTAL] = amounts + recipient_columns = recipient_columns.loc[recipient_mask].copy(deep=True) + frame_mask = np.zeros(len(frame.person), dtype=bool) + frame_mask[np.flatnonzero(origins.source.eq("acs"))] = recipient_mask + recipient_frame = frame.select(frame_mask) if frame_mask.any() else None + matrix = ( + model_input.encode_recipient_matrix( + recipient_columns, + entity="person", + entity_ids=recipient_columns.index.to_numpy(dtype="= 15 else "0", + "RINT_YN": "2" if age >= 15 else "0", + "DIV_YN": "2" if age >= 15 else "0", + "SUR_YN": "2" if age >= 15 else "0", + } + literals[105].update( + INT_VAL="140", + TRDINT_VAL="100", + INT_YN="1", + RINT_YN="1", + RINT_SC1="4", + RINT_VAL1="40", + DIV_VAL="140", + DIV_YN="1", + ) + literals[107].update( + INT_VAL="500", + TRDINT_VAL="500", + INT_YN="1", + DIV_VAL="500", + DIV_YN="1", + SUR_YN="1", + SUR_SC2="8", + ) + changes = {} + for name in ("INT_VAL", "DIV_VAL"): + values = people[name].to_numpy(copy=True) + for pid in (105, 106, 107, 108): + values[ids == pid] = int(literals[pid][name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in set( + (*owner.interest.READ_COLUMNS, *owner.dividend.READ_COLUMNS) + ) - set(owner.routing.COORDINATE_COLUMNS): + raw[name] = [ + literals[int(updated.loc[key, "person_id"])][name] + for key in raw.PERIDNUM + ] + # Historical parent values remain their own current-money observations. + for name in ("INT_VAL", "DIV_VAL"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "property-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return {**arguments, "store_root": tmp_path / "property-survey-store"} + + +@pytest.fixture(scope="module") +def actual(tmp_path_factory): + patch = pytest.MonkeyPatch() + root = tmp_path_factory.mktemp("property-source-composition") + try: + arguments = source_arguments(root, patch) + live = graph.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + frame_seal = owner._frame_seal(live.preparation._checked()[2].frame) + result = owner.qualify_current_property_income_sources( + live.preparation, live.allocated_population, live.clone_population + ) + assert owner._frame_seal(live.preparation._checked()[2].frame) == frame_seal + (root / "composition-metadata.json").write_text( + json.dumps( + { + "scope": "invented source-composition fixture; no model or native data", + "survey_stage_order": list(live.compiled.order), + "source_people": result.source_frame.n("person"), + "source_households": result.source_frame.n("household"), + "allocated_people": live.allocated_population.frame.n("person"), + "clone_people": live.clone_population.frame.n("person"), + "eligible_donors": len(result.donor_columns), + "eligible_recipients": len(result.recipient_columns), + }, + indent=2, + ) + + "\n" + ) + yield live, result, arguments + finally: + patch.undo() + + +def test_actual_source_composition_preserves_design_and_signed_anchor(actual): + live, result, _ = actual + assert len(result.origins) == 9 + assert len(result.donor_columns) == 1 + assert len(result.recipient_columns) == 3 + assert result.donor_frame.weights_for("household").kind is WeightKind.DESIGN + assert ( + live.allocated_population.frame.weights_for("household").kind + is WeightKind.IMPORTANCE + ) + row = result.donor_columns.iloc[0] + assert row.property_ordinary_interest == 100 + assert row.property_retirement_interest == 40 + assert row.property_dividends == 140 + assert row.property_broad_receipts == -400 + assert row.property_reported_total == -120 + selected = result.recipient_columns.property_reported_total + assert (selected < 0).any() + amount = np.float64(-800) * (np.float64(1000133) / 1000000) + anchor = result.recipient_diagnostics + exact_id = anchor.index[ + anchor.SERIALNO.eq("2024HU0000001") & anchor.SPORDER.eq("1") + ][0] + assert selected.loc[exact_id].view("uint64") == amount.view("uint64") + pd.testing.assert_frame_equal( + model_input.decode_recipient_matrix(result.recipient_matrix).features, + result.recipient_columns, + ) + assert result.recipient_diagnostics.excluded_under15.sum() == 1 + assert result.recipient_diagnostics.excluded_unknown_anchor.sum() == 2 + assert result.evidence["source_admission_issued"] is False + assert not result.evidence["model_fitted"] + assert not result.evidence["clone_attachment_performed"] + assert json.loads(result.origin_document)["persons"]["rows"] + + +def _values(result): + return ( + result.shared_predictors, + result.acs_anchor_values, + result.asec_interest_values, + result.asec_routing_values, + result.asec_dividend_values, + ) + + +@pytest.mark.parametrize( + "target", + [ + "asec_person", + "asec_native", + "acs_person", + "acs_native", + "donor_features", + "recipient_matrix", + "origin_document", + ], +) +def test_source_composition_refuses_axis_mismatch(actual, target): + _, result, _ = actual + values = list(_detached(_values(result))) + origin = result.origin_document + if target == "asec_person": + values[2].person.index = values[2].person.index[::-1] + elif target == "asec_native": + values[3].person["native_person_id"] = values[ + 3 + ].person.native_person_id.to_numpy()[::-1] + elif target == "acs_person": + values[1].anchors.index = values[1].anchors.index[::-1] + elif target == "acs_native": + values[1].anchors["native_person_id"] = values[ + 1 + ].anchors.native_person_id.to_numpy()[::-1] + elif target == "donor_features": + values[0].donor_columns.index = values[0].donor_columns.index[::-1] + elif target == "recipient_matrix": + matrix = model_input.decode_recipient_matrix(values[0].matrix) + features = matrix.features.iloc[::-1] + values[0] = replace( + values[0], + matrix=model_input.encode_recipient_matrix( + features, entity="person", entity_ids=features.index.to_numpy() + ), + ) + else: + document = json.loads(origin) + document["persons"]["rows"].reverse() + origin = owner.shared.codec.encode_json(document) + with pytest.raises(ValueError, match="PROPERTY_INCOME_SOURCES"): + owner._compose(tuple(values), origin) + + +@pytest.mark.parametrize( + "target", + [ + "basis", + "nullable_storage", + "donor_columns", + "recipient_columns", + "origin_document", + "source_frame", + "projection", + ], +) +def test_complete_description_seal_detects_every_returned_surface(actual, target): + _, result, _ = actual + before = owner.property_income_sources_seal(result) + changed = _detached(result) + if target == "basis": + changed.donor_basis.person.iloc[ + 0, changed.donor_basis.person.columns.get_loc("property_reported_total") + ] += 1 + elif target == "nullable_storage": + array = changed.acs_anchor_values.anchors.property_income_amount.array + position = np.flatnonzero(array._mask)[0] + array._data[position] = 321.0 + elif target == "donor_columns": + changed.donor_columns.iloc[0, 0] += 1 + elif target == "recipient_columns": + changed.recipient_columns.iloc[0, 0] += 1 + elif target == "origin_document": + changed = replace(changed, origin_document=changed.origin_document + b" ") + elif target == "source_frame": + changed.source_frame.person.iloc[ + 0, changed.source_frame.person.columns.get_loc("age") + ] += 1 + else: + changed = replace(changed, projection=changed.projection + b" ") + assert owner.property_income_sources_seal(changed) != before + assert owner.property_income_sources_seal(result) == before + + +def test_no_eligible_recipient_is_explicit_without_empty_matrix_or_zero_fill(actual): + _, result, _ = actual + values = list(_detached(_values(result))) + anchors = values[1].anchors + anchors["property_income_known"] = False + anchors["property_income_amount"] = pd.array([None] * len(anchors), dtype="Float64") + anchors["property_income_status"] = "missing_source_amount" + rebuilt = owner._compose(tuple(values), result.origin_document) + assert rebuilt.recipient_frame is None and rebuilt.recipient_matrix is None + assert len(rebuilt.recipient_columns) == 0 + assert len(rebuilt.recipient_diagnostics) == 5 + + +@pytest.mark.parametrize("target", ["columns", "geography_config"]) +def test_final_io_cannot_change_already_composed_values(actual, monkeypatch, target): + live, result, _ = actual + # The first test already ran every real source owner. Here the same detached + # values isolate the final real preparation I/O fence; no new issuer or + # source admission is mocked. Any unchecked descriptive replay still has to + # survive the real original-owner check and final full-value seal. + values = list(_detached(_values(result))) + config = None + if target == "geography_config": + config = owner.shared.host.survey_budget.geography.AtomicSurveyReconstruction( + "/invented/support.json", + "a" * 64, + ( + ("district", "invented-district"), + ("population", "invented-population"), + ("puma", "invented-puma"), + ), + 1, + ) + # This is only the option-lifetime fault injection. No support file is + # opened or admitted, and no geography reconstruction is claimed. + values[0] = replace(values[0], geography_config_payload=config.to_bytes()) + for module, name, value in ( + (owner.shared, "qualify_current_survey_predictors", values[0]), + (owner.acs, "qualify_current_acs_income_anchors", values[1]), + (owner.interest, "qualify_current_asec_interest", values[2]), + (owner.routing, "qualify_current_asec_income_routing", values[3]), + (owner.dividend, "qualify_current_asec_dividend", values[4]), + ): + monkeypatch.setattr(module, name, lambda *a, _value=value, **kw: _value) + original = owner._compose + completed = [] + + def compose(*args): + value = original(*args) + completed.append(value) + return value + + monkeypatch.setattr(owner, "_compose", compose) + open_path = Path.open + mutated = [] + + def open_during_final_io(path, *args, **kwargs): + stream = open_path(path, *args, **kwargs) + if completed and not mutated: + if target == "columns": + completed[0].donor_columns.iloc[0, 0] += 1 + else: + object.__setattr__(config, "seed", 2) + mutated.append(str(path)) + return stream + + # Keep the issuer method itself untouched: replacing it is rightly refused + # by its own live-code seal before the intended last-I/O fence is reached. + monkeypatch.setattr(Path, "open", open_during_final_io) + reason = ( + "FINAL_DERIVED_VALUES_CHANGED" + if target == "columns" + else "FINAL_GEOGRAPHY_CONFIG_CHANGED" + ) + with pytest.raises(ValueError, match=reason): + owner.qualify_current_property_income_sources( + live.preparation, + live.allocated_population, + live.clone_population, + geography_config=config, + ) + + assert len(mutated) == 1 + + +def test_unissued_plain_object_refuses_without_source_io(): + with pytest.raises(ValueError, match="PREPARATION_TYPE"): + owner.qualify_current_property_income_sources(object(), None, None) + + +def test_constructor_weight_copy_preserves_source_live_identity(): + before = owner.shared.source._live() + copied = _detached(Weights(np.array([1.0]), WeightKind.DESIGN)) + assert copied.values.tolist() == [1.0] + assert owner.shared.source._live() == before + + +def test_unissued_preparation_copy_refuses_before_composition(actual): + live, _, _ = actual + before = owner.shared.source._live() + # Copy only the descriptive payload into an unissued identity. Avoid the + # unrelated stdlib copy class-cache side effect in this ownership control. + copied = object.__new__(type(live.preparation)) + object.__setattr__(copied, "payload", live.preparation.payload) + with pytest.raises(ValueError): + owner.qualify_current_property_income_sources( + copied, + live.allocated_population, + live.clone_population, + ) + assert owner.shared.source._live() == before From 200a4fd1f854e81cb16e2676c9198ddc22151648 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:00:51 -0400 Subject: [PATCH 175/255] Record integrated property source and graph verification --- ...perty-source-model-integration-20260912.md | 82 +++++++++++++++++++ .../tests/test_us_spine_blindness.py | 4 + 2 files changed, 86 insertions(+) create mode 100644 experiments/us-property-source-model-integration-20260912.md diff --git a/experiments/us-property-source-model-integration-20260912.md b/experiments/us-property-source-model-integration-20260912.md new file mode 100644 index 000000000..e192d72cb --- /dev/null +++ b/experiments/us-property-source-model-integration-20260912.md @@ -0,0 +1,82 @@ +# Property source and model integration + +12 September 2026. These are bounded code and invented-input checks. They do +not certify a native population or a releasable dataset. + +The integrated property model learns four jointly drawn components from +original ASEC household DESIGN weights: ordinary interest, retirement-account +earnings, dividends and signed broad property receipts. A visible signed +reconciliation operation aligns recipient components with qualified ACS INTP. +Negative and zero net totals retain offsetting positive components and losses. +Broad property receipts are not yet a pure rental-income tax input, and +retirement-account earnings are not withdrawals. + +The source composition retains original source Frames, native identities, +knownness, reporting routes, allocation flags and full exclusion diagnostics. +It selects original ASEC donor persons and original ACS persons with known +adult anchors. Model columns are separate from the selected Frames, ready for +explicit graph operations. It does not complete unknown anchors or attach +modeled values to clones by itself. + +## Accepted checks + +| Component | Exact implementation | Evidence | +| --- | --- | --- | +| Four-component graph | `e2e38a3c75175abfeea3b8f25f0412afb6c100fb` | 22 tests; actual weighted QRF, raw draws, signed reconciliation and required cache replay. A delegated training-implementation change invalidates the cache identity. Independent repair review approved. | +| DIV and visible survivor routes | `90bc1bfbc2592530c8fe914f5ed60b3bc7b2e1e3` | 82 source controls; exact printed domains, unknown zeros, source joins and final ownership checks. Independent review and root hash verification approved. | +| Conservative property donor basis | `bc278d9808e41820b175da4c1b036c283929d6d7` | 60 tests; signed observations, exact axes, source discrepancies, explicit unused-slot derivation and weighted exclusions. Independent review approved. | +| Original source composition | `9bc5932af214881bcc7931cda2847fab60163dfa` | 21 controls on the owner's freeze and 21 again after integrating the corrected basis and hardened source domains. The latter ran in 47.82 seconds with 516,046,848 bytes peak RSS. Root source, test and receipt review approved. | + +The integrated source fixture executed the actual four-stage survey graph: +CREATE, allocation, complete initial support cloning and ownership attachment. +It had 9 original persons in 6 households, 18 cloned persons, 1 eligible ASEC +donor and 3 eligible ACS recipients. It ran no QRF, PUF or country engine. +The separate graph tests above establish numerical graph execution, not a +completed country host combining those two pieces. + +The integrated source receipt is +`8fef8af66c88f701d2f62422f8513b92e28a2626af4a9fc8481eb96fc32cdf96`; +its XML is +`8164608b6d96ea492a133f11f560ee1c688f0524198c4614d1e4ac6ce4aa4ae9`. +All 1,017 frozen source files and 15 explicitly admitted resources remained +unchanged; no unexpected guarded access occurred. + +The complete module inventory and existing source/metadata scanners passed +on the nine affected modules. Only actual source joins receive original-source +access classification. The pure basis, fiscal policy and numerical graph do +not. Literal and accessor checks still reject injected source-channel reads +in the two modules with reviewed dynamic selectors. The scoped receipt is +`3543fa52f0440db58a6592c6dc313c54a83b6348bd2bfd42cda3bcc66f2eb828`. +An earlier attempt to run the entire legacy source suite reached its CPU cap; +it is preserved as incomplete, not counted as a passing full-suite run. + +## A source finding that changed donor selection + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=49) +states that SRVS_VAL includes unedited third and fourth sources as well as +the two edited visible slots. Therefore, two visible survivor sources outside +estate/trust code 8 do not establish complete absence of that overlap. + +The first property donor bridge now requires known survivor nonreceipt with +readable NIU slots. Positive receipts have an explicit additional-source-scope +exclusion and design-weight mass summary. The qualifier's two-slot clearance +remains diagnostic. No source observations are rewritten, no survivor dollars +are assigned to a property component, and conservation of the ACS anchor is +not presented as protection against selection bias. + +## Remaining integration + +The next stage adds these original-person components and their knownness to +the actual pre-PUF financial graph, with one ACS draw shared across both +initial clone rows. It will reuse the existing checked financial run owner. +The existing tax leaves and legacy capital-gain conditioning cannot be called +rebased until the replacement explicitly executes and passes verification. +Pension/disability/survivor source detail, withdrawal regularity, taxability, +missing-anchor completion and finer property tax categories remain separate +measurement or model work. + +The running native pilot uses an earlier frozen financial implementation. +It is not evidence for the newly integrated property code. Native PUF +acceptance, applicable fiscal inputs, national/state/CD calibration, held-out +quality checks, progressive scaling and exact full/compact exports remain +release requirements. No publication, default promotion or deployment occurred. diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index d822f2560..cf5ca3849 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -142,6 +142,8 @@ "current_asec_child_support_source.py", # Paid/received support observations. "current_asec_dividend_source.py", # Dividend observations/survivor routes. "current_asec_interest_source.py", # ASEC interest -> original money owner. + # Composes qualified original source axes and DESIGN branches; no fit. + "current_property_income_sources.py", "current_asec_unemployment_source.py", # UC literal -> original ASEC ids. "current_survey_amounts.py", # Validate origin join; fan out to both clones. "current_survey_health_source.py", # Qualify exact ACS/ASEC source rosters. @@ -385,6 +387,8 @@ "current_asec_property_basis.py", # Qualified UC receipt/amount observations, preserving unknown zeros "current_asec_unemployment_source.py", + # Qualified original property donor/recipient branch composition + "current_property_income_sources.py", # Current UC/health amount qualification and exact two-clone attachment "current_survey_amounts.py", # Pure current-coverage recodes and original-person clone attachment From 8c10f8a43463a277e24c8d108fa7e89afb0fae02 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:15:16 -0400 Subject: [PATCH 176/255] Verify property model receipts against original population inputs --- changelog.d/property-model-receipts.added.md | 1 + docs/us-property-model-receipts.md | 65 ++++ .../graph_property_income_receipts.py | 251 +++++++++++++ .../test_us_graph_property_income_receipts.py | 338 ++++++++++++++++++ 4 files changed, 655 insertions(+) create mode 100644 changelog.d/property-model-receipts.added.md create mode 100644 docs/us-property-model-receipts.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py create mode 100644 packages/microcosm-build/tests/test_us_graph_property_income_receipts.py diff --git a/changelog.d/property-model-receipts.added.md b/changelog.d/property-model-receipts.added.md new file mode 100644 index 000000000..de28e20d2 --- /dev/null +++ b/changelog.d/property-model-receipts.added.md @@ -0,0 +1 @@ +Add source-blind verification of the four property fit/apply receipt pairs against authenticated original donor/recipient branches, with complete model history checks and deterministic application replay without refitting. diff --git a/docs/us-property-model-receipts.md b/docs/us-property-model-receipts.md new file mode 100644 index 000000000..1c16e9618 --- /dev/null +++ b/docs/us-property-model-receipts.md @@ -0,0 +1,65 @@ +# Property model receipt verification + +`verify_property_model_receipts(nodes, donor_population, recipient_population, +artifacts)` in `graph_property_income_receipts.py` reconstructs the existing +four fit and four apply receipt dictionaries. The caller passes the ordered +property fragment, the two original `Population` branches, and authenticated +artifact bytes keyed by `(node_id, output_name)`. Additional non-model fragment +nodes and their artifacts may be present; the country host verifies those. + +The caller must first bind the store bytes, typed producer outputs and exact +population dependencies. This helper loads the existing trusted model format, +which contains pickle. A self-consistent digest or a returned receipt does not +authenticate arbitrary model bytes or establish source authority. The owning +host retains responsibility for source qualification, complete population +identity, physical seals, borrowed inputs and final I/O checks. + +## Checks + +The helper reconstructs the shared graph declarations for the four ordered +property targets and compares their complete normative bytes. This binds input +slices, population versions, predictors, target order, seed, phase, fit options, +model/training sibling edges, prior raw-draw edges and output artifact types. +Training stays donor-only. The existing model protocol initializes its exact +chain-start state against the original design-weight donor without fitting. +Each trusted model must match the complete ordered training history, actual +consumed float64 donor bytes, donor pandas index, resolved weights, configuration +and preceding/succeeding training states. The first three property components +remain nonnegative and the four-component sum must equal the observed donor +aggregate. Both branches require finite float64 model columns and unique int64 +person IDs. + +Ordinary population apply checkpoints have a pandas-index identity but no +independent recipient-feature digest. The verifier therefore performs four +additional `apply_target` operations using the retained fitted models, current +recipient features, raw prefix and exact initial draw RNG. It compares the +resulting raw bytes and complete application states with the published +artifacts. It validates every model/raw hash in the ordered application history. +It never calls `fit_target` or a model-fitting convenience method. + +This replay establishes output consistency with the supplied feature values; +a model can produce the same output for different inputs. Exact feature and +person-ID provenance remains the caller's authenticated population binding. +The existing protocol uses pandas row labels for model state and raw draws; +the verifier does not substitute entity IDs or invent a new identity digest. +Graph owners must continue to bind the distinct int64 person axis themselves. + +## Scope and cost + +The return value uses the existing raw kernel receipt schema; the host adds and +checks the executor capability/input-writer envelope through its existing +`_states` path. It adds no status, +source issuer, receipt artifact type, model format or host framework. The +helper does not fit, reconcile, attach clones, assign tax treatment or modify +its input frames. Cost per call is one chain initialization, four trusted model +decodes and four deterministic apply replays over the recipient branch. The +owning host decides when to perform this validation; no native-scale runtime +claim follows from the small tests. + +Tests reuse the actual twelve-node property graph fixture with two trees, +44 invented donor persons, four recipient persons, non-default pandas row +labels, int64 IDs above `2**53` and zero-weight records. Original cold and +required-cache kernel receipts match the verifier. Separate controls alter +donor/recipient axes and values, design weights, graph declarations, model and +training bytes, model/raw history and draw RNG. Country engines, native sources, +the full PUF host and publication are outside these checks. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py new file mode 100644 index 000000000..1911a590e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_income_receipts.py @@ -0,0 +1,251 @@ +"""Verify property model receipts against authenticated population/artifact inputs. + +This is a source-blind numerical check, not a producer or source issuer. Callers +must authenticate store payloads and the typed producer/population dependency +closure before supplying model bytes: decoding a pickle is not authentication. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import qrf, qrf_target +from microcosm.fit.graph_legacy_qrf import ( + LegacyQRFApplyKernel, + legacy_qrf_apply_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import WeightKind +from microcosm.graph import Node +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.population import Population + +from . import graph_property_income as property_graph +from .property_income_constants import PROPERTY_COMPONENTS, PROPERTY_REPORTED_TOTAL + + +def _require(condition, reason): + if not condition: + raise ValueError("PROPERTY_MODEL_RECEIPTS_" + reason) + + +def _declarations(nodes, donor, recipient): + _require( + isinstance(nodes, tuple) + and all(type(node) is Node for node in nodes) + and len({node.id for node in nodes}) == len(nodes), + "NODE_ROSTER", + ) + fits = tuple( + n for n in nodes if n.kernel == property_graph.PropertyIncomeTrainKernel.ref + ) + applies = tuple(n for n in nodes if n.kernel == LegacyQRFApplyKernel.ref) + _require(len(fits) == len(applies) == len(PROPERTY_COMPONENTS), "MODEL_ROSTER") + # Declaration order must describe the ordered chain; other fragment nodes + # (source projection, attachment, reconciliation) are the host's concern. + first = fits[0] + _require(first.id.endswith(".fit.000"), "FIT_PREFIX") + prefix = first.id.removesuffix(".fit.000") + params = first.params + features = property_graph._features(params.get("predictors")) + seed, trees = params.get("seed"), params.get("n_estimators") + _require( + bool(prefix) + and type(seed) is int + and seed >= 0 + and type(trees) is int + and trees > 0 + and donor.version != recipient.version, + "PARAMETERS", + ) + phase = property_graph.PROTOCOL + ":" + prefix + expected_fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor.version, + entity="person", + predictors=features, + targets=PROPERTY_COMPONENTS, + seed=seed, + n_estimators=trees, + zero_atol=0, + phase=phase, + ) + expected_applies = legacy_qrf_apply_nodes( + prefix + ".apply", + population=recipient.version, + fit_nodes=expected_fits, + seed=seed, + phase=phase, + ) + expected_fits = tuple( + replace(node, kernel=property_graph.PropertyIncomeTrainKernel.ref) + for node in expected_fits + ) + _require( + all( + canonical_json(normative(actual)) == canonical_json(normative(expected)) + for actual, expected in zip( + (*fits, *applies), (*expected_fits, *expected_applies), strict=True + ) + ), + "DECLARATION", + ) + return fits, applies, features, seed, trees, phase + + +def _table(population, columns): + table = population.frame.person + _require( + table.person_id.dtype == np.dtype("int64") + and table.person_id.is_unique + and all(c in table and table[c].dtype == np.dtype("float64") for c in columns) + and np.isfinite(table.loc[:, list(columns)].to_numpy()).all(), + "PERSON_IDENTITY_OR_VALUES", + ) + return table + + +def verify_property_model_receipts( + nodes, donor_population, recipient_population, artifacts +): + """Return the published eight fit/apply receipts after numerical validation. + + ``nodes`` is the ordered property fragment (additional non-model nodes may + be present); populations are the exact original donor/recipient branches. + ``artifacts[(node_id, output_name)]`` holds caller-authenticated bytes. + All four target fits are checked against the actual design donor and + chain-start state without fitting again. Applications are replayed using + those models and the current recipient features, because the legacy apply + checkpoint carries an index but no independent feature digest. + + The caller owns population identity, source qualification and lifetime + checks. This function preserves the supplied frames and does not confer + authority on detached populations, arbitrary model bytes or its result. + """ + _require( + type(donor_population) is Population + and type(recipient_population) is Population + and isinstance(artifacts, Mapping), + "INPUT_TYPES", + ) + fits, applies, features, seed, trees, phase = _declarations( + nodes, donor_population, recipient_population + ) + required = { + (node.id, output.name) + for node in (*fits, *applies) + for output in node.artifact_outputs + } + _require( + all(key in artifacts and type(artifacts[key]) is bytes for key in required), + "ARTIFACT_ROSTER", + ) + donor = _table(donor_population, (*features, *PROPERTY_COMPONENTS)) + recipient = _table(recipient_population, features).loc[:, list(features)].copy() + weights = donor_population.frame.resolve_weights("person") + _require(weights.kind is WeightKind.DESIGN, "ORIGINAL_DESIGN_WEIGHTS") + _require( + (donor.loc[:, list(PROPERTY_COMPONENTS[:3])].to_numpy() >= 0).all() + and np.array_equal( + donor.loc[:, list(PROPERTY_COMPONENTS)].sum(axis=1).to_numpy(), + donor[PROPERTY_REPORTED_TOTAL].to_numpy(), + ), + "RESOLVED_DONOR_COMPONENTS", + ) + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": weights}), fits[0].inputs[0], donor + ) + model = qrf.RegimeGatedQRF( + seed=seed, n_estimators=trees, zero_atol=0, max_samples_leaf=None + ) + application_state = model.start_chain( + model_frame, list(features), list(PROPERTY_COMPONENTS), weights="design" + ) + before = qrf_target.LegacyQRFTrainingState.from_chain(application_state) + receipts, history, raw_history = {}, [], [] + raw = pd.DataFrame(index=recipient.index) + for i, (fit, apply, target) in enumerate( + zip(fits, applies, PROPERTY_COMPONENTS, strict=True) + ): + payload = artifacts[fit.id, "model"] + packet, after = codec.read_training(artifacts[fit.id, "training_state"]) + _require( + len(packet["models"]) == i + 1 + and packet["models"][:i] == history + and packet["models"][-1]["sha256"] == codec.sha(payload), + "TRAINING_HISTORY", + ) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=packet["models"][-1]["sha256"] + ) + _require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, (*features, *PROPERTY_COMPONENTS[:i], target) + ), + "TRAINING_DONOR", + ) + history.append( + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + } + ) + _require(packet["models"] == history, "TRAINING_HISTORY") + before = after + application, chain = codec.read_application(artifacts[apply.id, "apply_state"]) + raw_payload = artifacts[apply.id, "raw_draw"] + actual_raw = codec.read_raw_target( + raw_payload, target=target, index=recipient.index + ) + raw_history.append({"target": target, "sha256": codec.sha(raw_payload)}) + _require( + application["seed"] == seed + and application["models"] == history + and application["raw_targets"] == raw_history + and qrf_target.LegacyQRFTrainingState.from_chain(chain) == after + and chain.recipient_index == qrf._index_identity(recipient.index), + "APPLICATION_HISTORY", + ) + result = qrf_target.apply_target( + fitted, recipient, raw, state=application_state + ) + _require( + result.state == chain + and codec.encode_raw_target( + result.raw_draw, target=target, index=recipient.index + ) + == raw_payload, + "APPLICATION_VALUES", + ) + application_state = result.state + raw[target] = actual_raw + receipts[fit.id] = { + "phase": phase, + "target": target, + "training_id": fitted.training_id, + "model_sha256": codec.sha(payload), + "donor_rows": len(donor), + "entity": "person", + "weight_kind": "design", + "regime": fitted.regime, + } + receipts[apply.id] = { + "phase": phase, + "target": target, + "recipient_rows": len(recipient), + "entity": "person", + "model_sha256": codec.sha(payload), + "raw_sha256": codec.sha(raw_payload), + "regime": fitted.regime, + } + return receipts diff --git a/packages/microcosm-build/tests/test_us_graph_property_income_receipts.py b/packages/microcosm-build/tests/test_us_graph_property_income_receipts.py new file mode 100644 index 000000000..3120723d0 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_property_income_receipts.py @@ -0,0 +1,338 @@ +"""Receipt reconstruction over the real, small invented property graph.""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_property_income import FEATURES, assert_preserved, execute, setup + +from microcosm.build.us_runtime.graph_property_income_receipts import ( + verify_property_model_receipts, +) +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import graph_legacy_train, qrf, qrf_target +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph.population import Population + + +@pytest.fixture(scope="module") +def case(tmp_path_factory): + with pytest.MonkeyPatch.context() as patch: + patch.setenv("POPULACE_FIT_N_JOBS", "1") + patch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + values = setup(tmp_path_factory.mktemp("property-receipts")) + compiled, store, _, _, donor, recipient, _ = values + cold = execute(values) + replay = execute(values, resume="require") + assert len(compiled.order) == 12 + assert all(node.hit for node in replay.nodes.values()) + nodes = compiled.graph.nodes + artifacts = { + (node_id, name): store.load_bytes(key) + for node_id, node in cold.nodes.items() + for name, key in node.opaque_artifacts.items() + } + replay_artifacts = { + (node_id, name): store.load_bytes(key) + for node_id, node in replay.nodes.items() + for name, key in node.opaque_artifacts.items() + } + yield SimpleNamespace( + nodes=nodes, + donor=Population.from_frame(donor, "donor"), + recipient=Population.from_frame(recipient, "recipient"), + artifacts=artifacts, + replay_artifacts=replay_artifacts, + cold=cold, + replay=replay, + ) + + +def verify(case, **changes): + args = { + "nodes": case.nodes, + "donor_population": case.donor, + "recipient_population": case.recipient, + "artifacts": case.artifacts, + } + return verify_property_model_receipts(**{**args, **changes}) + + +def changed_population(population, *, person=None, weights=None): + frame = population.frame + changed = Frame( + { + entity: person + if entity == "person" and person is not None + else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + { + entity: weights if weights is not None else frame.weights_for(entity) + for entity in frame.weighted_entities + }, + metadata=frame.metadata, + ) + return Population.from_frame(changed, population.version) + + +def changed_packet(case, key, change): + artifacts = dict(case.artifacts) + packet = codec.decode_json(artifacts[key]) + change(packet) + artifacts[key] = codec.encode_json(packet) + return artifacts + + +def test_actual_cold_required_and_published_receipts_match_without_fitting( + case, monkeypatch +): + def no_fit(*args, **kwargs): + raise AssertionError("Receipt verification must never fit a model") + + monkeypatch.setattr(qrf_target, "fit_target", no_fit) + monkeypatch.setattr(graph_legacy_train, "fit_target", no_fit) + monkeypatch.setattr(qrf.RegimeGatedQRF, "fit_draw_next", no_fit) + calls = [] + original_apply = qrf_target.apply_target + + def apply(*args, **kwargs): + calls.append(args[0].target) + return original_apply(*args, **kwargs) + + monkeypatch.setattr(qrf_target, "apply_target", apply) + result = verify(case) + assert len(calls) == 4 + assert len(result) == 8 + for manifest in (case.cold, case.replay): + # The executor adds capabilities/input-writer provenance around the raw + # kernel receipt. The owning host's _states validates that outer layer. + assert result == { + node_id: { + key: value + for key, value in manifest.node(node_id).receipt.items() + if key != "capabilities" + } + for node_id in result + } + assert all(case.cold.node(n).receipt == case.replay.node(n).receipt for n in result) + assert verify(case, artifacts=case.replay_artifacts) == result + assert len(calls) == 8 + assert_preserved(case.donor.frame, case.cold.population("donor")) + assert_preserved(case.recipient.frame, case.cold.population("recipient")) + assert case.artifacts == case.replay_artifacts + # Table row labels and exact int64 IDs above 2**53 are separate axes. + assert case.recipient.frame.person.index.tolist() == [9, 2, 17, 41] + assert (case.recipient.frame.person.person_id > 2**53).all() + assert case.donor.frame.weights_for("household").values[-1] == 0 + + +@pytest.mark.parametrize("branch", ["donor", "recipient"]) +@pytest.mark.parametrize("mutation", ["feature", "row_order", "float32", "nonfinite"]) +def test_changed_population_inputs_refuse(case, branch, mutation): + population = getattr(case, branch) + person = population.frame.person.copy() + if mutation == "feature": + person.loc[:, FEATURES[0]] += 1e12 + person.loc[:, "age"] += 1e6 + if branch == "donor": + # Keep the declared property sum valid while changing consumed bytes. + person.loc[:, "property_broad_receipts"] += 1e12 + elif mutation == "row_order": + person.index = pd.Index(list(reversed(person.index)), name=person.index.name) + elif mutation == "float32": + person[FEATURES[0]] = person[FEATURES[0]].astype("float32") + else: + person.loc[person.index[0], FEATURES[0]] = np.nan + changed = changed_population(population, person=person) + with pytest.raises(ValueError): + verify(case, **{branch + "_population": changed}) + + +@pytest.mark.parametrize("branch", ["donor", "recipient"]) +@pytest.mark.parametrize("mutation", ["id_dtype", "duplicate_id"]) +def test_person_identity_invalidity_refuses_separately(case, branch, mutation): + # Exercise a post-construction table corruption; a normal Frame constructor + # also refuses invalid identities before this helper is reached. + population = changed_population(getattr(case, branch)) + person = population.frame.person + if mutation == "id_dtype": + person["person_id"] = person.person_id.astype("float64") + else: + person.loc[person.index[1], "person_id"] = person.person_id.iloc[0] + with pytest.raises(ValueError, match="PERSON_IDENTITY_OR_VALUES"): + verify(case, **{branch + "_population": population}) + + +@pytest.mark.parametrize("mutation", ["values", "kind"]) +def test_changed_original_design_weights_refuse(case, mutation): + original = case.donor.frame.weights_for("household") + weights = Weights( + original.values + (1 if mutation == "values" else 0), + WeightKind.CALIBRATED if mutation == "kind" else WeightKind.DESIGN, + ) + donor = changed_population(case.donor, weights=weights) + with pytest.raises(ValueError): + verify(case, donor_population=donor) + + +@pytest.mark.parametrize( + "mutation", + [ + "missing_fit", + "missing_apply", + "duplicate", + "swapped_fits", + "swapped_applies", + "wrong_seed", + "wrong_phase", + "wrong_features", + "wrong_target", + "wrong_edge", + "wrong_population", + "unexpected_param", + ], +) +def test_malformed_declarations_refuse(case, mutation): + nodes = list(case.nodes) + fit_index = next(i for i, n in enumerate(nodes) if n.id == "property.fit.000") + apply_index = next(i for i, n in enumerate(nodes) if n.id == "property.apply.000") + if mutation == "missing_fit": + nodes.pop(fit_index) + elif mutation == "missing_apply": + nodes.pop(apply_index) + elif mutation == "duplicate": + nodes.append(nodes[fit_index]) + elif mutation == "swapped_fits": + nodes[fit_index], nodes[fit_index + 1] = nodes[fit_index + 1], nodes[fit_index] + elif mutation == "swapped_applies": + nodes[apply_index], nodes[apply_index + 1] = ( + nodes[apply_index + 1], + nodes[apply_index], + ) + elif mutation == "wrong_edge": + node = nodes[apply_index] + nodes[apply_index] = replace( + node, + artifact_inputs=( + replace(node.artifact_inputs[0], producer="property.fit.001"), + *node.artifact_inputs[1:], + ), + ) + elif mutation == "wrong_population": + nodes[apply_index] = replace(nodes[apply_index], population="donor") + else: + index = apply_index if mutation == "wrong_seed" else fit_index + 1 + node = nodes[index] + field, value = { + "wrong_seed": ("seed", True), + "wrong_phase": ("phase", "foreign"), + "wrong_features": ("predictors", tuple(reversed(FEATURES))), + "wrong_target": ("target", "property_dividends"), + "unexpected_param": ("extra", 1), + }[mutation] + nodes[index] = replace(node, params={**node.params, field: value}) + with pytest.raises(ValueError): + verify(case, nodes=tuple(nodes)) + + +@pytest.mark.parametrize( + "node,output", + [ + ("property.fit.000", "model"), + ("property.fit.003", "training_state"), + ("property.apply.000", "raw_draw"), + ("property.apply.003", "apply_state"), + ], +) +@pytest.mark.parametrize("mutation", ["missing", "not_bytes"]) +def test_missing_or_unowned_payload_shape_refuses(case, node, output, mutation): + artifacts = dict(case.artifacts) + if mutation == "missing": + del artifacts[node, output] + else: + artifacts[node, output] = bytearray(artifacts[node, output]) + with pytest.raises(ValueError, match="ARTIFACT_ROSTER"): + verify(case, artifacts=artifacts) + + +@pytest.mark.parametrize( + "mutation", + [ + "model_bytes", + "model_swapped", + "training_id", + "training_donor_state", + "training_history", + "apply_model_history", + "apply_seed", + "apply_draw_rng", + "raw_and_hash", + ], +) +def test_altered_artifacts_refuse(case, mutation): + if mutation in ("model_bytes", "model_swapped"): + artifacts = dict(case.artifacts) + key = ("property.fit.000", "model") + artifacts[key] = ( + artifacts[key][:-1] + bytes([artifacts[key][-1] ^ 1]) + if mutation == "model_bytes" + else artifacts["property.fit.001", "model"] + ) + elif mutation == "training_id": + artifacts = changed_packet( + case, + ("property.fit.001", "training_state"), + lambda p: p["models"][-1].update(training_id="0" * 64), + ) + elif mutation == "training_donor_state": + artifacts = changed_packet( + case, + ("property.fit.000", "training_state"), + lambda p: p["state"].update(weight_sha256="0" * 64), + ) + elif mutation == "training_history": + artifacts = changed_packet( + case, + ("property.fit.001", "training_state"), + lambda p: p["models"][0].update(sha256="0" * 64), + ) + elif mutation == "apply_model_history": + artifacts = changed_packet( + case, + ("property.apply.001", "apply_state"), + lambda p: p["models"][0].update(sha256="0" * 64), + ) + elif mutation == "apply_seed": + artifacts = changed_packet( + case, ("property.apply.000", "apply_state"), lambda p: p.update(seed=999) + ) + elif mutation == "apply_draw_rng": + artifacts = changed_packet( + case, + ("property.apply.000", "apply_state"), + lambda p: p["state"].update(draw_rng_state=np.random.PCG64(912).state), + ) + else: + key = ("property.apply.000", "raw_draw") + raw = codec.read_raw_target( + case.artifacts[key], + target="property_ordinary_interest", + index=case.recipient.frame.person.index, + ) + payload = codec.encode_raw_target( + raw + 1, + target="property_ordinary_interest", + index=case.recipient.frame.person.index, + ) + artifacts = changed_packet( + case, + ("property.apply.000", "apply_state"), + lambda p: p["raw_targets"][0].update(sha256=codec.sha(payload)), + ) + artifacts[key] = payload + with pytest.raises(ValueError): + verify(case, artifacts=artifacts) From 227dc8df57c01b0fb5a3eaf5648dcda7151fd600 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:01:42 -0400 Subject: [PATCH 177/255] Preserve Weights validation and class state during copying --- changelog.d/weights-copy-protocol.fixed.md | 1 + packages/microcosm-frame/README.md | 8 + .../src/microcosm/frame/weights.py | 10 ++ .../tests/test_weights_schema.py | 147 ++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 changelog.d/weights-copy-protocol.fixed.md diff --git a/changelog.d/weights-copy-protocol.fixed.md b/changelog.d/weights-copy-protocol.fixed.md new file mode 100644 index 000000000..4a29fdf93 --- /dev/null +++ b/changelog.d/weights-copy-protocol.fixed.md @@ -0,0 +1 @@ +Reconstruct base `Weights` through validation during shallow copy, deep copy and pickle round trips, preserving exact values and read-only storage without changing class metadata. Preserve existing subclass copy protocols. diff --git a/packages/microcosm-frame/README.md b/packages/microcosm-frame/README.md index 062137103..0c0d408a7 100644 --- a/packages/microcosm-frame/README.md +++ b/packages/microcosm-frame/README.md @@ -29,6 +29,14 @@ no operator ever re-derives structure or silently corrupts weights: support installs via `microcosm-frame[axiom]`; the engine itself currently installs from an `axiom-rules-engine` checkout. +Shallow copy, deep copy and pickle round trips of base `Weights` objects +reconstruct through the validated constructor. They preserve exact float64 +values and weight kind, create independent read-only storage, and leave the +class namespace unchanged, including Python 3.14's deferred annotation state. +Subclass instances retain their existing default or custom copy protocol so +extra state and different constructors keep working; subclasses remain +responsible for validation and read-only storage during reconstruction. + The Axiom adapter requires callers to declare the exact RuleSpec authority roots used for compilation. It does not infer them from the module path, environment, current directory, or sibling checkouts: diff --git a/packages/microcosm-frame/src/microcosm/frame/weights.py b/packages/microcosm-frame/src/microcosm/frame/weights.py index d4419a3fa..758cb6a73 100644 --- a/packages/microcosm-frame/src/microcosm/frame/weights.py +++ b/packages/microcosm-frame/src/microcosm/frame/weights.py @@ -109,6 +109,16 @@ def __post_init__(self) -> None: values.setflags(write=False) object.__setattr__(self, "values", values) + def __reduce_ex__(self, protocol: int): + # Keep subclasses' existing state/custom protocols rather than assuming + # their constructor accepts only (values, kind). + if type(self) is not Weights: + return super().__reduce_ex__(protocol) + # Revalidate and restore read-only storage on copy/unpickle. An explicit + # reduction also avoids copyreg's __slotnames__ class mutation, which + # changes the captured annotation namespace on Python 3.14. + return Weights, (self.values, self.kind) + def __len__(self) -> int: return int(self.values.size) diff --git a/packages/microcosm-frame/tests/test_weights_schema.py b/packages/microcosm-frame/tests/test_weights_schema.py index 2257e7d2d..280e3bcfb 100644 --- a/packages/microcosm-frame/tests/test_weights_schema.py +++ b/packages/microcosm-frame/tests/test_weights_schema.py @@ -1,5 +1,9 @@ """Weights and schema primitives.""" +import copy +import pickle +from functools import partial + import numpy as np import pytest @@ -62,6 +66,149 @@ def test_kind_transition_authority(self) -> None: assert_kind_transition(WeightKind.CALIBRATED, WeightKind.IMPORTANCE) +def _trusted_pickle_roundtrip(value, *, protocol=pickle.DEFAULT_PROTOCOL): + # Only bytes constructed from this test's invented value are loaded. + return pickle.loads(pickle.dumps(value, protocol=protocol)) + + +_COPY_OPERATIONS = ( + pytest.param(copy.copy, id="shallow"), + pytest.param(copy.deepcopy, id="deep"), + pytest.param(_trusted_pickle_roundtrip, id="trusted-pickle"), +) +_ROUNDTRIP_OPERATIONS = ( + *_COPY_OPERATIONS[:2], + *( + pytest.param( + partial(_trusted_pickle_roundtrip, protocol=protocol), + id=f"trusted-pickle-{protocol}", + ) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1) + ), +) + + +@pytest.fixture +def cold_weights_namespace(): + # copyreg lazily caches slot names even on a dataclass without slots. Reset + # only that cache to exercise a first copy, and restore its original state. + missing = object() + previous = vars(Weights).get("__slotnames__", missing) + if previous is not missing: + delattr(Weights, "__slotnames__") + try: + yield + finally: + if "__slotnames__" in vars(Weights): + delattr(Weights, "__slotnames__") + if previous is not missing: + Weights.__slotnames__ = previous + + +class _PlainWeights(Weights): + pass + + +class _StatefulWeights(Weights): + def __init__(self, values, kind, *, label): + super().__init__(values, kind) + object.__setattr__(self, "label", label) + + +class _SlottedWeights(Weights): + __slots__ = ("label",) + + def __init__(self, values, kind, *, label): + super().__init__(values, kind) + object.__setattr__(self, "label", label) + + +class _CustomReducedWeights(_StatefulWeights): + def __reduce__(self): + return _restore_custom_weights, (self.values, self.kind, self.label) + + +def _restore_custom_weights(values, kind, label): + return _CustomReducedWeights(values, kind, label=label) + + +class TestWeightsCopy: + @pytest.mark.parametrize("operation", _ROUNDTRIP_OPERATIONS) + @pytest.mark.parametrize("kind", tuple(WeightKind)) + def test_preserves_exact_values_and_constructor_invariants(self, operation, kind): + original = Weights( + np.array([0.0, -0.0, np.nextafter(0.0, 1.0), 1.25, 2.0**53 + 2.0]), + kind, + ) + copied = operation(original) + assert type(copied) is Weights + assert copied is not original + assert copied.kind is kind + assert copied.values.dtype == np.float64 + np.testing.assert_array_equal( + copied.values.view(np.uint64), original.values.view(np.uint64) + ) + assert not copied.values.flags.writeable + assert not np.shares_memory(copied.values, original.values) + with pytest.raises(ValueError, match="read-only"): + copied.values[0] = 4.0 + assert not original.values.flags.writeable + + @pytest.mark.parametrize("operation", _ROUNDTRIP_OPERATIONS) + def test_copy_does_not_mutate_class_namespace( + self, operation, cold_weights_namespace + ): + original = Weights(np.array([1.0]), WeightKind.DESIGN) + before = {name: id(value) for name, value in vars(Weights).items()} + operation(original) + # On Python 3.14 this namespace is also captured by the deferred + # annotation function, so a new __slotnames__ entry changes live seals. + assert {name: id(value) for name, value in vars(Weights).items()} == before + assert "__slotnames__" not in vars(Weights) + + @pytest.mark.parametrize("operation", _COPY_OPERATIONS) + @pytest.mark.parametrize( + ("field", "value", "error", "match"), + [ + ("values", np.array([-1.0]), ValueError, "non-negative"), + ("kind", "design", TypeError, "WeightKind"), + ], + ) + def test_reconstruction_revalidates_corrupted_objects( + self, operation, field, value, error, match + ): + original = Weights(np.array([1.0]), WeightKind.DESIGN) + object.__setattr__(original, field, value) + with pytest.raises(error, match=match): + operation(original) + + @pytest.mark.parametrize("operation", _COPY_OPERATIONS) + @pytest.mark.parametrize( + "subclass", + [_PlainWeights, _StatefulWeights, _SlottedWeights, _CustomReducedWeights], + ) + def test_existing_subclass_state_and_protocol_are_preserved( + self, operation, subclass + ): + kwargs = {} if subclass is _PlainWeights else {"label": ["source"]} + original = subclass(np.array([0.0, 2.5]), WeightKind.IMPORTANCE, **kwargs) + copied = operation(original) + assert type(copied) is subclass + assert copied is not original + assert copied.kind is original.kind + np.testing.assert_array_equal(copied.values, original.values) + if kwargs: + assert copied.label == original.label + if operation is copy.copy: + assert copied.label is original.label + else: + assert copied.label is not original.label + # Subclasses retain their own/default protocol, including extra state + # and constructors that do not accept the base class's two arguments. + if subclass is _CustomReducedWeights: + assert not copied.values.flags.writeable + + class TestMassChange: def test_factor_and_reason_are_recorded(self) -> None: change = MassChange(factor=2.0, reason="oversampled the rare stratum") From 73e114480170080d2caf3e9e42564d72aec1d503 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:32:25 -0400 Subject: [PATCH 178/255] Add opt-in current survey property graph fragment --- .../current-survey-property-graph.added.md | 1 + docs/us-current-survey-property-graph.md | 53 + .../graph_current_survey_property.py | 1042 +++++++++++++++++ .../test_us_current_survey_property_graph.py | 482 ++++++++ 4 files changed, 1578 insertions(+) create mode 100644 changelog.d/current-survey-property-graph.added.md create mode 100644 docs/us-current-survey-property-graph.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py create mode 100644 packages/microcosm-build/tests/test_us_current_survey_property_graph.py diff --git a/changelog.d/current-survey-property-graph.added.md b/changelog.d/current-survey-property-graph.added.md new file mode 100644 index 000000000..45b5f9bd0 --- /dev/null +++ b/changelog.d/current-survey-property-graph.added.md @@ -0,0 +1 @@ +Add an opt-in 16-node survey property-income fragment with original DESIGN donor selection, joint ACS draws, signed reconciliation, explicit unknownness, and exact clone attachment while preserving legacy tax leaves and their stated capital-gains limitation. diff --git a/docs/us-current-survey-property-graph.md b/docs/us-current-survey-property-graph.md new file mode 100644 index 000000000..68cf42d7c --- /dev/null +++ b/docs/us-current-survey-property-graph.md @@ -0,0 +1,53 @@ +# Current survey property-income graph + +The opt-in `graph_current_survey_property.py` fragment adds 16 operations to the existing US financial graph. It puts qualified ASEC components and modeled ACS components on the complete initial clone pair. The country financial host retains the original source owners, verifies every graph observation and artifact, and issues the result. The fragment creates no separate issuance registry. + +`PropertyIncomeOptions(scales=..., atol=..., rtol=..., n_estimators=...)` requires all four settings. The options object is frozen and has a canonical `to_bytes()` representation. No numeric modeling default is supplied. The first three components are nonnegative; broad property receipts can be signed. A negative or zero anchor can coexist with positive components and an offsetting negative broad-property component. + +## Visible operations + +| Operation | Meaning | +| --- | --- | +| `survey_property.source_projection` | Bind the actual qualified property composition, the existing financial predictor projection/matrix, original host artifacts, and geography validation when present. Emit a projection and exact donor/recipient matrices. | +| `survey_property.asec_eligible_donor` | Filter the original CREATE by the exact eligible ASEC person IDs. Preserve original household DESIGN weights, including their person mapping. | +| `survey_property.asec_donor_columns` | Materialize shared predictor columns, reported total, and four qualified components. | +| `survey_property.acs_eligible_recipient` | Filter original CREATE to ACS persons whose source anchor is known and in the adult reporting universe. | +| `survey_property.acs_recipient_columns` | Materialize shared predictors and adjusted INTP as the reported-total feature. | +| `survey_property.fit.000`–`.003` | Fit the existing DESIGN-weighted chained QRF over the four components. | +| `survey_property.apply.000`–`.003` | Draw once per eligible original ACS person, preserving ordered prior-component conditioning. | +| `survey_property.draws` | Retain raw draws and their complete model/application history. | +| `survey_property.reconcile` | Apply the shared signed-total reconciliation rule with the explicit scales and tolerances. | +| `survey_property.attach` | Join original source identities to both initial clones and retain all existing rows, entities, weights, geography, metadata, and legacy financial leaves. | + +Both model branches leave CREATE before importance allocation. Their FILTER operations return person keep masks, preserving actual raw source columns. Shared predictor materialization, including the existing analytical treatment of ACS earnings NIU, happens through declared column operations. The FILTER does not substitute the separately prepared feature frame for CREATE. + +The projection binds complete descriptive physical seals, including unknown masks and excluded rows, not just the finite fitting matrices. It carries the donor exclusion summary and recipient exclusion counts. No excluded person silently becomes a zero-valued donor or recipient. Empty eligible donor or recipient branches are currently refused before fitting; an empty-branch execution policy is not implemented. + +## Attachment and knownness + +The attachment owns the four `property_*` components, their four `draw_property_*` columns, `property_reported_total`, reconciliation adjustments, nullable active-bound diagnostics, residual and objective, two source-basis discrepancy columns, and two knownness flags. `owned_columns()` is the authoritative roster. + +ASEC persons retain each individually qualified component and reported total, including NaN for an unresolved value. Donor exclusion does not erase otherwise known amounts. The source-basis discrepancies remain available, and the complete source qualification/exclusion descriptions stay bound to the projection. ASEC raw-draw and reconciliation diagnostics are absent because no draw or reconciliation is performed on these observations. + +Eligible ACS persons receive one ordered joint draw and its reconciled components. Their original `person_id` and pandas row index are checked separately, then the original support ID, native source ID, source label, and exact clone roles establish the two-row fan-out. Excluded ACS persons receive NaN for model components and the attached anchor, with false `property_anchor_known` and `property_components_known`. Their original source anchor status remains in the qualified source projection; under-15 values are not silently converted into a new analytical zero. + +`property_anchor_known` describes the attached anchor's finite value. `property_components_known` describes all four attached components being finite; it is distinct from eligibility for donor fitting. Reconciliation bound flags use nullable booleans so an unperformed reconciliation does not become a false observation. + +## Tax and period boundaries + +The eight existing financial tax leaves remain unchanged. Attachment reads all eight and explicitly follows the legacy financial attachment, checks them against its retained draw history, then adds property columns. The legacy capital-gains chain still conditions on legacy INT and DIV draws. The receipt records `tax_split_rebased: false` and the capital-gains limitation. `property_legacy_interest_draw_minus_reconciled_interest` measures the legacy ACS interest draw minus the reconciled ordinary plus retirement interest; it is a diagnostic, not a new tax assignment. + +Broad property receipts are not identified with rental income or a tax-law leaf. Retirement-account earnings are not retirement withdrawals. Tax-leaf rebasing and a CAP model conditioned on reconciled components require a separate declared successor. + +Periods remain those of the qualified sources: ASEC 2025 interview and calendar-2024 income; ACS 2024 rolling prior twelve months, adjusted by ADJINC to the declared price basis. No equivalence of the reporting windows is asserted. The person-level conditional chain is not a certification of household-level dependence or held-out fit quality. + +## Host integration and verification + +- `current_survey_property_nodes(qualified, clone_frame, host_pins=..., options=...)` declares the fragment. +- `register_property_kernels(...)` registers the country and shared numerical kernels over the retained source owners. +- `reconstruct_property_results(..., artifacts=..., legacy_matrix_producer_key=...)` returns eight expected non-fit results: projection, two filters, two model-column operations, draws, reconciliation, and attachment. The host patches these at their declared population versions and checks actual observations. The separate model-receipt verifier binds the eight fit/apply receipts to actual donor/recipient populations and stored model bytes. +- `verify_materialized_property_income(..., legacy_population=..., population=...)` requalifies actual source owners and compares the full final population against attachment applied to the independently checked legacy population. + +Reconstruction validates the complete ordered raw-draw history, training-model references, recipient index, person IDs and summaries. It invokes the existing deterministic reconciliation rule to derive the expected reconciliation columns; it never fits or draws another model. Store artifact type, producer-key and key validation remains a prerequisite owned by the host. The host must retain its existing final source, implementation, cache, full-population and issuance checks after relevant I/O. + +Changing only scales affects the reconciliation and attachment declarations, leaving source selection and QRF fitting declarations unchanged. The absence of property options in a host call preserves the existing strict legacy path. Graph replay validates cached output observations as well as labels; a cache hit is not source admission or a release verdict. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py new file mode 100644 index 000000000..c4133c340 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_property.py @@ -0,0 +1,1042 @@ +"""Opt-in property components on the complete initial survey clone frame. + +The existing country host retains source authority and issues its financial +run. These nodes borrow that owner's sources; neither options nor descriptive +projections are an admission token. Legacy tax leaves and their CAP chain remain +unchanged. Broad property receipts and retirement-account earnings are not tax +rental income or retirement withdrawals. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import graph_signed_reconciliation as signed_graph +from microcosm.fit import model_input, qrf +from microcosm.fit.graph_legacy_qrf import LegacyQRFApplyKernel +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.kernel import KernelContext +from microcosm.graph.keys import opaque_artifact_key + +from . import current_property_income_sources as sources +from . import current_survey_predictors as shared +from . import graph_current_survey_predictors as financial +from . import graph_property_income as model +from . import survey_population_replay as replay +from .property_income_constants import ( + PROPERTY_COMPONENTS, + PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, +) + +host = shared.host +PROTOCOL = "microcosm.us.current-survey-property.v1" +PREFIX = "survey_property" +PROJECTION_NODE = PREFIX + ".source_projection" +DONOR_NODE = PREFIX + ".asec_eligible_donor" +DONOR_COLUMNS_NODE = PREFIX + ".asec_donor_columns" +RECIPIENT_NODE = PREFIX + ".acs_eligible_recipient" +RECIPIENT_COLUMNS_NODE = PREFIX + ".acs_recipient_columns" +ATTACH_NODE = PREFIX + ".attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_property_projection", 1) +CAP_LIMITATION = "legacy_CAP_conditions_on_legacy_INT_DIV; reconciled_component_consistency_not_claimed" +LEGACY_DIFFERENCE = "property_legacy_interest_draw_minus_reconciled_interest" +BASIS_DIAGNOSTICS = ( + "property_interest_component_discrepancy", + "property_reported_minus_component_total", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_PROPERTY_" + reason) + + +@dataclass(frozen=True, kw_only=True) +class PropertyIncomeOptions: + scales: tuple[float, float, float, float] + atol: float + rtol: float + n_estimators: int + + def __post_init__(self): + # Reuse the real numeric declaration's domain checks, including finite + # positive scales and finite nonnegative tolerances; no fitted model. + self.nodes((PROPERTY_REPORTED_TOTAL,)) + + def nodes(self, features): + return model.property_income_nodes( + PREFIX, + donor_population=DONOR_NODE, + recipient_population=RECIPIENT_NODE, + features=features, + seed=shared.SEED, + n_estimators=self.n_estimators, + scales=self.scales, + atol=self.atol, + rtol=self.rtol, + ) + + def to_bytes(self): + return codec.encode_json(self.document()) + + def document(self): + return { + "scales": self.scales, + "atol": self.atol, + "rtol": self.rtol, + "n_estimators": self.n_estimators, + } + + +def _features(qualified): + return ( + *shared.feature_columns(qualified.shared_predictors.demographic_conditioning), + PROPERTY_REPORTED_TOTAL, + ) + + +def _diagnostics(): + return signed_graph._diagnostics(PROPERTY_COMPONENTS, PREFIX + "_reconciliation") + + +def owned_columns(): + adjustments, active, residual, objective = _diagnostics() + floats = ( + *PROPERTY_COMPONENTS, + *PROPERTY_DRAW_COLUMNS, + PROPERTY_REPORTED_TOTAL, + *adjustments, + residual, + objective, + LEGACY_DIFFERENCE, + *BASIS_DIAGNOSTICS, + ) + return ( + *tuple(Owned("person", c, "float64") for c in floats), + *tuple(Owned("person", c, "boolean") for c in active), + Owned("person", "property_anchor_known", "bool"), + Owned("person", "property_components_known", "bool"), + ) + + +def _seal_json(value): + if type(value) is bytes: + return {"bytes_sha256": codec.sha(value)} + if type(value) is tuple: + return [_seal_json(x) for x in value] + if type(value) is dict: + return {k: _seal_json(v) for k, v in value.items()} + return value + + +def _projection_values(qualified): + require(type(qualified) is sources.QualifiedPropertyIncomeSources, "QUALIFIED_TYPE") + require( + qualified.donor_frame is not None + and qualified.recipient_frame is not None + and qualified.recipient_matrix is not None, + "EMPTY_MODEL_BRANCH", + ) + donor_matrix = model_input.encode_recipient_matrix( + qualified.donor_columns, + entity="person", + entity_ids=qualified.donor_columns.index.to_numpy(dtype=" 0 + assert ( + final.person.loc[ + mask, [*graph.PROPERTY_COMPONENTS, graph.PROPERTY_REPORTED_TOTAL] + ] + .isna() + .all() + .all() + ) + assert ( + not final.person.loc[ + mask, ["property_anchor_known", "property_components_known"] + ] + .to_numpy() + .any() + ) + assert ( + observed[graph.RECIPIENT_NODE].frame.person.person_id.to_numpy() + == qualified.recipient_columns.index.to_numpy() + ).all() + assert ( + observed[graph.DONOR_NODE].frame.resolve_weights("person").kind + is WeightKind.DESIGN + ) + assert ( + observed[graph.DONOR_NODE].frame.person.person_id.to_numpy() + == qualified.donor_columns.index.to_numpy() + ).all() + + +@pytest.mark.parametrize( + "which", ["raw", "state", "summary", "reconciliation", "projection"] +) +def test_tampered_graph_artifacts_refuse(complete, which): + live, q, pins, options, _, manifest, _, artifacts, *_ = complete + copied = dict(artifacts) + key = { + "raw": (graph.PREFIX + ".apply.001", "raw_draw"), + "state": (graph.PREFIX + ".apply.001", "apply_state"), + "summary": (graph.PREFIX + ".draws", "summary"), + "reconciliation": (graph.PREFIX + ".reconcile", "summary"), + "projection": (graph.PROJECTION_NODE, "projection"), + }[which] + if which == "raw": + raw = graph.codec.read_raw_target( + copied[key], + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ).copy() + raw[0] += 1 + copied[key] = graph.codec.encode_raw_target( + raw, + target=graph.PROPERTY_COMPONENTS[1], + index=q.recipient_frame.person.index, + ) + else: + data = graph.codec.decode_json(copied[key]) + data["unexpected"] = True + copied[key] = graph.codec.encode_json(data) + with pytest.raises((ValueError, KeyError)): + graph.reconstruct_property_results( + q, + live.clone_population.frame, + host_pins=pins, + options=options, + artifacts=copied, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) + + +def test_scales_change_only_reconciliation_and_attachment_declarations(complete): + live, q, pins, options, *_ = complete + old = graph.current_survey_property_nodes( + q, live.clone_population.frame, host_pins=pins, options=options + ) + new = graph.current_survey_property_nodes( + q, + live.clone_population.frame, + host_pins=pins, + options=replace(options, scales=(2.0, 1.0, 1.0, 1.0)), + ) + assert [a.id for a, b in zip(old, new, strict=True) if a != b] == [ + graph.PREFIX + ".reconcile", + graph.ATTACH_NODE, + ] + + +def test_origin_identity_refusal_and_order_independence(complete): + live, q, pins, options, _, manifest, observed, artifacts, *_ = complete + ids, axis = graph._clone_lookup(q, live.clone_population.frame) + reordered = replace(q, origins=q.origins.iloc[::-1]) + assert np.array_equal( + graph._clone_lookup(reordered, live.clone_population.frame)[0], ids + ) + bad = q.origins.copy(deep=True) + bad.iloc[0, bad.columns.get_loc("native_person_id")] += 1 + with pytest.raises(ValueError, match="CLONE_ORIGIN_IDENTITY"): + graph._clone_lookup(replace(q, origins=bad), live.clone_population.frame) + assert axis.dtype == np.dtype("int64") + + +@pytest.mark.parametrize("target", ["donor_frame", "recipient_frame"]) +def test_empty_branch_refuses_without_model_defaults(complete, target): + live, q, pins, options, *_ = complete + with pytest.raises(ValueError, match="EMPTY_MODEL_BRANCH"): + graph.current_survey_property_nodes( + replace(q, **{target: None}), + live.clone_population.frame, + host_pins=pins, + options=options, + ) + + +def test_declaration_without_source_execution(monkeypatch): + from microcosm.frame import US_SCHEMA, Frame, Weights + + tables = { + e: pd.DataFrame( + { + US_SCHEMA.entity_id_column(e): np.array([1], dtype="int64"), + "fixture_" + e: np.array([1.0]), + } + ) + for e in US_SCHEMA.entities + } + for e in US_SCHEMA.group_entities: + tables["person"][US_SCHEMA.membership_column(e)] = np.array([1], dtype="int64") + tables["person"]["employment_income"] = 5.0 + frame = Frame( + tables, US_SCHEMA, {"household": Weights(np.array([1.0]), WeightKind.DESIGN)} + ) + features = (*shared.FEATURES, graph.PROPERTY_REPORTED_TOTAL) + q = SimpleNamespace( + source_frame=frame, + donor_frame=frame, + recipient_frame=frame, + donor_columns=pd.DataFrame(columns=(*features, *graph.PROPERTY_COMPONENTS)), + recipient_columns=pd.DataFrame(columns=features), + shared_predictors=SimpleNamespace( + donor_frame=frame, + demographic_conditioning=False, + geography_config_payload=None, + ), + ) + # Isolate declaration shape only. No fixture object claims source authority; + # the full graph fixture below uses the actual source qualifiers unchanged. + monkeypatch.setattr(graph, "_params", lambda *args: {"protocol": graph.PROTOCOL}) + nodes = graph.current_survey_property_nodes( + q, frame, host_pins={}, options=graph.PropertyIncomeOptions(**OPTIONS) + ) + assert len(nodes) == 16 + assert set(nodes[-1].params) == {"protocol", *OPTIONS} + + context_tables = {} + for entity in frame.entities: + structural = [US_SCHEMA.entity_id_column(entity)] + if entity == "person": + structural.extend( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + context_tables[entity] = frame.table(entity)[ + structural + [c for c in frame.table(entity) if c not in structural] + ].copy() + for name in shared.OUTPUTS: + context_tables["person"][name] = 0.0 + context = graph.KernelContext( + node=nodes[-1], + tables=context_tables, + weights={e: frame.resolve_weights(e) for e in frame.entities}, + strata=frame.strata, + params=nodes[-1].params, + rng=None, + ) + graph._clone_context(context, frame) + assert set(shared.OUTPUTS).issubset( + next(x.columns for x in nodes[-1].inputs if x.entity == "person") + ) + + +@pytest.mark.parametrize("field", ["property_reported_total", shared.OUTPUTS[0]]) +def test_final_verifier_refuses_changed_owned_or_retained_values(complete, field): + from test_us_current_survey_puf_transfer import _copy_population + + live, _, pins, options, _, manifest, observed, loaded, *_ = complete + changed = _copy_population(observed[graph.ATTACH_NODE]) + table = changed.frame.person + row = table.index[np.flatnonzero(np.isfinite(table[field].to_numpy()))[0]] + table.loc[row, field] += 1 + with pytest.raises((ValueError, AssertionError)): + graph.verify_materialized_property_income( + live.preparation, + live.allocated_population, + live.clone_population, + legacy_population=observed[financial.ATTACH_NODE], + population=changed, + host_pins=pins, + options=options, + artifacts=loaded, + legacy_matrix_producer_key=manifest.node(financial.PROJECTION_NODE).key, + ) From 5ff83f53e3f6f5940889150d8f4fd6c4590525d5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:27:18 -0400 Subject: [PATCH 179/255] Integrate qualified property income into the checked US financial graph --- .../atomic-property-financial-host.added.md | 1 + docs/us-atomic-property-financial.md | 73 ++++ docs/us-uk-release-path.md | 11 +- .../us-property-financial-host-20260912.md | 73 ++++ .../graph_atomic_survey_financial.py | 262 ++++++++++++++- .../build/us_runtime/graph_survey_puf55.py | 11 +- ...test_us_graph_atomic_property_financial.py | 318 ++++++++++++++++++ .../tests/test_us_spine_blindness.py | 9 + 8 files changed, 734 insertions(+), 24 deletions(-) create mode 100644 changelog.d/atomic-property-financial-host.added.md create mode 100644 docs/us-atomic-property-financial.md create mode 100644 experiments/us-property-financial-host-20260912.md create mode 100644 packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py diff --git a/changelog.d/atomic-property-financial-host.added.md b/changelog.d/atomic-property-financial-host.added.md new file mode 100644 index 000000000..460a98daf --- /dev/null +++ b/changelog.d/atomic-property-financial-host.added.md @@ -0,0 +1 @@ +Add an explicit property-income extension to the checked US financial host, retaining complete legacy populations and validating model receipts, reconstructed outputs and required cache replay. diff --git a/docs/us-atomic-property-financial.md b/docs/us-atomic-property-financial.md new file mode 100644 index 000000000..05e096d50 --- /dev/null +++ b/docs/us-atomic-property-financial.md @@ -0,0 +1,73 @@ +# Current survey property-income host + +`run_atomic_survey_financial` can append the current property-income graph to +the existing financial composition. Pass an explicit `PropertyIncomeOptions` +through `property_income`; omitting it retains the nineteen-node path. + +```python +from microcosm.build.us_runtime.graph_atomic_survey_financial import ( + run_atomic_survey_financial, +) +from microcosm.build.us_runtime.graph_current_survey_property import ( + PropertyIncomeOptions, +) + +options = PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), + atol=1e-10, + rtol=1e-12, + n_estimators=2, +) +run = run_atomic_survey_financial( + **source_arguments, + geography_config=atomic_geography, + property_income=options, + n_estimators=2, + return_values=True, +) +checked = run.checked_view() +``` + +The small tree counts above are development settings. The caller supplies the +existing authenticated source arguments and atomic geography configuration. +They are not replaced by a downloaded graph, a detached Frame or a receipt. + +The thirty-five-node composition consists of the nine-node survey, clone and +atomic-geography prefix; ten existing financial nodes; and sixteen property +nodes. The new branches select eligible original ASEC donors and original ACS +recipients before allocation. Four ordered, DESIGN-weighted conditional models +produce raw component draws, followed by reconciliation to a known adult ACS +property-income anchor. An explicit attachment carries each original result to +both initial clones. Original ASEC components remain separately qualified even +when that person is excluded from fitting. See the +[fragment contract](us-current-survey-property-graph.md). + +The output adds 24 component, raw-draw, diagnostic and knownness columns. It +preserves the complete preceding financial population: other columns, entity +membership, geography, schema, metadata, weights, strata, ownership and mass +history. Missing source anchors and unresolved components retain missing +values and explicit knownness. This mode does not infer property-income zeros +for children outside the survey question's universe. + +The runner retains both the preceding and extended populations. Before issuance, +execution and required cache replay verify property model training against the +original donor and replay four applications against actual recipient features. +Later issued-run checks requalify the existing source owners, validate the legacy +financial output, reconstruct the complete extended population and check the +verified model artifacts' byte identities. They do not refit or reapply the +models. Copied public result dataclasses have no run authority. See the +[model receipt contract](us-property-model-receipts.md). + +This first extension leaves the eight existing financial leaves unchanged. +Its run document therefore records `tax_split_rebased: false` and the legacy +capital-gain model's conditioning on its earlier interest and dividend draws. +The new property components do not yet establish coherent replacement tax +inputs. A later rebase must use ordinary interest rather than account earnings, +preserve separately known ASEC observations, and explicitly complete any +required unknown inputs before full PUF use. + +The PUF host derives its upstream node count and checks the actual final +financial writer. Its recipient nodes read all columns of that population, +including the new property fields, so the dependency is present in the graph. +This interface does not establish that a native PUF run, tax rebase, calibration +or release has passed. Candidate execution evidence belongs in `experiments/`. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index 53e873a6b..aae4317e7 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -199,9 +199,14 @@ allocation flags through the retained source owner; its 35 invented-source controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) projects joint component draws onto a caller-qualified total while allowing declared signed components and preserving loss offsets; its 45 numeric controls -pass. These components are not yet invoked by the financial or PUF host. -The donor measurement bridge, scales, named graph reconciliation operation and -dependent model replay remain to be implemented and verified. +pass. An explicit property-income option now integrates the source bridge, +four conditional component models, signed reconciliation and clone attachment +into the existing financial host. Its actual invented 35-node construction and +required replay pass, preserving the full preceding population. The new columns +still coexist with the earlier tax inputs: the tax-input rebase, retirement +bridge and complete PUF/native adoption remain separate work. The focused PUF +handoff retest passes after correction of an invalid invented source record; +see the [scoped host evidence](../experiments/us-property-financial-host-20260912.md). The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale diff --git a/experiments/us-property-financial-host-20260912.md b/experiments/us-property-financial-host-20260912.md new file mode 100644 index 000000000..87c12e0a0 --- /dev/null +++ b/experiments/us-property-financial-host-20260912.md @@ -0,0 +1,73 @@ +# Property-income integration in the US financial graph + +The existing checked financial host now accepts an explicit property-income +configuration. Its ordinary graph remains 19 nodes. The configured graph adds +16 operations: qualified donor and recipient projections, four conditional +fits and draws, signed reconciliation, and attachment to the complete initial +survey clones after atomic geography. + +This is development integration. It adds 24 component, draw, diagnostic and +knownness columns while retaining the preceding eight financial leaves. It does +not yet rebase their tax treatment, change the legacy capital-gain model's +conditioning, complete missing original-channel inputs, or certify native data. + +## Independent verification + +The invented-source host test executed an ordinary 19-node build, an extended +35-node build sharing the 19-node prefix, and required replay with all 35 cache +hits. The receiving population contains 18 people in 12 households. Eight +controls passed: default behavior, real extended execution/replay and issuance, +complete preceding-population preservation, source/clone identity and unknowns, +three retained-state mutation cases, and refusal of a copied run dataclass. + +The complete comparison includes every preceding entity table and column, +relationships, row order, schema, geography, metadata, design and current +weights, strata, owners and mass history. Fitting uses original design-weighted +donors; attachment shares the resulting property values across each original +person's two initial clones. Missing anchors and unresolved components stay +unknown. Negative property totals remain valid where the declared signed +component permits them. + +The run took 729.20 seconds and peaked at 584,957,952 bytes RSS. Source and +resource hashes matched before and after. Its receipt SHA256 is +`90137cd08aa151e9db92a8bc5c1c51c2448f8e3bceb6d3b54d6fdf3ff9530e23`. +This was **eight passes and one failure**, not a fully passing suite. The PUF +recipient test correctly refused an invalid invented Social Security observation: +the composed fixture had made a person age 14 without blanking that question. +The guard also denied a metadata lookup of its own error-output path during +failure formatting. Neither exception has been relabelled as a clean run. + +The fixture correction blanks that exact person's income literals before the +archive, catalogue, source pins and real preparation are constructed. It changes +no production source validator. Separate source/base/cold/required fixtures let +the focused PUF handoff test execute one real cold graph without repeating the +eight accepted controls. Independent review confirms their test-function bodies +and both production files are unchanged. The targeted handoff test then passed +through the actual source qualifier and both recipient graph declarations in +467.14 seconds, peaking at 545,980,416 bytes RSS. All 1,022 source and 15 resource +hashes remained identical, with no unexpected access refusals or child processes. +Its receipt SHA256 is +`9cd769de79b72a6e68069dc6c9f46db090d9cfe1a8e11d17e8d753a86bd5c388`. +Independent review approves the combined scope of eight earlier controls and +this corrected follow-up; it does not turn the earlier attempt into nine passes. + +## Verification and interpretation boundaries + +The runner checks actual training/model/raw-draw receipts against the qualified +donor and recipient before issuing either a cold or required-replay result. +Later checked views requalify the retained source owners and population contents +and require those verified artifact bytes to remain identical. They do not +perform another fit or replay the model applications. A descriptive copied +dataclass cannot acquire execution authority. + +The PUF host now derives its upstream node count and final writer from the +checked financial run. The new recipient test covers qualification and graph +dependencies; it does not execute the complete 261-node two-route PUF graph. +The separate ongoing native pilot uses its earlier frozen source and supplies +no acceptance for this new extension. + +Next steps are a separately verified tax-input rebase, explicit completion of +required unknown inputs, full PUF/enrichment execution, and candidate-specific +engine, calibration, geographic-support and quality checks. The public +[host contract](../docs/us-atomic-property-financial.md) and +[release plan](../docs/us-uk-release-path.md) describe those remaining boundaries. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py index d89986093..21de59046 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -1,8 +1,9 @@ """Checked atomic survey geography plus current financial development output. -This nineteen-node composition retains the raw allocation and the pre-financial -clone separately. It grants neither a budget successor nor PUF recipient or -release authority. Support bytes establish integrity, not publisher provenance. +The base nineteen nodes retain the raw allocation and the pre-financial clone +separately. An explicit property-income option adds sixteen nodes and retains +the complete legacy financial population alongside its extended output. Support +bytes establish integrity, not publisher provenance or release eligibility. """ from __future__ import annotations @@ -13,10 +14,13 @@ from dataclasses import dataclass, replace from types import FunctionType, SimpleNamespace +import numpy as np +import pandas as pd + from microcosm.fit import qrf_target from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel -from microcosm.graph import KernelResult, compile_graph, run_graph +from microcosm.graph import KernelResult, StructuralDelta, compile_graph, run_graph from microcosm.graph import population as population_ops from microcosm.graph.artifact_edges import typed_contracts from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys @@ -93,6 +97,17 @@ class _FinancialRunState: manifest_populations: tuple prefix_manifest_populations: tuple live: dict + property_income: object = None + property_income_bytes: bytes | None = None + legacy_financial_population: Population | None = None + legacy_financial_stamp: object = None + + +def _property_module(): + """Load the explicitly selected extension without changing the default path.""" + from . import graph_current_survey_property + + return graph_current_survey_property def _manifest_population_seals(manifest, compiled): @@ -123,6 +138,16 @@ def _run_entry(run): return entry +def financial_output_node(run): + """Name the final writer of an already issued financial run.""" + state = _run_entry(run)[2] + return ( + financial.ATTACH_NODE + if state.property_income is None + else _property_module().ATTACH_NODE + ) + + def _run_document(run, state): """Portable ancestry omits timing/cache hits and private physical seals.""" return codec.encode_json( @@ -145,10 +170,30 @@ def _run_document(run, state): (e, c, writer) for (e, c), writer in run.financial_population.owners.items() ), - "owned_columns": list(values.OUTPUTS), + "owned_columns": [ + *values.OUTPUTS, + *( + () + if state.property_income is None + else (owned.column for owned in _property_module().owned_columns()) + ), + ], "demographic_conditioning": state.demographic_conditioning, "n_estimators": state.n_estimators, "release_eligible": False, + **( + {} + if state.property_income is None + else { + "property_income": codec.decode_json(state.property_income_bytes), + "property_node_count": 16, + "tax_split_rebased": False, + "capital_gains_conditioning": _property_module().CAP_LIMITATION, + "legacy_financial_frame_sha256": values.source._frame_identity( + state.legacy_financial_population.frame + ), + } + ), } ) @@ -208,6 +253,14 @@ def _pure_run(run, entry): "FINANCIAL_RUN_SOURCE_CHANGED", ) values.source._pure_final(state.preparation_entry[2]) + if state.property_income is not None: + require( + type(state.property_income) is _property_module().PropertyIncomeOptions + and state.property_income.to_bytes() == state.property_income_bytes + and reconstruction._population_stamp(state.legacy_financial_population) + == state.legacy_financial_stamp, + "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED", + ) for name, population, stamp in state.populations: actual = ( run.financial_population if name == "financial" else getattr(prefix, name) @@ -217,7 +270,8 @@ def _pure_run(run, entry): "FINANCIAL_RUN_POPULATION_CHANGED", ) require( - _run_document(run, state) == entry[1] and _live() == state.live, + _run_document(run, state) == entry[1] + and _live(state.property_income) == state.live, "FINAL_FINANCIAL_RUN_SEAL", ) require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") @@ -256,7 +310,11 @@ def check_atomic_survey_financial_run(run): prefix.preparation, prefix.allocated_population, prefix.clone_population, - population=run.financial_population, + population=( + run.financial_population + if state.property_income is None + else state.legacy_financial_population + ), projection=state.projection, matrix=state.matrix, matrix_producer_key=keys[financial.PROJECTION_NODE], @@ -271,6 +329,20 @@ def check_atomic_survey_financial_run(run): demographic_conditioning=state.demographic_conditioning, geography_config=prefix.geography_config, ) + if state.property_income is not None: + _property_module().verify_materialized_property_income( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + legacy_population=state.legacy_financial_population, + population=run.financial_population, + host_pins=codec.decode_json(state.pins), + options=state.property_income, + artifacts=loaded, + legacy_matrix_producer_key=keys[financial.PROJECTION_NODE], + demographic_conditioning=state.demographic_conditioning, + geography_config=prefix.geography_config, + ) result = CheckedAtomicSurveyFinancialRun( entry[1], codec.sha(entry[1]), run.financial_population ) @@ -290,6 +362,8 @@ def _issue_run( implementations, loaded, live, + property_income=None, + legacy_financial_population=None, ): """Called only after this runner's complete materialization/replay checks.""" prefix = result.prefix @@ -350,6 +424,12 @@ def _issue_run( _manifest_population_seals(result.manifest, result.compiled), _manifest_population_seals(prefix.manifest, prefix.compiled), live, + property_income, + None if property_income is None else property_income.to_bytes(), + legacy_financial_population, + None + if legacy_financial_population is None + else reconstruction._population_stamp(legacy_financial_population), ) identifier = id(result) @@ -363,7 +443,7 @@ def forget(reference): _pure_run(result, _run_entry(result)) -def _live(): +def _live(property_income=None): """Pure final fence over this composition and its existing owner closure.""" result = dict(values.host.survey_budget._live()) modules = ( @@ -378,6 +458,40 @@ def _live(): sys.modules[LegacyQRFTrainKernel.__module__], sys.modules[LegacyQRFApplyMatrixKernel.__module__], ) + if property_income is not None: + from . import current_asec_property_basis, graph_property_income_receipts + + extension = _property_module() + modules = ( + *modules, + extension, + extension.sources, + current_asec_property_basis, + extension.sources.acs, + extension.sources.interest, + extension.sources.routing, + extension.sources.dividend, + extension.model, + extension.signed_graph, + graph_property_income_receipts, + sys.modules[extension.LegacyQRFApplyKernel.__module__], + ) + result["property_contract"] = values.source._runtime_marker( + ( + extension.PROTOCOL, + extension.PREFIX, + extension.CAP_LIMITATION, + extension.LEGACY_DIFFERENCE, + extension.BASIS_DIAGNOSTICS, + extension.PROPERTY_COMPONENTS, + extension.PROPERTY_DRAW_COLUMNS, + extension.PROPERTY_REPORTED_TOTAL, + extension.model.PROTOCOL, + extension.sources.PROTOCOL, + current_asec_property_basis.OTHER_PROPERTY_CATEGORIES, + current_asec_property_basis.OTHER_UNSPECIFIED_CATEGORY, + ) + ) for module in modules: for name, value in vars(module).items(): if isinstance(value, FunctionType): @@ -547,13 +661,23 @@ def run_atomic_survey_financial( geography_config, demographic_conditioning=False, n_estimators=100, + property_income=None, resume="auto", return_values=False, ): - """Execute and verify all nineteen nodes, including required cache replay.""" + """Verify the base financial graph and its explicitly selected extension.""" require(type(return_values) is bool, "RETURN_VALUES_FLAG") values.feature_columns(demographic_conditioning) - live = _live() + property_graph = None if property_income is None else _property_module() + if property_graph is not None: + require( + type(property_income) is property_graph.PropertyIncomeOptions, + "PROPERTY_OPTIONS_TYPE", + ) + property_income_bytes = property_income.to_bytes() + else: + property_income_bytes = None + live = _live(property_income) config_bytes = values.host.survey_budget._config_payload(geography_config) require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") prefix = atomic.run_atomic_survey_population( @@ -598,6 +722,17 @@ def run_atomic_survey_financial( geography_config=geography_config, ) projection_bytes, matrix_bytes = qualified.projection, qualified.matrix + property_qualified = ( + None + if property_graph is None + else property_graph.sources.qualify_current_property_income_sources( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + ) geography = reconstruction.reconstruct_atomic_survey_geography( prefix.preparation, prefix.allocated_population, geography_config ) @@ -621,11 +756,26 @@ def run_atomic_survey_financial( host_pins=pins, n_estimators=n_estimators, ) + property_nodes = ( + () + if property_graph is None + else property_graph.current_survey_property_nodes( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, + ) + ) compiled = compile_graph( - replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, *nodes)) + replace( + prefix.compiled.graph, + nodes=(*prefix.compiled.graph.nodes, *nodes, *property_nodes), + ) ) require( - len(prefix.compiled.order) == 9 and len(compiled.order) == 19, + len(prefix.compiled.order) == 9 + and len(property_nodes) == (0 if property_graph is None else 16) + and len(compiled.order) == 19 + len(property_nodes), "ATOMIC_COMPILER_ROSTER", ) gate_edge = financial._geography_edge() @@ -657,6 +807,17 @@ def run_atomic_survey_financial( ) kernels.register(LegacyQRFTrainKernel()) kernels.register(LegacyQRFApplyMatrixKernel()) + if property_graph is not None: + property_graph.register_property_kernels( + kernels, + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + options=property_income, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) _, source_keys = _source_paths_and_keys(compiled, sources, store) keys, implementations = _all_node_keys(compiled, kernels, source_keys) base_expected = { @@ -746,10 +907,40 @@ def observe(node_id, population): "paired_draws": "source_origin_join", "host_weights_changed": False, } + property_results = ( + {} + if property_graph is None + else property_graph.reconstruct_property_results( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + ) + ) + receipts.update({name: result.receipt for name, result in property_results.items()}) expected, current = {}, {} for node_id in compiled.order: node, version = compiled.graph.node(node_id), compiled.versions[node_id] - if node_id == donor_node.id: + if node_id in property_results: + incumbent = ( + version if node.structural is StructuralDelta.NONE else node.base + ) + property_result = property_results[node_id] + if node.structural is StructuralDelta.FILTER: + frame = current[incumbent].frame + entity = frame.schema.person_entity + id_column = frame.schema.entity_id_column(entity) + ids = pd.Index(frame.table(entity)[id_column], name=id_column) + mask = property_result.keep.reindex(ids).to_numpy( + dtype=np.bool_, copy=True + ) + property_result = replace( + property_result, frame=frame.select(mask), keep=None + ) + population = population_ops.patch(current[incumbent], node, property_result) + elif node_id == donor_node.id: population = donor elif node_id == columns_node.id: population = donor_columns @@ -769,14 +960,29 @@ def observe(node_id, population): population = current[version] expected[node_id] = current[version] = population atomic.same_replayed_population(population, observed[node_id]) + if property_graph is not None: + from .graph_property_income_receipts import verify_property_model_receipts + + receipts.update( + verify_property_model_receipts( + property_nodes, + expected[property_graph.DONOR_COLUMNS_NODE], + expected[property_graph.RECIPIENT_COLUMNS_NODE], + loaded, + ) + ) expected_stamps = { n: reconstruction._population_stamp(p) for n, p in expected.items() } states = atomic._states(compiled, kernels, source_keys, expected, receipts) survey._check_node_states(manifest, states) + final_node = ( + financial.ATTACH_NODE if property_graph is None else property_graph.ATTACH_NODE + ) + legacy_population = observed[financial.ATTACH_NODE] result = AtomicSurveyFinancialRunValues( prefix, - observed[financial.ATTACH_NODE], + observed[final_node], manifest, compiled, store, @@ -797,7 +1003,7 @@ def observe(node_id, population): prefix.preparation, prefix.allocated_population, prefix.clone_population, - population=result.financial_population, + population=legacy_population, projection=projection_bytes, matrix=matrix_bytes, matrix_producer_key=matrix_key, @@ -808,6 +1014,24 @@ def observe(node_id, population): demographic_conditioning=demographic_conditioning, geography_config=geography_config, ) + if property_graph is not None: + property_graph.verify_materialized_property_income( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + legacy_population=legacy_population, + population=result.financial_population, + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + require( + property_income.to_bytes() == property_income_bytes, + "PROPERTY_OPTIONS_CHANGED", + ) values.source._pure_final(entry[2]) require( values.source._ISSUED.get(id(prefix.preparation)) is entry @@ -819,7 +1043,7 @@ def observe(node_id, population): and result.compiled is compiled and result.store is store and result.kernels is kernels - and result.financial_population is observed[financial.ATTACH_NODE] + and result.financial_population is observed[final_node] and result.projection == projection_bytes and result.matrix == matrix_bytes and all( @@ -844,7 +1068,7 @@ def observe(node_id, population): and tuple(sorted(prefix.sources.items())) == source_items and graph_to_json(compiled.graph) == declaration and compiled == compile_graph(compiled.graph) - and _live() == live, + and _live(property_income) == live, "ATOMIC_FINAL_BINDINGS", ) for name, (population, stamp) in retained.items(): @@ -889,5 +1113,9 @@ def observe(node_id, population): implementations=implementations, loaded=loaded, live=live, + property_income=property_income, + legacy_financial_population=None + if property_graph is None + else legacy_population, ) return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py index 4de2e1c52..bd02a95db 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py @@ -1,4 +1,4 @@ -"""Extend an issued postclone nineteen-node financial run with two PUF55 routes. +"""Extend an issued postclone financial run with two PUF55 routes. The existing financial run is an explicit prerequisite. This host compiles and executes the extension once; it does not claim that the upstream run's entire @@ -243,7 +243,7 @@ def _construct( ): """Construct actual declarations and bind actual source keys without running.""" financial.check_atomic_survey_financial_run(financial_run) - require(len(financial_run.compiled.order) == 19, "UPSTREAM_NINETEEN_NODES") + upstream_count = len(financial_run.compiled.order) require(type(seed) is int and 0 <= seed < 2**64, "SEED") require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") qualified = recipient.values.qualify_puf55_survey_recipients(financial_run) @@ -283,10 +283,13 @@ def _construct( nodes=nodes, ) compiled = compile_graph(graph) - require(len(compiled.order) == 25 + 110 * len(boundary.routes), "NODE_COUNT") + require( + len(compiled.order) == upstream_count + 6 + 110 * len(boundary.routes), + "NODE_COUNT", + ) order = compiled.order require( - order.index(financial.financial.ATTACH_NODE) + order.index(financial.financial_output_node(financial_run)) < order.index(recipient.PROJECTION_NODE) < order.index(recipient.MATRIX_NODE) < order.index(attach.FILTER_NODE) diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py new file mode 100644 index 000000000..f26803e21 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_financial.py @@ -0,0 +1,318 @@ +"""Issued 19/35-node financial graphs over invented source files and blocks.""" + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_asec_demographics import _demographic_arguments +from test_us_current_property_income_sources import source_arguments +from test_us_graph_atomic_survey_population import _support_payload +from test_us_survey_social_security import _social_security_arguments + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_current_survey_property as property_graph +from microcosm.build.us_runtime import graph_puf55_survey_recipients as puf_recipient +from microcosm.graph import compile_graph + + +@pytest.fixture(scope="module") +def property_host_source(tmp_path_factory): + import test_us_current_asec_demographics as demographic_fixture_module + import test_us_current_asec_income_routing as routing_fixture + import test_us_survey_population_preparation as survey_fixture_module + + from microcosm.build.us_runtime import current_asec_demographics as demographics + + root = tmp_path_factory.mktemp("atomic-property-financial") + with pytest.MonkeyPatch.context() as patch: + # Compose complete invented member bytes before the real source issuer + # runs. Only fixture constructors and their registry pins are replaced. + # The nesting order is original source -> Social Security -> demographics + # -> routing -> property. Each extension preserves prior columns and + # rebuilds the retained attachment; preparation is issued only afterward. + def social_security_fixture(path, monkeypatch, *, zero): + assert zero is False + return _social_security_arguments(path, monkeypatch, ambiguous=False) + + def demographic_fixture(path, monkeypatch): + return _demographic_arguments(path, monkeypatch, unknown=False, zero=False) + + patch.setattr(demographic_fixture_module, "fixture", social_security_fixture) + patch.setattr(routing_fixture, "fixture", demographic_fixture) + original_person = survey_fixture_module._person + + def person(*args, **kwargs): + row = original_person(*args, **kwargs) + # The property fixture makes this person age 14. Its other income + # questions must also be blank before the source archive is built. + if row["SERIALNO"] == "2024GQ0000001": + row.update(SSP="", SSIP="") + return row + + patch.setattr(survey_fixture_module, "_person", person) + arguments = source_arguments(root, patch) + patch.setattr( + demographics.demographic, + "_MEMBER_PINS", + property_graph.sources.routing.coverage._MEMBER_PINS, + ) + payload, source_ids = _support_payload() + support = root / "invented-block-support.npz" + support.write_bytes(payload) + config = runner.reconstruction.AtomicSurveyReconstruction( + support_path=str(support), + support_sha256=hashlib.sha256(payload).hexdigest(), + source_ids=tuple(sorted(source_ids.items())), + seed=17, + ) + options = property_graph.PropertyIncomeOptions( + scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2 + ) + call = { + **arguments, + "geography_config": config, + "demographic_conditioning": True, + "n_estimators": 2, + "return_values": True, + } + yield SimpleNamespace(root=root, call=call, options=options) + + +@pytest.fixture(scope="module") +def property_host_base(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=None, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_cold(property_host_source): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="auto" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def property_host_warm(property_host_source, property_host_cold): + case = property_host_source + run = runner.run_atomic_survey_financial( + **case.call, property_income=case.options, resume="require" + ) + yield run + runner.values.source.verify_survey_population_preparation(run.prefix.preparation) + + +@pytest.fixture(scope="module") +def actual_property_host(request, property_host_source): + # Explicit ordering demonstrates reuse of the unchanged 19-node prefix. + # A targeted PUF test requests only the cold host and need not rerun these + # separately accepted baseline/replay checks. + base = request.getfixturevalue("property_host_base") + cold = request.getfixturevalue("property_host_cold") + warm = request.getfixturevalue("property_host_warm") + (property_host_source.root / "host-acceptance.json").write_text( + json.dumps( + { + "scope": "invented source-issued financial host; no native or engine", + "base_nodes": len(base.compiled.order), + "extended_nodes": len(cold.compiled.order), + "cold_hits": sum(n.hit for n in cold.manifest.nodes.values()), + "required_hits": sum(n.hit for n in warm.manifest.nodes.values()), + "persons": cold.financial_population.frame.n("person"), + "households": cold.financial_population.frame.n("household"), + "owned_property_columns": [ + o.column for o in property_graph.owned_columns() + ], + "tax_split_rebased": False, + }, + indent=2, + ) + + "\n" + ) + return SimpleNamespace( + base=base, cold=cold, warm=warm, options=property_host_source.options + ) + + +def test_default_graph_and_payload_remain_nineteen_nodes(actual_property_host): + run = actual_property_host.base + document = runner.codec.decode_json(run.checked_view().payload) + assert len(run.compiled.order) == 19 + assert runner.financial_output_node(run) == runner.financial.ATTACH_NODE + assert document["owned_columns"] == list(runner.values.OUTPUTS) + assert ( + not {"property_income", "property_node_count", "tax_split_rebased"} + & document.keys() + ) + assert not any( + n.startswith(property_graph.PREFIX + ".") for n in run.compiled.order + ) + + +def test_actual_extended_host_cold_required_and_checked_issuance(actual_property_host): + case = actual_property_host + assert len(case.cold.compiled.order) == len(case.warm.compiled.order) == 35 + assert case.cold.manifest.key == case.warm.manifest.key + assert sum(n.hit for n in case.cold.manifest.nodes.values()) == 19 + assert all(n.hit for n in case.warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + case.cold.financial_population, case.warm.financial_population + ) + checked = [run.checked_view() for run in (case.cold, case.warm)] + assert checked[0].payload == checked[1].payload + document = runner.codec.decode_json(checked[0].payload) + assert document["property_income"] == runner.codec.decode_json( + case.options.to_bytes() + ) + assert document["property_node_count"] == 16 + assert document["tax_split_rebased"] is False + assert document["capital_gains_conditioning"] == property_graph.CAP_LIMITATION + assert document["release_eligible"] is False + assert runner.financial_output_node(case.cold) == property_graph.ATTACH_NODE + + +def test_extended_host_retains_complete_legacy_population(actual_property_host): + base, extended = actual_property_host.base, actual_property_host.cold + state = runner._run_entry(extended)[2] + runner.atomic.same_replayed_population( + base.financial_population, state.legacy_financial_population + ) + before, after = base.financial_population, extended.financial_population + for entity in before.frame.entities: + pd.testing.assert_frame_equal( + after.frame.table(entity)[before.frame.table(entity).columns], + before.frame.table(entity), + check_exact=True, + ) + assert after.version == before.version + assert after.frame.schema == before.frame.schema + assert after.frame.links == before.frame.links + assert after.frame.metadata == before.frame.metadata + assert after.mass_ledger == before.mass_ledger + assert after.frame.mass_log == before.frame.mass_log + pd.testing.assert_series_equal(after.frame.strata, before.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + after.design_weights[entity], before.design_weights[entity] + ) + np.testing.assert_array_equal( + after.frame.weights_for(entity).values, + before.frame.weights_for(entity).values, + ) + assert dict(after.owners) == { + **before.owners, + **{ + ("person", o.column): property_graph.ATTACH_NODE + for o in property_graph.owned_columns() + }, + } + + +def test_property_values_and_unknowns_follow_original_clone_identity( + actual_property_host, +): + run = actual_property_host.cold + people = run.financial_population.frame.person + source_id = runner.values.provenance.support_source_id_column("person") + columns = [o.column for o in property_graph.owned_columns()] + for _, pair in people.groupby(source_id, sort=False): + assert len(pair) == 2 + pd.testing.assert_series_equal( + pair.iloc[0][columns], pair.iloc[1][columns], check_names=False + ) + assert (~people.property_anchor_known).any() + assert ( + people.loc[ + ~people.property_anchor_known, property_graph.PROPERTY_REPORTED_TOTAL + ] + .isna() + .all() + ) + known = people.property_components_known + assert known.any() and (~known).any() + channel = runner.values.provenance.support_channel_column("person") + modeled = known & people[channel].astype(str).eq("acs") + assert modeled.any() + values = people.loc[modeled, list(property_graph.PROPERTY_COMPONENTS)] + np.testing.assert_allclose( + values.sum(axis=1), + people.loc[modeled, property_graph.PROPERTY_REPORTED_TOTAL], + atol=1e-10, + rtol=1e-12, + ) + assert (values.iloc[:, :3] >= 0).all().all() + assert (people.property_reported_total < 0).any() + + +@pytest.mark.parametrize("defect", ("options", "legacy", "final")) +def test_retained_host_lifetime_rejects_mutation(actual_property_host, defect): + run = actual_property_host.cold + state = runner._run_entry(run)[2] + if defect == "options": + original = state.property_income.atol + object.__setattr__(state.property_income, "atol", original + 1.0) + try: + with pytest.raises( + ValueError, match="PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + ): + run.checked_view() + finally: + object.__setattr__(state.property_income, "atol", original) + else: + population = ( + state.legacy_financial_population + if defect == "legacy" + else run.financial_population + ) + table = population.frame.person + column = ( + runner.values.OUTPUTS[0] + if defect == "legacy" + else property_graph.PROPERTY_REPORTED_TOTAL + ) + index = table.index[table[column].notna()][0] + original = table.loc[index, column] + table.loc[index, column] = original + 1.0 + try: + with pytest.raises( + ValueError, + match=( + "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED" + "|FINANCIAL_RUN_POPULATION_CHANGED" + "|FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED" + ), + ): + run.checked_view() + finally: + table.loc[index, column] = original + runner._pure_run(run, runner._run_entry(run)) + + +def test_detached_run_copy_has_no_issuer_authority(actual_property_host): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(actual_property_host.cold).checked_view() + + +def test_puf_recipient_reads_complete_final_property_writer(property_host_cold): + run = property_host_cold + qualified = puf_recipient.values.qualify_puf55_survey_recipients(run) + nodes = puf_recipient.puf55_survey_recipient_nodes(qualified) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *nodes)) + ) + assert len(compiled.order) == 37 + for node in nodes: + assert property_graph.ATTACH_NODE in compiled.predecessors[node.id] + inputs = next(s.columns for s in node.inputs if s.entity == "person") + assert {o.column for o in property_graph.owned_columns()}.issubset(inputs) + assert qualified.financial_run is run diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index cf5ca3849..cc2a397f7 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -149,6 +149,8 @@ "current_survey_health_source.py", # Qualify exact ACS/ASEC source rosters. "current_survey_health_coverage.py", # Validate the same two-clone join. "graph_current_survey_health.py", # Declare the attachment's identity inputs. + # Rejoin qualified original property values to both initial clones. + "graph_current_survey_property.py", "current_social_security_source.py", # Source-owned SS totals per row. "current_survey_geography.py", # Origin roster and draw keys. "current_survey_predictors.py", # Source-qualified predictor prep. @@ -399,12 +401,16 @@ "fiscal_leaf_policy.py", # Typed source, recode and attachment health graph fragment "graph_current_survey_health.py", + # Qualified original property branches and exact paired attachment + "graph_current_survey_property.py", # Grouped dense calibration over retained fiscal measurements "graph_fiscal_dense_calibration.py", # Measured fiscal inputs and exact retained population/target axes "graph_fiscal_measurement.py", # Source-blind weighted QRF chain, raw draws and signed reconciliation "graph_property_income.py", + # Source-blind fit/history and recipient-draw verification + "graph_property_income_receipts.py", # Source-blind PUF clone placement and original-channel preservation "graph_puf55_route_attachment.py", # Complete PUF composition and retained checked-output lifetime @@ -3473,6 +3479,9 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: # Static leaf/entity closure and engine metadata/default dictionaries, # not population provenance. Assumption execution remains disabled. "fiscal_leaf_policy.py", + # Declared numerical target columns and (node, artifact) byte maps; + # no original-source routing or provenance selectors. + "graph_property_income_receipts.py", # Reviewed 2026-09-12: exact geography fields/CSR masks, typed artifact # names and producer keys, retained registry entries and node histories. # These four remain scanned for provenance columns and accessor calls. From d91662e2c56b6074bf9648a6eecaad68ba2d9371 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:12:15 -0400 Subject: [PATCH 180/255] Qualify current ASEC retirement source details --- ...ent-asec-retirement-detail-source.added.md | 1 + ...s-current-asec-retirement-detail-source.md | 101 +++ .../current_asec_retirement_detail_source.py | 600 ++++++++++++++++++ ...s_current_asec_retirement_detail_source.py | 402 ++++++++++++ .../tests/test_us_spine_blindness.py | 4 + 5 files changed, 1108 insertions(+) create mode 100644 changelog.d/us-current-asec-retirement-detail-source.added.md create mode 100644 docs/us-current-asec-retirement-detail-source.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py diff --git a/changelog.d/us-current-asec-retirement-detail-source.added.md b/changelog.d/us-current-asec-retirement-detail-source.added.md new file mode 100644 index 000000000..799cbc4b2 --- /dev/null +++ b/changelog.d/us-current-asec-retirement-detail-source.added.md @@ -0,0 +1 @@ +Qualify current ASEC pension, disability and survivor source details with exact original-person joins, preserved unknownness and unallocated aggregate comparisons. New literal fields remain separate from retained money; no retirement regularity, taxability or ACS component mapping is inferred. diff --git a/docs/us-current-asec-retirement-detail-source.md b/docs/us-current-asec-retirement-detail-source.md new file mode 100644 index 000000000..bfc752d32 --- /dev/null +++ b/docs/us-current-asec-retirement-detail-source.md @@ -0,0 +1,101 @@ +# Current ASEC retirement details + +`current_asec_retirement_detail_source` qualifies pension, disability and +survivor source details from the current ASEC member owned by an authenticated +survey preparation. This is source evidence for a later ACS retirement bridge. +It produces no model, tax leaf, payment-frequency classification or new issuer. + +## Source and field ownership + +The source is the [2025 ASEC public dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +SHA256 `5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`, +for calendar-year 2024 income. Page references below are physical PDF pages; +the corresponding printed pages are 6C-23 through 6C-39. + +| Amount fields | Qualification | +| --- | --- | +| `PNSN_VAL`, `ANN_VAL`, `DST_VAL1/2`, `DST_VAL1_YNG/2_YNG` | Reuse immutable printed domains from the existing income-routing owner. Compare exact retained current-money values and validity. They remain reference observations here; their canonical receipt/account interpretation stays with that owner. | +| `DIS_VAL1/2` | Explicit dictionary domains on page 44, also compared against actual retained current-money values and validity. | +| `PEN_VAL1/2`, `SUR_VAL1/2`, `DSAB_VAL`, `DBTN_VAL`, `SRVS_VAL` | Independently qualify literals from the exact authenticated source member using published width, range, universe and zero semantics. These seven fields are absent from the current retained-money roster; the qualifier never fabricates a `ready.field()` for them. | + +The live money domains must agree with the eight existing fields. A future +money roster containing one of the seven independent fields fails until the +corresponding retained comparison is reviewed. The amount-entry cache contains +immutable entries; each public call returns a detached mapping. + +The complete source join verifies native person keys, household IDs, person +line numbers and source ages. Selected original ASEC persons retain their +actual composed person IDs. No clone or PUF population is modified. The source +literal table covers the complete current source roster; the person table +covers selected ASEC persons. + +## Knownness and comparisons + +`PEN_YN`, `DIS_YN` and `SUR_YN` are separate from their two source codes and +amounts (pages 44–45, 47 and 49–50). A positive readable amount is canonical +only with a valid positive source code and yes receipt. A no receipt, NIU source +slot and zero amount yields explicitly derived known nonreceipt. A yes receipt +with a declared source and zero amount stays ambiguous. A yes receipt with a +zero source slot and amount stays unreported, not an observed analytical zero. +Missing, malformed, out-of-range and contradictory values stay distinct. + +People under 15 never acquire analytical zeros. Unreadable outside-universe +answers stay unresolved; readable nonzero outside-universe evidence is flagged +as contradictory. `DIS_CS` and `DIS_HP` are preserved source answers. The latter +includes limited work as well as prevented work and does not establish the +ACS disability-pension criterion. + +Four raw numerical comparisons retain total-minus-slot differences: + +- `PNSN_VAL - PEN_VAL1 - PEN_VAL2`; +- `DSAB_VAL - DIS_VAL1 - DIS_VAL2`; +- `SRVS_VAL - SUR_VAL1 - SUR_VAL2`; +- `DBTN_VAL - DST_VAL1 - DST_VAL2`. + +These comparisons include readable published none/NIU zeros; they are not +proof of analytical completeness. Missing or malformed components produce an +unknown comparison. No difference is allocated to a pension, taxable income +or missing source. `DBTN_VAL` is retirement distributions, not disability, and +its printed formula uses the main slots rather than young-person slots. + +`SRVS_VAL` includes edited sources 1/2 and unedited sources 3/4 (page 49). +Its visible pair is therefore never declared exhaustive. Source labels retain +Railroad Retirement, estate/trust, annuity and unspecified categories without +assigning them to an ACS aggregate. In particular, positive survivor receipt +with two visible non-property routes does not prove the absence of additional +estate/trust income. + +Allocation and top-code flags preserve their own published domains. Source +allocation flags with values 0/1/9 do not acquire all intermediate header-range +values. `I_SURVL2` retains its printed `SURV_VAL2` universe spelling. Allocation +does not invalidate an otherwise observed amount; a disclosure flag does not +explain a particular component discrepancy by itself. + +## Why the ACS bridge remains separate + +The [2024 ACS questionnaire, question 43g](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +asks about regular account withdrawals, while [ASEC Q98Ar](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf#page=205) +asks about any withdrawal/distribution. “Regular IRA” is an account label, not +payment frequency. Account type alone does not establish taxability. The +qualifier assigns neither regularity nor tax treatment and makes no ACS +retirement-component allocation. + +## API and validation boundary + +`project_retirement_detail_literals(DataFrame)` is a descriptive pure +projection. `qualify_current_asec_retirement_detail(preparation)` requires the +actual preparation and original source ownership, captures the pinned member, +checks source identity and retained money, and requalifies the parent after +the last source I/O. Its last checks include actual owner identity and a full +physical result seal. + +`retirement_detail_values_seal(values)` covers table axes, dtypes, exact values, +nullable Float64 backing storage and masks, literals and detached evidence. +The returned frozen dataclass contains mutable descriptive tables and grants no +source admission. A consuming host retains/revalidates the actual parent and +the result at its own boundaries; a copied dataclass is not an issuer. + +Tests use only invented records and the real source-preparation path. They +cover source joins, retained versus independent fields, member-byte drift, +late parent mutation and result mutation beneath null masks. No native source +or country engine is needed, and passing tests do not certify a release. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py new file mode 100644 index 000000000..a83dcfa13 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py @@ -0,0 +1,600 @@ +"""Qualified ASEC pension, disability and survivor source details. + +Published amounts and component comparisons are observations, not a complete +ACS retirement bridge. No regularity, taxability, overlap allocation or source +admission is inferred. A host retains and requalifies the actual preparation. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import tempfile +from dataclasses import asdict, dataclass +from functools import lru_cache +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as physical +from . import current_asec_dividend_source as dividend +from . import current_asec_income_routing_source as routing + +PROTOCOL = "microcosm.us.current-asec-retirement-detail-source.v1" +REFERENCE_FIELDS = ( + "PNSN_VAL", + "ANN_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", +) +# Exact dictionary entries not supplied by the routing owner's domain mapping. +# Name, width, position, physical PDF page, printed page, maximum, universe. +DETAIL_AMOUNT_ENTRIES = ( + ("PEN_VAL1", 6, 558, 47, "6C-26", 999999, "PEN_SC1 > 0"), + ("PEN_VAL2", 6, 564, 47, "6C-26", 999999, "PEN_SC2 > 0"), + ("DIS_VAL1", 6, 465, 44, "6C-23", 999999, "DIS_SC1 > 0"), + ("DIS_VAL2", 6, 471, 44, "6C-23", 999999, "DIS_SC2 > 0"), + ("SUR_VAL1", 6, 652, 50, "6C-29", 999999, "SUR_YN = 1"), + ("SUR_VAL2", 6, 658, 50, "6C-29", 999999, "SUR_YN = 1"), + ("DSAB_VAL", 6, 485, 45, "6C-24", 999999, "DIS_VAL1 > 0 OR DIS_VAL2 > 0"), + ("DBTN_VAL", 7, 452, 44, "6C-23", 9999999, "DST_VAL1 > 0 OR DST_VAL2 > 0"), + ("SRVS_VAL", 6, 628, 49, "6C-28", 999999, "SUR_YN = 1"), +) +AMOUNT_FIELDS = (*REFERENCE_FIELDS, *(entry[0] for entry in DETAIL_AMOUNT_ENTRIES)) +RETAINED_MONEY_FIELDS = (*REFERENCE_FIELDS, "DIS_VAL1", "DIS_VAL2") +INDEPENDENT_LITERAL_FIELDS = tuple( + n for n in AMOUNT_FIELDS if n not in RETAINED_MONEY_FIELDS +) +RECEIPT_ENTRIES = MappingProxyType( + { + "PEN_YN": routing.RECEIPT_ENTRIES["PEN_YN"], + "SUR_YN": routing.ReceiptEntry(*dividend.RECEIPT_ENTRIES["SUR_YN"]), + "DIS_YN": routing.ReceiptEntry( + 1, 477, 45, "6C-24", "All Persons aged 15+", "niu" + ), + "DIS_CS": routing.ReceiptEntry( + 1, 459, 44, "6C-23", "All Persons aged 15+", "niu" + ), + "DIS_HP": routing.ReceiptEntry( + 1, 460, 44, "6C-23", "All Persons aged 15+", "niu" + ), + } +) +PENSION_CODES = MappingProxyType( + { + 0: "niu", + 1: "Company pension", + 2: "Union pension", + 3: "Federal government pension", + 4: "State government pension", + 5: "Local government pension", + 6: "US Military pension", + 7: "US Railroad Retirement", + 8: "Other", + } +) +DISABILITY_CODES = MappingProxyType( + { + 0: "NIU", + 1: "worker's compensation", + 2: "company or union disability", + 3: "federal government disability", + 4: "US military retirement disability", + 5: "state or local government employee disability", + 6: "US railroad retirement disability", + 7: "accident or disability insurance", + 8: "blacklung miners disability", + 9: "state temporary sickness", + 10: "other or don't know", + } +) +# Source code width, position, physical page, printed page, printed universe. +SOURCE_ENTRIES = MappingProxyType( + { + "PEN_SC1": (1, 556, 47, "6C-26", "PEN_YN = 1"), + "PEN_SC2": (1, 557, 47, "6C-26", "PEN_VAL2 > 0"), + "DIS_SC1": (2, 461, 44, "6C-23", "DIS_YN = 1"), + "DIS_SC2": (2, 463, 44, "6C-23", "DIS_YN = 1"), + **dividend.SURVIVOR_ENTRIES, + } +) +SOURCE_CODES = MappingProxyType( + {"PEN": PENSION_CODES, "DIS": DISABILITY_CODES, "SUR": dividend.SURVIVOR_CODES} +) +# Position, physical page, printed page, universe, supported published values. +ALLOCATION_ENTRIES = ( + ("I_PENSC1", 850, 56, "6C-35", "PEN_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_PENSC2", 851, 56, "6C-35", "PEN_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_PENVAL1", 852, 57, "6C-36", "PEN_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_PENVAL2", 853, 57, "6C-36", "PEN_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISCS", 812, 54, "6C-33", "DIS_CS > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISHP", 813, 54, "6C-33", "DIS_HP > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISSC1", 814, 54, "6C-33", "DIS_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_DISSC2", 815, 54, "6C-33", "DIS_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_DISVL1", 816, 54, "6C-33", "DIS_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISVL2", 817, 54, "6C-33", "DIS_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_DISYN", 818, 54, "6C-33", "DIS_YN > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURSC1", 873, 58, "6C-37", "SUR_SC1 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_SURSC2", 874, 58, "6C-37", "SUR_SC2 > 0", routing.ALLOCATION_DSTSC_CODES), + ("I_SURVL1", 875, 58, "6C-37", "SUR_VAL1 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURVL2", 876, 58, "6C-37", "SURV_VAL2 > 0", routing.ALLOCATION_ANNVAL_CODES), + ("I_SURYN", 877, 58, "6C-37", "SUR_YN > 0", routing.ALLOCATION_ANNVAL_CODES), +) +TOPCODE_ENTRIES = ( + ("TPEN_VAL1", 913, 60, "6C-39", "PEN_VAL1 > 0"), + ("TPEN_VAL2", 914, 60, "6C-39", "PEN_VAL2 > 0"), + ("TDISVAL1", 903, 59, "6C-38", "DIS_VAL1 > 0"), + ("TDISVAL2", 904, 59, "6C-38", "DIS_VAL2 > 0"), + ("TSURVAL1", 670, 50, "6C-29", "SUR_VAL1 > 0"), + ("TSURVAL2", 671, 50, "6C-29", "SUR_VAL2 > 0"), +) +READ_COLUMNS = ( + *routing.COORDINATE_COLUMNS, + *AMOUNT_FIELDS, + *RECEIPT_ENTRIES, + *SOURCE_ENTRIES, + *(e[0] for e in ALLOCATION_ENTRIES), + *(e[0] for e in TOPCODE_ENTRIES), +) +COMPARISONS = ( + ("pension", "PNSN_VAL", "PEN_VAL1", "PEN_VAL2"), + ("disability", "DSAB_VAL", "DIS_VAL1", "DIS_VAL2"), + ("survivor", "SRVS_VAL", "SUR_VAL1", "SUR_VAL2"), + ("distribution", "DBTN_VAL", "DST_VAL1", "DST_VAL2"), +) + + +def require(condition, reason): + if not condition: + raise ValueError("ASEC_RETIREMENT_DETAIL_" + reason) + + +@lru_cache(maxsize=1) +def _cached_amount_entries(): + reference = routing.printed_amount_entries() + entries = [(name, reference[name]) for name in REFERENCE_FIELDS] + for ( + name, + width, + position, + page, + printed, + maximum, + universe, + ) in DETAIL_AMOUNT_ENTRIES: + entries.append( + ( + name, + routing.PrintedAmountEntry( + name, + width, + position, + printed, + page, + 0, + maximum, + universe, + "0 = none or niu; 1-" + str(maximum) + " = income amount", + False, + (), + "none_or_niu_not_distinguishable_from_amount_alone", + ), + ) + ) + require(len(entries) == len(set(n for n, _ in entries)), "DUPLICATE_AMOUNT") + return tuple(entries) + + +def amount_entries(): + """Detached mapping over immutable, source-pinned amount entries.""" + return dict(_cached_amount_entries()) + + +amount_entries.cache_clear = _cached_amount_entries.cache_clear + + +def _domain_agreement(ready): + entries = amount_entries() + routing._require_live_amount_domains( + ready, + { + name: ( + entries[name].encoded_minimum, + entries[name].encoded_maximum, + entries[name].zero_semantics, + ) + for name in RETAINED_MONEY_FIELDS + }, + ) + # A future expanded money roster must add real retained-bit comparisons; + # it may not silently leave those new fields on the independent path. + require( + not ( + set(INDEPENDENT_LITERAL_FIELDS) + & {f.name for f in ready.bindings.spec.fields} + ), + "UNREVIEWED_RETAINED_DOMAIN", + ) + + +def _amount_pair(token, entry): + require(type(token) is str and len(token) <= routing.TOKEN_MAX_CHARS, "TOKEN_BOUND") + if token == "": + return None, "missing" + pattern = ( + ("-?" if entry.encoded_minimum < 0 else "") + + r"[0-9]{1," + + str(entry.printed_length) + + "}" + ) + if re.fullmatch(pattern, token, re.ASCII) is None: + return None, "malformed" + value = int(token) + if not entry.encoded_minimum <= value <= entry.encoded_maximum: + return None, "outside_printed_range" + return value, "in_printed_range" + + +def _slot_status(age, receipt, source_code, amount): + value, status = amount + if status not in ("missing", "in_printed_range"): + return "invalid_amount_literal" + kind = "missing" if value is None else "zero" if value == 0 else "nonzero" + baseline = routing.receipt_status( + bool(age >= 15), receipt, kind, net_measure=False, zero_is_dollars=False + ) + if age < 15: + if ( + receipt[1] != "in_printed_range" + or source_code[1] != "in_printed_range" + or value is None + ): + return "unresolved_outside_reporting_universe" + return ( + "outside_reporting_universe" + if receipt[0] == source_code[0] == 0 and value == 0 + else "contradictory_outside_reporting_universe" + ) + if receipt[1] != "in_printed_range" or value is None: + return baseline + code, code_status = source_code + if code_status != "in_printed_range": + return "unresolved_source_slot" + if receipt[0] == 1 and code == 0: + return "unreported_source_slot" if value == 0 else "contradictory_source_slot" + if receipt[0] in (0, 2) and code != 0: + return "contradictory_source_slot" + return baseline + + +def project_retirement_detail_literals(ordered): + """Preserve literals/knownness; aggregate comparisons do not allocate dollars.""" + require( + type(ordered) is pd.DataFrame + and ordered.columns.is_unique + and set(READ_COLUMNS) <= set(ordered), + "COLUMNS", + ) + age_pairs = [routing.literal_code(t, range(100), width=2) for t in ordered.A_AGE] + require(all(s == "in_printed_range" for _, s in age_pairs), "AGE_LITERAL") + ages = np.array([v for v, _ in age_pairs], dtype=np.int64) + out = pd.DataFrame(index=range(len(ordered))) + parsed, codes = {}, {} + for name, entry in amount_entries().items(): + pairs = [_amount_pair(t, entry) for t in ordered[name]] + parsed[name] = pairs + out[name + "_literal"] = pd.array(ordered[name].tolist(), dtype="string") + out[name + "_literal_status"] = pd.array([s for _, s in pairs], dtype="string") + out[name + "_published_amount"] = pd.array( + [v for v, _ in pairs], dtype="Float64" + ) + for name, entry in RECEIPT_ENTRIES.items(): + frame, values, statuses = routing._codes_frame( + name, + ordered[name], + routing.RECEIPT_CODE_DOMAIN, + {0: entry.zero_label_as_printed, 1: "yes", 2: "no"}, + width=entry.printed_length, + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for name, entry in SOURCE_ENTRIES.items(): + domain = SOURCE_CODES[name[:3]] + frame, values, statuses = routing._codes_frame( + name, ordered[name], domain, domain, width=entry[0] + ) + out = pd.concat([out, frame], axis=1) + codes[name] = list(zip(values, statuses, strict=True)) + for family in SOURCE_CODES: + for slot in (1, 2): + name = family + "_VAL" + str(slot) + labels = [ + _slot_status(age, receipt, code, amount) + for age, receipt, code, amount in zip( + ages, + codes[family + "_YN"], + codes[family + "_SC" + str(slot)], + parsed[name], + strict=True, + ) + ] + known = np.array( + [s in routing.KNOWN_AMOUNT_STATUSES for s in labels], dtype=bool + ) + out[name + "_reporting_status"] = pd.array(labels, dtype="string") + out[name + "_amount_known"] = known + out[name + "_amount"] = out[name + "_published_amount"].where(known) + for prefix, total, first, second in COMPARISONS: + suffix = "main_slots" if prefix == "distribution" else "visible_slots" + # Only a difference of readable published numbers, including raw NIU + # zeros. A zero difference is not analytic completeness or absence. + out[prefix + "_total_minus_" + suffix] = ( + out[total + "_published_amount"] + - out[first + "_published_amount"] + - out[second + "_published_amount"] + ) + for name, _, _, _, _, allowed in ALLOCATION_ENTRIES: + frame, _, _ = routing._codes_frame(name, ordered[name], allowed, width=1) + out = pd.concat([out, frame], axis=1) + for name, *_ in TOPCODE_ENTRIES: + frame, _, _ = routing._codes_frame(name, ordered[name], (0, 1), width=1) + out = pd.concat([out, frame], axis=1) + out["source_age"] = ages + out["survivor_visible_slots_exhaustive"] = False + out["taxability_assigned"] = False + out["regularity_assigned"] = False + out["acs_retirement_component_assigned"] = False + return out + + +@dataclass(frozen=True) +class CurrentAsecRetirementDetailValues: + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def retirement_detail_values_seal(values): + """Physical value seal, including nullable backing storage and exact bits.""" + require(type(values) is CurrentAsecRetirementDetailValues, "VALUES_TYPE") + digest = hashlib.sha256(PROTOCOL.encode()) + for table in (values.person, values.asec_literals): + require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE_TYPE") + digest.update( + physical._json( + { + "columns": list(table.columns), + "columns_axis": physical.checkpoint._index_spec( + table.columns, label="retirement detail columns" + ), + "index": physical.checkpoint._index_spec( + table.index, label="retirement detail" + ), + } + ) + ) + physical._series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + series = table[column] + if isinstance(series.dtype, pd.Float64Dtype): + # The checkpoint digest supports NumPy floats, but deliberately + # excludes Float64 extension arrays. Keep the nullable tag and + # hash both physical arrays, including values under null masks. + digest.update(physical._json({"dtype": "Float64", "nullable": True})) + physical._series_digest(digest, pd.Series(series.array._data)) + physical._series_digest(digest, pd.Series(series.array._mask)) + else: + physical._series_digest(digest, series) + digest.update(physical._json(values.evidence)) + return digest.hexdigest() + + +def _capture_member(root, pin): + _, member, _, digest, rows, size = pin + with tempfile.TemporaryDirectory(prefix="microcosm-asec-retirement-detail-") as tmp: + path = Path(tmp) / member + identity = routing.coverage._capture( + root / "asec" / member, + path, + size=size, + digest=digest, + budget=[routing.coverage._BODY_MAX], + ) + raw = routing._read_capture(path, rows=rows, columns=READ_COLUMNS, patterns={}) + require( + routing.coverage._identity(path.stat(follow_symlinks=False)) == identity + and routing._file_sha(path) == digest, + "CAPTURE_CHANGED", + ) + return raw + + +def _compare_amount(ready, positions, name, literals): + field = ready.field(name) + entry = amount_entries()[name] + pairs = [_amount_pair(t, entry) for t in literals] + require( + all(s in ("missing", "in_printed_range") for _, s in pairs), "AMOUNT_LITERAL" + ) + valid = field.validity[positions] == 1 + require(np.array_equal(valid, [v is not None for v, _ in pairs]), "AMOUNT_VALIDITY") + expected = np.array( + [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 + ) + require( + np.array_equal( + field.amounts[positions][valid].view("uint64"), + expected[valid].view("uint64"), + ), + "AMOUNT_BITS", + ) + return field + + +def qualify_current_asec_retirement_detail(preparation): + """Borrow the original preparation, capture once, and requalify before return.""" + source = routing.source + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state, native = entry[2], entry[2].native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + _domain_agreement(ready) + header, native_document = json.loads(ready.header), json.loads(issued[1]) + require( + header["target_year"] == routing.CURRENT_INCOME_YEAR + and header["semantic"] == "annual_current_money" + and native_document["source_year"] + == native_document["income_year"] + == routing.CURRENT_INCOME_YEAR + and native_document["survey_year"] == (routing.CURRENT_INCOME_YEAR + 1), + "PERIOD", + ) + pins = [ + p for p in routing.coverage._MEMBER_PINS if p[0] == routing.CURRENT_INCOME_YEAR + ] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "MEMBER_BINDING", + ) + raw = _capture_member(state.root, pins[0]) + positions = np.flatnonzero( + np.asarray(parent.scope.person_years) == routing.CURRENT_INCOME_YEAR + ) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent.frame.person.iloc[positions][parent_name].to_numpy(), + ), + "PARENT_COORDINATE", + ) + fields = { + name: _compare_amount(ready, positions, name, ordered[name]) + for name in RETAINED_MONEY_FIELDS + } + basis = project_retirement_detail_literals(ordered) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + for field_name, field in fields.items(): + for name in ("statuses", "validity", "zero_origin"): + basis[field_name + "_parent_" + name] = getattr(field, name)[positions] + selected = state.frame.person.loc[ + state.frame.person[routing.support_channel_column("person")].eq("asec") + ] + native_ids = selected[routing.spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + literals = ordered.copy() + literals.index = basis.index.copy() + evidence = { + "protocol": PROTOCOL, + "dictionary_url": routing.DICTIONARY_URL, + "dictionary_sha256": routing.DICTIONARY_SHA256, + "amount_entries": { + name: asdict(entry) for name, entry in amount_entries().items() + }, + "receipt_entries": { + name: entry._asdict() for name, entry in RECEIPT_ENTRIES.items() + }, + "source_entries": dict(SOURCE_ENTRIES), + "source_codes": {name: dict(values) for name, values in SOURCE_CODES.items()}, + "allocation_entries": ALLOCATION_ENTRIES, + "allocation_entry_width": 1, + "allocation_header_range": "0:9; supported value domains retained separately", + "topcode_entries": TOPCODE_ENTRIES, + "topcode_entry_width": 1, + "retained_money_fields": RETAINED_MONEY_FIELDS, + "independently_qualified_literal_fields": INDEPENDENT_LITERAL_FIELDS, + "comparison_scope": "raw readable published amounts, including none/NIU zeros; not a decomposition", + "survivor_total_scope": "SUR_VAL1/2 edited amounts plus unedited sources 3 and 4", + "distribution_total_scope": "printed DBTN_VAL formula uses DST_VAL1 + DST_VAL2, not YNG slots", + "annuity_distribution_reference_scope": "raw retained observations only; canonical receipt/route interpretation remains with income routing", + "universe_notes": [ + "I_SURVL2 prints SURV_VAL2; preserve spelling rather than invent a compiler predicate", + "DIS_CS describes leaving work for health; DIS_HP also includes limited work", + "Regular IRA is an account type, not evidence of regular withdrawals", + ], + "preparation_sha256": routing._sha(entry[1]), + "asec_native_sha256": routing._sha(issued[1]), + "money_header_sha256": routing._sha(ready.header), + "source_member_sha256": digest, + "source_year": routing.CURRENT_INCOME_YEAR, + "survey_year": (routing.CURRENT_INCOME_YEAR + 1), + "complete_source_rows": rows, + "selected_rows": len(out), + "read_columns": READ_COLUMNS, + "taxability_assigned": False, + "regularity_assigned": False, + "acs_retirement_component_assigned": False, + "aggregate_discrepancy_allocated": False, + "under15_completed_with_zero": False, + "source_admission_issued": False, + "release_eligible": False, + } + # Freeze detached JSON-compatible metadata before its seal; do not retain the + # module's mutable constant dictionaries inside a returned descriptive view. + result = CurrentAsecRetirementDetailValues( + out, literals, json.loads(json.dumps(evidence)) + ) + seal = retirement_detail_values_seal(result) + require( + preparation._checked() is entry and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require(retirement_detail_values_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py new file mode 100644 index 000000000..4fad40d07 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py @@ -0,0 +1,402 @@ +"""Retirement detail source observations; no tax or regularity model.""" + +import copy +import hashlib +import importlib +import shutil +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_detail_source" + ) + + +def row(owner, **changes): + return { + **{name: "0" for name in owner.READ_COLUMNS}, + "PERIDNUM": "0000000000000000000001", + "PH_SEQ": "1", + "A_LINENO": "1", + "A_AGE": "40", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "6", + "DIS_VAL1": "200", + "DSAB_VAL": "200", + "SUR_YN": "1", + "SUR_SC1": "8", + "SUR_VAL1": "300", + "SRVS_VAL": "450", + **changes, + } + + +def project(owner, **changes): + return owner.project_retirement_detail_literals( + pd.DataFrame([row(owner, **changes)]) + ).iloc[0] + + +def test_source_components_and_extra_survivor_total_remain_distinct(owner): + value = project(owner) + assert value.PEN_VAL1_amount == 100 + assert value.DIS_VAL1_amount == 200 + assert value.SUR_VAL1_amount == 300 + assert value.SRVS_VAL_published_amount == 450 + assert value.survivor_total_minus_visible_slots == 150 + assert not value.survivor_visible_slots_exhaustive + assert not value.taxability_assigned + assert not value.regularity_assigned + assert not value.acs_retirement_component_assigned + + +@pytest.mark.parametrize( + "receipt,code,amount,status,known", + [ + ("1", "1", "100", "known_receipt", True), + ("1", "1", "0", "ambiguous_recipient_zero", False), + ("1", "0", "0", "unreported_source_slot", False), + ("1", "0", "100", "contradictory_source_slot", False), + ("2", "0", "0", "known_nonreceipt", True), + ("2", "1", "0", "contradictory_source_slot", False), + ("2", "0", "100", "contradictory_no_nonzero", False), + ("0", "0", "0", "niu", False), + ("", "1", "100", "missing_receipt_literal", False), + ("1", "", "100", "unresolved_source_slot", False), + ], +) +@pytest.mark.parametrize("family", ["PEN", "DIS", "SUR"]) +def test_slot_knownness_requires_receipt_and_source( + owner, family, receipt, code, amount, status, known +): + value = project( + owner, + **{family + "_YN": receipt, family + "_SC1": code, family + "_VAL1": amount}, + ) + assert value[family + "_VAL1_reporting_status"] == status + assert bool(value[family + "_VAL1_amount_known"]) is known + + +@pytest.mark.parametrize("age", ["0", "14"]) +def test_under15_has_no_analytical_zero(owner, age): + value = project(owner, A_AGE=age, PEN_YN="0", PEN_SC1="0", PEN_VAL1="0") + assert value.PEN_VAL1_reporting_status == "outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +@pytest.mark.parametrize( + "token,status", + [ + ("", "missing"), + ("-1", "malformed"), + ("1.2", "malformed"), + ("1000000", "malformed"), + ("NaN", "malformed"), + ], +) +def test_new_amount_literal_domains_are_strict(owner, token, status): + value = project(owner, SUR_VAL1=token) + assert value.SUR_VAL1_literal == token + assert value.SUR_VAL1_literal_status == status + assert not value.SUR_VAL1_amount_known + + +def test_public_amount_mapping_is_detached_and_values_immutable(owner): + entries = owner.amount_entries() + expected = entries["SUR_VAL1"] + entries["SUR_VAL1"] = entries["PEN_VAL1"] + assert owner.amount_entries()["SUR_VAL1"] == expected + + +def test_reference_ira_annuity_and_aggregate_do_not_acquire_new_meanings(owner): + value = project(owner, ANN_VAL="-1", DST_VAL1="100", DST_VAL2="50", DBTN_VAL="120") + assert value.ANN_VAL_published_amount == -1 + assert value.distribution_total_minus_main_slots == -30 + assert not value.regularity_assigned and not value.taxability_assigned + + +def test_full_physical_seal_covers_masked_backing_and_metadata(owner): + raw = pd.DataFrame([row(owner, SUR_VAL1="0")]) + result = owner.CurrentAsecRetirementDetailValues( + owner.project_retirement_detail_literals(raw), raw, {} + ) + seal = owner.retirement_detail_values_seal(result) + assert owner.retirement_detail_values_seal(copy.deepcopy(result)) == seal + for kind in ("masked_backing", "mask", "literal", "evidence"): + changed = copy.deepcopy(result) + if kind == "masked_backing": + changed.person.SUR_VAL1_amount.array._data[0] = 17.0 + elif kind == "mask": + changed.person.SUR_VAL1_amount.array._mask[0] = False + elif kind == "literal": + changed.asec_literals.loc[0, "SUR_VAL1"] = "1" + else: + changed.evidence["invented"] = True + assert owner.retirement_detail_values_seal(changed) != seal + + +@pytest.mark.parametrize("field", ["PEN_YN", "PEN_SC1", "PEN_VAL1"]) +def test_unreadable_under15_values_are_unresolved_not_contradictions(owner, field): + value = project(owner, **{"A_AGE": "14", field: ""}) + assert value.PEN_VAL1_reporting_status == "unresolved_outside_reporting_universe" + assert not value.PEN_VAL1_amount_known + + +def test_source_allocation_domains_do_not_inherit_full_header_range(owner): + value = project(owner, I_PENSC1="4", I_PENVAL1="4", TPEN_VAL1="1", I_SURVL2="") + assert value.I_PENSC1_literal_status == "outside_printed_range" + assert value.I_PENVAL1_literal_status == "in_printed_range" + assert value.I_SURVL2_literal_status == "missing" + assert value.PEN_VAL1_amount == 100 + assert value.TPEN_VAL1_code == 1 + assert ( + next(e for e in owner.ALLOCATION_ENTRIES if e[0] == "I_SURVL2")[4] + == "SURV_VAL2 > 0" + ) + + +@pytest.mark.parametrize("change", ["bound", "duplicate", "missing", "new_retained"]) +def test_live_money_roster_is_exact_for_existing_fields_only(owner, change): + entries = owner.amount_entries() + fields = [ + SimpleNamespace( + name=name, + entity="person", + grain="person", + column=name, + minimum=entries[name].encoded_minimum, + maximum=entries[name].encoded_maximum, + zero_semantics=entries[name].zero_semantics, + ) + for name in owner.RETAINED_MONEY_FIELDS + ] + owner._domain_agreement( + SimpleNamespace(bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields))) + ) + if change == "bound": + fields[0].maximum += 1 + elif change == "duplicate": + fields.append(fields[0]) + elif change == "missing": + fields.pop() + else: + fields.append(SimpleNamespace(name="SRVS_VAL")) + with pytest.raises(ValueError): + owner._domain_agreement( + SimpleNamespace( + bindings=SimpleNamespace(spec=SimpleNamespace(fields=fields)) + ) + ) + + +def retirement_arguments(owner, tmp_path, monkeypatch): + from test_us_asec_coverage_authentication import _changed_parent + from test_us_survey_population_preparation import fixture + + from microcosm.build.frame_checkpoint import load_frame_checkpoint + from microcosm.build.us_runtime import asec_person_income_source as restoration + + arguments = fixture(tmp_path, monkeypatch) + folder = arguments["source_dir"] / "asec" + parent_path, attachment = folder / "parent.h5", folder / "household-attachment.h5" + people = load_frame_checkpoint(parent_path).frame.person + ids = people.person_id.to_numpy() + zeros = { + name: "0" + for name in ( + *owner.AMOUNT_FIELDS, + *owner.RECEIPT_ENTRIES, + *owner.SOURCE_ENTRIES, + ) + } + literals = { + 105: row(owner, I_PENVAL1="4"), + 106: row(owner, **zeros, A_AGE="14"), + 107: row( + owner, + **{**zeros, "A_AGE": "70", "PEN_YN": "2", "DIS_YN": "2", "SUR_YN": "2"}, + ), + 108: row(owner, PEN_VAL1="0", SUR_VAL1="0"), + } + changes = {} + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + values = people[name].to_numpy(copy=True) + for person_id, literal in literals.items(): + values[ids == person_id] = int(literal[name]) + changes[name] = values + _changed_parent(parent_path, attachment, monkeypatch, changes) + updated = load_frame_checkpoint(parent_path).frame.person.set_index("PERIDNUM") + paths, pins = {}, [] + for year, member, archive, *_ in owner.routing.coverage._MEMBER_PINS: + path = folder / f"pppub{year - 1999}.csv" + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + for name in (*owner.RETAINED_MONEY_FIELDS, "A_AGE"): + raw[name] = [str(int(updated.loc[key, name])) for key in raw.PERIDNUM] + for name in ( + set(owner.READ_COLUMNS) + - set(owner.routing.COORDINATE_COLUMNS) + - set(owner.RETAINED_MONEY_FIELDS) + ): + raw[name] = [ + literals.get(int(updated.loc[key, "person_id"]), row(owner))[name] + for key in raw.PERIDNUM + ] + raw.iloc[::-1].to_csv(path, index=False) + payload = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(payload).hexdigest(), + len(raw), + len(payload), + ) + ) + paths[year] = path + for module in (owner.routing.coverage, restoration): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "retirement-restored-money" + restoration.restore_asec_person_income_source( + parent_path, attachment, member_paths=paths, output_dir=output + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, folder / "person-income-attachment.h5" + ) + return arguments + + +def prepared(owner, tmp_path, monkeypatch): + return owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch) + ) + + +def test_actual_owner_qualifies_new_literals_without_fabricating_money_fields( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + compare, compared = owner._compare_amount, [] + + def seen(ready, positions, name, literals): + compared.append(name) + assert name not in owner.INDEPENDENT_LITERAL_FIELDS + return compare(ready, positions, name, literals) + + monkeypatch.setattr(owner, "_compare_amount", seen) + result = owner.qualify_current_asec_retirement_detail(parent) + assert tuple(compared) == owner.RETAINED_MONEY_FIELDS + values = result.person.set_index("native_person_id") + assert values.loc[105, "PEN_VAL1_amount"] == 100 + assert values.loc[105, "survivor_total_minus_visible_slots"] == 150 + assert values.loc[107, "SUR_VAL1_amount"] == 0 + assert not values.loc[106, "SUR_VAL1_amount_known"] + assert not values.loc[108, "SUR_VAL1_amount_known"] + assert not result.evidence["source_admission_issued"] + assert not result.evidence["taxability_assigned"] + assert owner.retirement_detail_values_seal( + owner.qualify_current_asec_retirement_detail(parent) + ) == owner.retirement_detail_values_seal(result) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(copy.copy(parent)) + + +@pytest.mark.parametrize("field", ["PERIDNUM", "A_AGE", "DIS_VAL1"]) +def test_actual_owner_requires_retained_identity_and_money( + owner, tmp_path, monkeypatch, field +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(*args): + raw = capture(*args) + if field == "PERIDNUM": + raw.index = list(raw.index[:-1]) + ["9999999999999999999999"] + else: + raw.iloc[0, raw.columns.get_loc(field)] = "99" + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_pins_new_survivor_dollars_to_original_member( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture = owner._capture_member + + def changed(root, pin): + # Only the invented fixture is touched. Even an unretained money field + # must match the exact already admitted source-member bytes. + path = root / "asec" / pin[1] + raw = pd.read_csv(path, dtype=str, keep_default_na=False) + raw["SRVS_VAL"] = "999" + raw.to_csv(path, index=False) + return capture(root, pin) + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize( + "mutation", ["amount_bit", "masked_backing", "literal", "evidence"] +) +def test_actual_owner_final_seal_covers_returned_values( + owner, tmp_path, monkeypatch, mutation +): + parent = prepared(owner, tmp_path, monkeypatch) + seal, calls = owner.retirement_detail_values_seal, [] + + def changed(value): + before = seal(value) + if not calls: + calls.append(True) + array = value.person.SUR_VAL1_amount.array + if mutation == "amount_bit": + i = np.flatnonzero(~array._mask)[0] + array._data[i] = np.nextafter(array._data[i], np.inf) + elif mutation == "masked_backing": + array._data[np.flatnonzero(array._mask)[0]] = 17.0 + elif mutation == "literal": + value.asec_literals.iloc[ + 0, value.asec_literals.columns.get_loc("SUR_YN") + ] = "2" + else: + value.evidence["taxability_assigned"] = True + return before + + monkeypatch.setattr(owner, "retirement_detail_values_seal", changed) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_asec_retirement_detail(parent) + + +def test_actual_owner_requalifies_parent_after_last_capture( + owner, tmp_path, monkeypatch +): + parent = prepared(owner, tmp_path, monkeypatch) + capture, calls = owner._capture_member, [] + + def changed(*args): + raw = capture(*args) + table = parent._checked()[2].frame.person + table.iloc[0, table.columns.get_loc("age")] += 1 + calls.append(True) + return raw + + monkeypatch.setattr(owner, "_capture_member", changed) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + assert calls diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index cc2a397f7..76c7f5844 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -144,6 +144,8 @@ "current_asec_interest_source.py", # ASEC interest -> original money owner. # Composes qualified original source axes and DESIGN branches; no fit. "current_property_income_sources.py", + # Original ASEC pension/disability/survivor details; no population model. + "current_asec_retirement_detail_source.py", "current_asec_unemployment_source.py", # UC literal -> original ASEC ids. "current_survey_amounts.py", # Validate origin join; fan out to both clones. "current_survey_health_source.py", # Qualify exact ACS/ASEC source rosters. @@ -387,6 +389,8 @@ # Pure supplied-table property donor basis and exclusion diagnostics; # no source owner or source-spine access exemption "current_asec_property_basis.py", + # Qualified ASEC retirement details and raw aggregate/slot comparisons + "current_asec_retirement_detail_source.py", # Qualified UC receipt/amount observations, preserving unknown zeros "current_asec_unemployment_source.py", # Qualified original property donor/recipient branch composition From 7452c2635709be835282f9b6ce0bd7c89c6daede Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 12 Sep 2026 14:38:29 -0400 Subject: [PATCH 181/255] Preserve declared annuity NIU normalization in retirement qualification --- .../us-asec-retirement-annuity-niu.fixed.md | 1 + ...s-current-asec-retirement-detail-source.md | 7 +++ .../current_asec_retirement_detail_source.py | 14 +++++ ...s_current_asec_retirement_detail_source.py | 61 ++++++++++++++++++- 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 changelog.d/us-asec-retirement-annuity-niu.fixed.md diff --git a/changelog.d/us-asec-retirement-annuity-niu.fixed.md b/changelog.d/us-asec-retirement-annuity-niu.fixed.md new file mode 100644 index 000000000..0ddba37c7 --- /dev/null +++ b/changelog.d/us-asec-retirement-annuity-niu.fixed.md @@ -0,0 +1 @@ +Match published annuity NIU literals to the authenticated money owner's normalized zero and NIU status while preserving raw retirement source values. Reject mismatched dollar-zero and NIU encodings before qualification. diff --git a/docs/us-current-asec-retirement-detail-source.md b/docs/us-current-asec-retirement-detail-source.md index bfc752d32..e125201e4 100644 --- a/docs/us-current-asec-retirement-detail-source.md +++ b/docs/us-current-asec-retirement-detail-source.md @@ -89,6 +89,13 @@ checks source identity and retained money, and requalifies the parent after the last source I/O. Its last checks include actual owner identity and a full physical result seal. +The retained-money comparison follows the existing owner's annuity encoding: +published `ANN_VAL=-1` matches normalized positive zero with `DECLARED_NIU`, +while the descriptive literal and published amount remain `-1`. A dollar-zero +literal cannot substitute for that NIU/status pair. Other valid dollar amounts +retain exact float64 comparison. This distinction is tested through actual +invented source preparation, as well as direct mismatched-status controls. + `retirement_detail_values_seal(values)` covers table axes, dtypes, exact values, nullable Float64 backing storage and masks, literals and detached evidence. The returned frozen dataclass contains mutable descriptive tables and grants no diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py index a83dcfa13..1fa6292bd 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_detail_source.py @@ -428,6 +428,20 @@ def _compare_amount(ready, positions, name, literals): expected = np.array( [np.nan if v is None else v for v, _ in pairs], dtype=np.float64 ) + if name in REFERENCE_FIELDS: + routing.amount_observations(field, positions, literals) + # ANN's published -1 is retained as +0 with DECLARED_NIU by the money owner. + # Normalize only that declared non-dollar code for this comparison; the + # separate literal/published projection below keeps the original -1. + niu = valid & np.isin(expected, entry.nonmoney_codes) + require( + np.array_equal( + field.statuses[positions] == routing.money.CodebookStatus.DECLARED_NIU, + niu, + ), + "AMOUNT_NIU_STATUS", + ) + expected[niu] = 0.0 require( np.array_equal( field.amounts[positions][valid].view("uint64"), diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py index 4fad40d07..21f6fe1b5 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_detail_source.py @@ -199,7 +199,7 @@ def test_live_money_roster_is_exact_for_existing_fields_only(owner, change): ) -def retirement_arguments(owner, tmp_path, monkeypatch): +def retirement_arguments(owner, tmp_path, monkeypatch, *, annuity_niu=False): from test_us_asec_coverage_authentication import _changed_parent from test_us_survey_population_preparation import fixture @@ -220,7 +220,7 @@ def retirement_arguments(owner, tmp_path, monkeypatch): ) } literals = { - 105: row(owner, I_PENVAL1="4"), + 105: row(owner, I_PENVAL1="4", ANN_VAL="-1" if annuity_niu else "0"), 106: row(owner, **zeros, A_AGE="14"), 107: row( owner, @@ -282,6 +282,63 @@ def prepared(owner, tmp_path, monkeypatch): ) +def test_actual_annuity_niu_preserves_literal_and_normalized_owner( + owner, tmp_path, monkeypatch +): + parent = owner.routing.source.prepare_authenticated_survey_population( + **retirement_arguments(owner, tmp_path, monkeypatch, annuity_niu=True) + ) + result = owner.qualify_current_asec_retirement_detail(parent) + values = result.person.set_index("native_person_id") + money = owner.routing.money + assert values.loc[105, "ANN_VAL_literal"] == "-1" + assert values.loc[105, "ANN_VAL_published_amount"] == -1 + assert values.loc[105, "ANN_VAL_parent_validity"] == 1 + assert ( + values.loc[105, "ANN_VAL_parent_statuses"] == money.CodebookStatus.DECLARED_NIU + ) + assert values.loc[105, "ANN_VAL_parent_zero_origin"] == money.ZeroOrigin.NOT_ZERO + assert values.loc[107, "ANN_VAL_literal"] == "0" + assert ( + values.loc[107, "ANN_VAL_parent_statuses"] != money.CodebookStatus.DECLARED_NIU + ) + literals = result.asec_literals + assert literals.loc[literals.ANN_VAL.eq("-1")].shape[0] == 1 + + capture = owner._capture_member + + def substitute_dollar_zero(*args): + raw = capture(*args) + raw.loc[raw.ANN_VAL.eq("-1"), "ANN_VAL"] = "0" + return raw + + monkeypatch.setattr(owner, "_capture_member", substitute_dollar_zero) + with pytest.raises(ValueError): + owner.qualify_current_asec_retirement_detail(parent) + + +@pytest.mark.parametrize("literal,niu_status", [("-1", False), ("0", True)]) +def test_annuity_literal_and_owner_niu_status_must_agree(owner, literal, niu_status): + money = owner.routing.money + status = ( + money.CodebookStatus.DECLARED_NIU + if niu_status + else money.CodebookStatus.ZERO_NONE_OR_NIU + ) + field = money.MoneyField( + "ANN_VAL", + np.array([0.0], dtype=np.float64).tobytes(), + np.array([status], dtype="u1").tobytes(), + np.array([1], dtype="u1").tobytes(), + np.array([0], dtype="u1").tobytes(), + ) + ready = SimpleNamespace(field=lambda name: field) + with pytest.raises(ValueError): + owner._compare_amount( + ready, np.array([0], dtype=np.int64), "ANN_VAL", [literal] + ) + + def test_actual_owner_qualifies_new_literals_without_fabricating_money_fields( owner, tmp_path, monkeypatch ): From aaf31747af4a69ad124e78cf554de8badd374297 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:25:35 -0400 Subject: [PATCH 182/255] Add explicit UK household lineage projection for atomic geography --- .../uk-atomic-household-lineage.added.md | 1 + docs/uk-atomic-household-lineage.md | 28 + .../uk_runtime/atomic_household_lineage.py | 212 +++++++ .../tests/test_uk_atomic_household_lineage.py | 588 ++++++++++++++++++ 4 files changed, 829 insertions(+) create mode 100644 changelog.d/uk-atomic-household-lineage.added.md create mode 100644 docs/uk-atomic-household-lineage.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py create mode 100644 packages/microcosm-build/tests/test_uk_atomic_household_lineage.py diff --git a/changelog.d/uk-atomic-household-lineage.added.md b/changelog.d/uk-atomic-household-lineage.added.md new file mode 100644 index 000000000..b40a2a605 --- /dev/null +++ b/changelog.d/uk-atomic-household-lineage.added.md @@ -0,0 +1 @@ +Add a pure UK household-lineage adapter that derives stable atomic-geography keys from explicit source identities and selection/expansion records, preserving exact IDs and refusing ambiguous ancestry without granting source authority or changing the full-build host. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md new file mode 100644 index 000000000..067353f48 --- /dev/null +++ b/docs/uk-atomic-household-lineage.md @@ -0,0 +1,28 @@ +# UK household identity after cloning + +The shared atomic-area operators need a stable household key after initial cloning. `uk_runtime.atomic_household_lineage` projects supplied original-source identities through explicit selection and expansion records, using the existing `household_draw_key` encoding. It adds no assignment algorithm, data reader, issuance registry or full-build host. + +A key includes the declared source, source vintage, original household ID and ordered clone path. It excludes final offset IDs, row order, sample size, calibration weights and financial values. Exact int64 IDs above `2**53` remain distinct. Original households keep their previous path; a new child appends its explicitly supplied branch ordinal. An original branch does not gain an inferred zero ordinal. + +## Contract + +`project_atomic_household_keys(roots, *, steps, final_ids)` returns a detached two-column table: int64 `household_id` and nullable-string `geography_household_key`, in the requested final order. Roots have exactly `household_id`, `source`, `source_vintage` and `source_household_id`. An empty selection is a valid description, not evidence that an empty dataset is useful or releasable. + +Steps are immutable descriptions: + +- `HouseholdExpansion(branch, before_ids, after_ids, parent_pairs, child_ordinals)` carries exact household axes, the household part of the shared executor's `receipt["expand"]`, and explicit child ordinals. The existing ordered branches are SPI support, CGT incidence, CGT band donors and geographic support. Each arriving household needs one known preceding parent and one ordinal. Two children of the same parent cannot reuse an ordinal in the same branch. +- `HouseholdSelection(before_ids, after_ids)` explicitly removes households while preserving their keys. This supports the source-family sample between the completed source spine and geographic expansion. It does not check that the sample respected the host's family-sampling rule. + +Each before-axis must have exactly the previous step's household IDs. Expansion cannot remove incumbents; selection cannot add households. Unknown or null parents, repeated or reordered branches, incomplete parent/ordinal coverage, duplicate keys and a mismatched final axis refuse. Axis ordering may change, because ordering is not identity. The function never guesses ancestry from ID arithmetic, support flags or model values. + +## Host responsibilities + +These checks establish internal consistency of supplied descriptions. A dataclass, DataFrame or serialized receipt does not prove its source. The receiving host must retain and check the actual original source owner, complete before/after populations and source/EXPAND/FILTER evidence. It must authenticate the complete structural stage roster, the explicit branch-ordinal convention, and the exact final receiving household axis before and after relevant I/O. A coherent invented history can pass this pure helper; it is not an admission gate. + +The host can extract the household pairs from its verified shared EXPAND receipt without decoding household ID offsets. The helper returns the key column for the existing `uk_atomic_assignment_definition` and `atomic_geography_nodes` APIs. Support-byte provenance, observed-region constraints and versioned mapping relations remain separate responsibilities. Larger geographies continue to derive from the assigned atomic area. + +No production UK host is changed here. María's full-build graph currently uses its existing ladder draw after geographic expansion. Adopting this key and shared assignment must be coordinated in that host. The existing source stage order also interleaves enrichment before later SPI/CGT cloning; moving atomic assignment before every enrichment stage requires a separate staged migration. This helper does not claim that migration, native FRS qualification or a new UK release. + +## Verification scope + +Invented tests cover the full four-branch path, explicit selection, exact large IDs, source and ordinal changes, stable keys under row reordering and pool growth, unknown/colliding ancestry refusal and detached output. A real shared CREATE/EXPAND/FILTER graph supplies the household lineage receipt and replays from cache. Its projected keys then feed the existing seven-node atomic graph, including three system imports, assignment, derivation and its gate. That graph also completes required replay and preserves the supplied entity tables, weights, strata and metadata. No native source, private archive, country engine or external resource is used. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py new file mode 100644 index 000000000..dcca8ac98 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_lineage.py @@ -0,0 +1,212 @@ +"""Project supplied source/structural lineage to stable UK geography keys. + +This is a pure descriptive adapter, not a source owner or graph admission gate. +The host must bind roots, every before/after axis, EXPAND parent receipt and +explicit ordinal to its actual retained observations before and after use. +No identifier is reconstructed from offsets, weights, financial flags or order. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from numbers import Integral + +import numpy as np +import pandas as pd + +from . import atomic_household_identity as identity +from .atomic_area_support import IDENTITY_COLUMN + +_ROOT_COLUMNS = ("household_id", "source", "source_vintage", "source_household_id") + + +@dataclass(frozen=True) +class HouseholdExpansion: + """One checked EXPAND's household axis and explicitly supplied ordinals. + + parent_pairs is the immutable household portion of the shared executor's + ``receipt['expand']`` mapping. child_ordinals names each new household's + declared branch ordinal; its position in the receipt is not an ordinal. + Existing households retain their previous path without adding a zero step. + Constructing this object does not authenticate any of these claims. + """ + + branch: str + before_ids: tuple[int, ...] + after_ids: tuple[int, ...] + parent_pairs: tuple[tuple[int, int], ...] + child_ordinals: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class HouseholdSelection: + """A checked FILTER's explicit household axes; no new identity is created.""" + + before_ids: tuple[int, ...] + after_ids: tuple[int, ...] + + +def _require(condition, reason): + if not condition: + raise ValueError("UK atomic household lineage: " + reason) + + +def _integer(value, *, ordinal=False): + _require( + isinstance(value, Integral) and not isinstance(value, (bool, np.bool_)), + "exact integer required", + ) + value = int(value) + _require( + 0 <= value < 2**32 if ordinal else 0 < value < 2**63, + "ordinal or ID outside its exact declared range", + ) + return value + + +def _axis(values): + _require(type(values) is tuple, "axes must be immutable tuples") + ids = tuple(_integer(value) for value in values) + _require(len(ids) == len(set(ids)), "duplicate household ID") + return ids + + +def _pairs(values, *, ordinals=False): + _require(type(values) is tuple, "pairs must be an immutable tuple") + pairs = {} + for row in values: + _require(type(row) is tuple and len(row) == 2, "exact immutable pair required") + target = _integer(row[0]) + source = _integer(row[1], ordinal=ordinals) + _require(target not in pairs, "duplicate pair target") + pairs[target] = source + return pairs + + +def _roots(roots): + _require(type(roots) is pd.DataFrame, "a plain root description table is required") + _require( + roots.columns.is_unique + and set(roots.columns) == set(_ROOT_COLUMNS) + and len(roots) > 0, + "exact nonempty root description columns required", + ) + for column in ("household_id", "source_household_id"): + _require(roots[column].dtype == np.dtype("int64"), "root IDs must be int64") + # Only immutable validated scalar values survive this copy. pandas row + # labels, column ordering and physical row order do not enter the key. + copied = roots.loc[:, list(_ROOT_COLUMNS)].copy(deep=True) + result = {} + initial_keys = set() + for row in copied.itertuples(index=False, name=None): + household_id, source, vintage, source_id = row + household_id = _integer(household_id) + source_id = _integer(source_id) + _require(household_id not in result, "duplicate root household ID") + key = identity.household_draw_key( + source=source, + source_vintage=vintage, + source_household_id=source_id, + clone_path=(), + ) + _require(key not in initial_keys, "duplicate original source identity") + initial_keys.add(key) + result[household_id] = (source, vintage, source_id, ()) + return result + + +def project_atomic_household_keys( + roots: pd.DataFrame, + *, + steps: tuple[HouseholdExpansion | HouseholdSelection, ...], + final_ids: np.ndarray, +) -> pd.DataFrame: + """Return exact final household IDs and canonical geography keys. + + Membership of each before axis must equal the preceding result; physical + ordering may differ because it is not identity. Every added row must have + one known preceding parent and one explicit ordinal unique for that parent + in this branch. Branch stages must follow the existing UK branch ordering. + A selection may only remove IDs; unexplained arrivals or losses refuse. + Empty selection results are valid descriptions, not viable build evidence. + + The output follows final_ids and is detached. All supplied descriptions + remain unissued; this checks internal consistency, not source authenticity, + complete-population ancestry, a sampling rule or release eligibility. + """ + current = _roots(roots) + _require(type(steps) is tuple, "steps must be an immutable tuple") + _require( + type(final_ids) is np.ndarray + and final_ids.dtype == np.dtype("int64") + and final_ids.ndim == 1, + "final household axis must be a one-dimensional int64 ndarray", + ) + final = _axis(tuple(final_ids)) + previous_branch = -1 + for step in steps: + _require( + type(step) in (HouseholdExpansion, HouseholdSelection), + "unsupported structural description", + ) + before = _axis(step.before_ids) + after = _axis(step.after_ids) + _require( + set(before) == set(current), "before axis differs from retained history" + ) + if type(step) is HouseholdSelection: + _require(set(after) <= set(before), "selection contains new households") + current = {target: current[target] for target in after} + continue + _require( + type(step.branch) is str and step.branch in identity._BRANCHES, + "unknown UK structural branch", + ) + position = identity._BRANCHES.index(step.branch) + _require(position > previous_branch, "repeated or reordered structural branch") + previous_branch = position + _require(set(before) <= set(after), "expansion removed incumbent households") + arrivals = set(after) - set(before) + parents = _pairs(step.parent_pairs) + ordinals = _pairs(step.child_ordinals, ordinals=True) + _require( + set(parents) == arrivals == set(ordinals), + "parents and ordinals must cover exactly the new households", + ) + _require(set(parents.values()) <= set(before), "parent absent before expansion") + branch_ids = [(parents[target], ordinals[target]) for target in arrivals] + _require( + len(branch_ids) == len(set(branch_ids)), + "repeated ordinal for one parent in this branch", + ) + extended = dict(current) + for target in arrivals: + source, vintage, source_id, path = current[parents[target]] + new_path = (*path, (step.branch, ordinals[target])) + # Reuse the existing complete path contract; no alternate key + # encoding or static branch roster is introduced here. + identity.household_draw_key( + source=source, + source_vintage=vintage, + source_household_id=source_id, + clone_path=new_path, + ) + extended[target] = (source, vintage, source_id, new_path) + current = {target: extended[target] for target in after} + _require(set(final) == set(current), "final axis differs from completed history") + keys = [ + identity.household_draw_key( + source=current[target][0], + source_vintage=current[target][1], + source_household_id=current[target][2], + clone_path=current[target][3], + ) + for target in final + ] + _require(len(keys) == len(set(keys)), "final geography keys are not unique") + return pd.DataFrame( + { + "household_id": np.asarray(final, dtype=np.int64), + IDENTITY_COLUMN: pd.array(keys, dtype=pd.StringDtype(storage="python")), + } + ) diff --git a/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py new file mode 100644 index 000000000..ace6106f1 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_household_lineage.py @@ -0,0 +1,588 @@ +"""Supplied lineage and real invented Graph/Frame controls; no FRS or engine.""" + +import json +from dataclasses import FrozenInstanceError, replace + +import numpy as np +import pandas as pd +import pytest +from test_uk_atomic_area_support import payloads + +from microcosm.build import atomic_geography +from microcosm.build.graph_atomic_geography import ( + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.atomic_household_lineage import ( + HouseholdExpansion, + HouseholdSelection, + project_atomic_household_keys, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def roots(ids=(1, 2)): + return pd.DataFrame( + { + "household_id": np.asarray(ids, dtype=np.int64), + "source": pd.array(["invented-frs"] * len(ids), dtype="string"), + "source_vintage": pd.array(["invented-2024-25"] * len(ids), dtype="string"), + "source_household_id": np.asarray(ids, dtype=np.int64), + } + ) + + +def expansion(branch, before, pairs, ordinals=None): + return HouseholdExpansion( + branch=branch, + before_ids=tuple(before), + after_ids=(*before, *(c for c, _ in pairs)), + parent_pairs=tuple(pairs), + child_ordinals=tuple((c, 1) for c, _ in pairs) + if ordinals is None + else ordinals, + ) + + +def chain(): + a = expansion("spi_support_channel", (1, 2), ((11, 1), (12, 2))) + b = expansion("cgt_incidence_clone", a.after_ids, ((21, 1), (31, 11))) + c = expansion("cgt_band_donors", b.after_ids, ((41, 31),)) + s = HouseholdSelection(c.after_ids, (1, 11, 21, 31, 41)) + d = expansion("geographic_support", s.after_ids, ((101, 1), (141, 41))) + return a, b, c, s, d + + +def project(source=None, steps=None, ids=None): + source = roots() if source is None else source + steps = chain() if steps is None else steps + ids = steps[-1].after_ids if ids is None else ids + return project_atomic_household_keys( + source, steps=steps, final_ids=np.asarray(ids, dtype=np.int64) + ) + + +def by_id(table): + return table.set_index("household_id")[IDENTITY_COLUMN].sort_index() + + +def test_explicit_full_chain_selection_and_clone_keys(): + result = project() + assert tuple(result.household_id) == chain()[-1].after_ids + assert list(result) == ["household_id", IDENTITY_COLUMN] + assert result.household_id.dtype == np.dtype("int64") + assert result[IDENTITY_COLUMN].dtype == pd.StringDtype(storage="python") + expected_paths = { + 1: (), + 11: (("spi_support_channel", 1),), + 21: (("cgt_incidence_clone", 1),), + 31: (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + 41: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + 101: (("geographic_support", 1),), + 141: ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ("geographic_support", 1), + ), + } + actual = by_id(result) + for target, path in expected_paths.items(): + assert actual[target] == household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=1, + clone_path=path, + ) + assert actual.is_unique + + +def test_axes_and_pair_order_do_not_supply_identity(): + steps = tuple( + replace( + x, + before_ids=x.before_ids[::-1], + after_ids=x.after_ids[::-1], + **( + { + "parent_pairs": x.parent_pairs[::-1], + "child_ordinals": x.child_ordinals[::-1], + } + if isinstance(x, HouseholdExpansion) + else {} + ), + ) + for x in chain() + ) + result = project(roots().iloc[::-1], steps, steps[-1].after_ids) + pd.testing.assert_series_equal(by_id(result), by_id(project())) + + +def test_exact_large_ids_and_growth_preserve_existing_keys(): + big = 2**53 + 7 + first = expansion("geographic_support", (big,), ((big + 2, big),)) + more = expansion( + "geographic_support", + (big, big + 1), + ((big + 2, big), (big + 3, big + 1)), + ((big + 2, 1), (big + 3, 9)), + ) + small = project(roots((big,)), (first,)) + large = project(roots((big, big + 1)), (more,)) + pd.testing.assert_series_equal(by_id(small), by_id(large).loc[by_id(small).index]) + assert by_id(large).is_unique + assert str(big) in by_id(small).iloc[0] + + +def test_source_and_explicit_ordinal_change_keys_without_numeric_inference(): + expected = by_id(project()) + source = roots() + source.loc[0, "source_vintage"] = "different-vintage" + changed = by_id(project(source)) + assert (changed != expected).all() + steps = chain() + changed = by_id( + project( + steps=(*steps[:-1], replace(steps[-1], child_ordinals=((101, 2), (141, 8)))) + ) + ) + assert (changed.loc[[101, 141]] != expected.loc[[101, 141]]).all() + pd.testing.assert_series_equal(changed.drop([101, 141]), expected.drop([101, 141])) + + +def test_noop_and_empty_selection_are_explicit_and_detached(): + original = roots() + saved = original.copy(deep=True) + no_steps = project(original, (), (2, 1)) + assert no_steps.household_id.tolist() == [2, 1] + empty = project(original, (HouseholdSelection((1, 2), ()),), ()) + assert empty.empty and empty.household_id.dtype == np.dtype("int64") + no_steps.loc[0, IDENTITY_COLUMN] = "changed descriptive output" + pd.testing.assert_frame_equal(original, saved) + with pytest.raises(FrozenInstanceError): + chain()[0].branch = "changed" + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_id", + "float_id", + "bool_id", + "zero_id", + "duplicate_root_identity", + "missing_source", + "blank_source", + "extra_column", + "duplicate_column", + "invalid_original", + "empty_roots", + ], +) +def test_roots_refuse_unqualified_or_ambiguous_descriptions(defect): + source = roots() + if defect == "duplicate_id": + source.loc[1, "household_id"] = 1 + elif defect == "float_id": + source["household_id"] = source.household_id.astype(float) + elif defect == "bool_id": + source["household_id"] = [True, False] + elif defect == "zero_id": + source.loc[0, "household_id"] = 0 + elif defect == "duplicate_root_identity": + source.loc[1, "source_household_id"] = 1 + elif defect == "missing_source": + source.loc[0, "source"] = pd.NA + elif defect == "blank_source": + source.loc[0, "source_vintage"] = " " + elif defect == "extra_column": + source["guess"] = 1 + elif defect == "duplicate_column": + source.columns = ["household_id", "source", "source", "source_household_id"] + elif defect == "invalid_original": + source["source_household_id"] = [1.0, 2.0] + elif defect == "empty_roots": + source = source.iloc[:0] + with pytest.raises((TypeError, ValueError)): + project(source) + + +@pytest.mark.parametrize( + "defect", + [ + "unknown_branch", + "reordered_branch", + "repeated_branch", + "mutable_steps", + "mutable_axis", + "duplicate_before", + "duplicate_after", + "unexplained_before", + "deleted_incumbent", + "unknown_parent", + "null_parent", + "same_step_parent", + "missing_parent", + "duplicate_parent_target", + "unexplained_child", + "missing_ordinal", + "extra_ordinal", + "duplicate_ordinal", + "negative_ordinal", + "bool_ordinal", + "float_child", + "overflow_child", + "selection_arrival", + "implicit_selection", + "final_extra", + "final_duplicate", + ], +) +def test_invalid_steps_and_final_axis_refuse(defect): + steps = list(chain()) + first = steps[0] + final = steps[-1].after_ids + if defect == "unknown_branch": + steps[0] = replace(first, branch="income_guess") + elif defect == "reordered_branch": + steps[0] = replace(first, branch="geographic_support") + elif defect == "repeated_branch": + steps[1] = replace(steps[1], branch=first.branch) + elif defect == "mutable_steps": + with pytest.raises((TypeError, ValueError)): + project(steps=steps) + return + elif defect == "mutable_axis": + steps[0] = replace(first, before_ids=[1, 2]) + elif defect == "duplicate_before": + steps[0] = replace(first, before_ids=(1, 1, 2)) + elif defect == "duplicate_after": + steps[0] = replace(first, after_ids=(*first.after_ids, 11)) + elif defect == "unexplained_before": + steps[0] = replace(first, before_ids=(1, 3)) + elif defect == "deleted_incumbent": + steps[0] = replace(first, after_ids=(1, 11, 12)) + elif defect == "unknown_parent": + steps[0] = replace(first, parent_pairs=((11, 3), (12, 2))) + elif defect == "null_parent": + steps[0] = replace(first, parent_pairs=((11, None), (12, 2))) + elif defect == "same_step_parent": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 11))) + elif defect == "missing_parent": + steps[0] = replace(first, parent_pairs=((11, 1),)) + elif defect == "duplicate_parent_target": + steps[0] = replace(first, parent_pairs=((11, 1), (11, 2))) + elif defect == "unexplained_child": + steps[0] = replace(first, after_ids=(*first.after_ids, 13)) + elif defect == "missing_ordinal": + steps[0] = replace(first, child_ordinals=((11, 1),)) + elif defect == "extra_ordinal": + steps[0] = replace(first, child_ordinals=(*first.child_ordinals, (99, 1))) + elif defect == "duplicate_ordinal": + steps[0] = replace(first, parent_pairs=((11, 1), (12, 1))) + elif defect == "negative_ordinal": + steps[0] = replace(first, child_ordinals=((11, -1), (12, 1))) + elif defect == "bool_ordinal": + steps[0] = replace(first, child_ordinals=((11, True), (12, 1))) + elif defect == "float_child": + steps[0] = replace(first, parent_pairs=((11.0, 1), (12, 2))) + elif defect == "overflow_child": + steps[0] = replace( + first, + after_ids=(1, 2, 11, 2**63), + parent_pairs=((11, 1), (2**63, 2)), + child_ordinals=((11, 1), (2**63, 1)), + ) + elif defect == "selection_arrival": + steps[3] = replace(steps[3], after_ids=(*steps[3].after_ids, 99)) + elif defect == "implicit_selection": + del steps[3] + elif defect == "final_extra": + final = (*final, 99) + elif defect == "final_duplicate": + final = (*final, final[0]) + with pytest.raises((TypeError, ValueError)): + project(steps=tuple(steps), ids=final) + + +class _FrameKernel(KernelBase): + ref = "invented.uk.lineage.frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def __init__(self, frame): + self.frame = frame + + def run(self, context): + return KernelResult(frame=self.frame) + + +class _ExpandKernel(KernelBase): + ref = "invented.uk.lineage.expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context): + return KernelResult( + expand={ + e: pd.Series( + [1, 2, 3], + index=pd.Index([11, 12, 13], name=e + "_id"), + dtype="int64", + ) + for e in ("person", "benunit", "household") + }, + weights=Weights(np.full(6, 0.5), WeightKind.IMPORTANCE), + ) + + +class _SelectKernel(KernelBase): + ref = "invented.uk.lineage.select@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER + ) + + def run(self, context): + ids = context.tables["person"].person_id + return KernelResult( + keep=pd.Series( + ids.isin([1, 3, 11, 13]).to_numpy(), + index=pd.Index(ids, name="person_id"), + ) + ) + + +def test_actual_expand_filter_receipts_feed_shared_atomic_graph_and_replay(tmp_path): + ids = np.arange(1, 4, dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [21, 33, 44], + } + ) + benunit = pd.DataFrame({"benunit_id": ids, "family_marker": [4, 5, 6]}) + household = pd.DataFrame( + { + "household_id": ids, + "region": pd.array( + ["LONDON", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + } + ) + frame = Frame( + {"person": person, "benunit": benunit, "household": household}, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(np.ones(3), WeightKind.DESIGN)}, + pd.Series(["invented"] * 3, name="stratum"), + metadata={"note": "invented UK lineage"}, + ) + columns = ( + Owned("person", "age", "int64"), + Owned("benunit", "family_marker", "int64"), + Owned("household", "region", "string"), + ) + graph = compile_graph( + Graph( + "uk", + (SourceRef("invented", "raw-bytes-v1"),), + ( + Node( + "root", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=columns, + ), + Node( + "expand", + _ExpandKernel.ref, + base="root", + structural=StructuralDelta.EXPAND, + mass="conserve", + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + ), + Node( + "select", + _SelectKernel.ref, + base="expand", + structural=StructuralDelta.FILTER, + mass="free", + inputs=(Slice("person", ("age",)),), + ), + ), + ) + ) + registry = KernelRegistry() + for k in (_FrameKernel(frame), _ExpandKernel(), _SelectKernel()): + registry.register(k) + source = tmp_path / "invented.txt" + source.write_text("invented rows; no external survey") + store = ContentStore(tmp_path / "lineage-store") + cold = run_graph(graph, sources={"invented": source}, store=store, kernels=registry) + replay = run_graph( + graph, + sources={"invented": source}, + store=store, + kernels=registry, + resume="require", + ) + assert all(n.hit for n in replay.nodes.values()) + pair_rows = cold.nodes["expand"].receipt["expand"]["household"] + assert pair_rows == replay.nodes["expand"].receipt["expand"]["household"] + expanded = cold.population("expand") + selected = cold.population("select") + steps = ( + HouseholdExpansion( + "spi_support_channel", + tuple(ids), + tuple(expanded.table("household").household_id), + tuple(tuple(p) for p in pair_rows), + ((11, 1), (12, 1), (13, 1)), + ), + HouseholdSelection( + tuple(expanded.table("household").household_id), + tuple(selected.table("household").household_id), + ), + ) + keys = project( + roots((1, 2, 3)), steps, tuple(selected.table("household").household_id) + ) + selected_tables = {e: selected.table(e).copy(deep=True) for e in selected.entities} + selected_tables["household"][IDENTITY_COLUMN] = keys[IDENTITY_COLUMN].array.copy() + keyed = Frame( + selected_tables, + selected.schema, + {"household": selected.weights_for("household")}, + selected.strata, + metadata=selected.metadata, + mass_log=selected.mass_log, + ) + support_payloads = payloads() + definition = uk_atomic_assignment_definition(support_payloads, seed=43) + all_columns = (*columns, Owned("household", IDENTITY_COLUMN, "string")) + atomic_nodes = atomic_geography_nodes( + definition, all_columns, base="keyed", emit_validation_artifact=True + ) + assignment_graph = compile_graph( + Graph( + "uk", + ( + SourceRef("invented", "raw-bytes-v1"), + *(SourceRef(SOURCES[s], "raw-bytes-v1") for s in support_payloads), + ), + ( + Node( + "keyed", + _FrameKernel.ref, + structural=StructuralDelta.CREATE, + sources=("invented",), + outputs=all_columns, + ), + *atomic_nodes, + ), + ) + ) + source_paths = {"invented": source} + for system, payload in support_payloads.items(): + path = tmp_path / (system + ".npz") + path.write_bytes(payload) + source_paths[SOURCES[system]] = path + atomic_registry = KernelRegistry() + atomic_registry.register(_FrameKernel(keyed)) + register_atomic_geography_kernels(atomic_registry) + atomic_store = ContentStore(tmp_path / "atomic-store") + assigned = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + ) + required = run_graph( + assignment_graph, + sources=source_paths, + store=atomic_store, + kernels=atomic_registry, + resume="require", + ) + assert len(assigned.nodes) == 7 and all(n.hit for n in required.nodes.values()) + actual = assigned.population("keyed") + for entity in keyed.entities: + pd.testing.assert_frame_equal( + actual.table(entity)[list(keyed.table(entity))], + keyed.table(entity), + check_index_type=False, + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, keyed.weights_for("household").values + ) + pd.testing.assert_series_equal(actual.strata, keyed.strata, check_index_type=False) + assert actual.metadata == keyed.metadata and actual.mass_log == keyed.mass_log + assert assigned.nodes["geography.gate"].receipt["outcome"] == "pass" + supports = { + k: atomic_geography.decode_atomic_support(v) + for k, v in support_payloads.items() + } + reordered = keyed.table("household").iloc[::-1] + expected = atomic_geography.assign_atomic(reordered, definition, supports) + for c in expected: + pd.testing.assert_series_equal( + expected[c].astype(pd.StringDtype(storage="python")).reset_index(drop=True), + actual.table("household")[c] + .iloc[::-1] + .astype(pd.StringDtype(storage="python")) + .reset_index(drop=True), + check_names=False, + ) + + (tmp_path / "acceptance.json").write_text( + json.dumps( + { + "scope": "invented actual shared CREATE/EXPAND/FILTER and atomic graph; pure descriptive lineage helper", + "lineage_nodes": len(cold.nodes), + "lineage_required_hits": sum(n.hit for n in replay.nodes.values()), + "atomic_nodes": len(assigned.nodes), + "atomic_required_hits": sum(n.hit for n in required.nodes.values()), + "root_households": frame.n("household"), + "expanded_households": expanded.n("household"), + "selected_households": selected.n("household"), + "source_admission": False, + "native_inputs": False, + }, + indent=2, + ) + + "\n" + ) From 47efa3fb4c060e31cfa5e621b34ea99b7f8dacb2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:35:09 -0400 Subject: [PATCH 183/255] Refresh country spec pins for reviewed solver source changes --- changelog.d/country-bundle-shared-source-pins.fixed.md | 1 + .../tests/test_spec_engine_country_bundles.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/country-bundle-shared-source-pins.fixed.md diff --git a/changelog.d/country-bundle-shared-source-pins.fixed.md b/changelog.d/country-bundle-shared-source-pins.fixed.md new file mode 100644 index 000000000..089ac9751 --- /dev/null +++ b/changelog.d/country-bundle-shared-source-pins.fixed.md @@ -0,0 +1 @@ +Update the AM, BE and UK country-bundle identity checks after the reviewed calibration snapshot metadata changes. Their authored resources and draw-site bindings remain unchanged; their resolved spec digests include the shared solver source attestation. diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index d9c38c0e9..5c0ecaa3c 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,10 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "258b940476e9f611d5e12d471dead7427479f36267eb40ed459fcd2f3dbd18ae" +# Resolved country identities include the shared legacy-v1 seed protocol. +# Reviewed snapshot iteration labels in solve.py change its source attestation; +# AM/BE/UK authored resources and draw-site bindings remain unchanged. +AM_SPEC_SHA256 = "ab8458f8520ffe8325bf9193a7c4f3cb76bade3944522fba5f9ab3f9c4b235b6" @pytest.mark.parametrize( @@ -45,7 +48,7 @@ ), ( "be", - "a61b65f7a1424c090081632fb1ddacdb85dae1b85920faff5dd7cb144cb1b5a1", + "44cabc2b42e47090605a9e9947eb71e0b894d008babdfe7415faf894aa330bf2", { "household.household_id", "person.person_id", @@ -55,7 +58,7 @@ ), ( "uk", - "66837fd0fad4956566655f558d10f8d39e83523ad873b17e9f37ce0cb5012724", + "fcaa57d66c3831e174917e64319cd7c651b5d8aad89dd13327ad6f7b4d5c85f3", { "benunit.benunit_id", "household.household_id", From d084711667139d20c302191cd067b316632ea133 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:51:37 -0400 Subject: [PATCH 184/255] Recheck retained survey observations after final I/O --- .../survey-final-observation-seal.fixed.md | 1 + .../us_runtime/graph_survey_population.py | 9 ++++++++- .../tests/test_us_graph_survey_population.py | 20 ++++++++++++++++--- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 changelog.d/survey-final-observation-seal.fixed.md diff --git a/changelog.d/survey-final-observation-seal.fixed.md b/changelog.d/survey-final-observation-seal.fixed.md new file mode 100644 index 000000000..bfa186d97 --- /dev/null +++ b/changelog.d/survey-final-observation-seal.fixed.md @@ -0,0 +1 @@ +Recheck the complete detached survey population observations after final source and artifact I/O, so late mutations cannot escape through the values retained for downstream graph composition. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py index 624a088d8..a262f2682 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py @@ -796,7 +796,8 @@ def _check_node_states(manifest, expected): class SurveyPopulationRunValues: """Values retained from one completed run, without additional authority. - These are the actual objects used by the runner, not decoded substitutes. + Populations are the runner's detached observations, checked against the + complete attached manifest Frames before return, not decoded substitutes. Downstream source-qualified boundaries must check the issued preparation and complete populations themselves. Freezing this container does not seal its contents, authenticate a copy, or admit a later graph descendant. @@ -1164,6 +1165,12 @@ def observe(node_id, population): ) _check_design_anchors(observed[node.id], cloned_design) for node_id, state in expected_states.items(): + # Observations are detached from executable/store populations. They can + # be retained for composition, so seal their values after the last I/O + # as well as the separately checked manifest Frames. + _same_frame( + manifest.population(compiled.versions[node_id]), observed[node_id].frame + ) _check_population_state(observed[node_id], **state) _require( manifest.mass_ledger(compiled.versions[node_id]) == state["ledger"], diff --git a/packages/microcosm-build/tests/test_us_graph_survey_population.py b/packages/microcosm-build/tests/test_us_graph_survey_population.py index 76d10506b..b3a603a88 100644 --- a/packages/microcosm-build/tests/test_us_graph_survey_population.py +++ b/packages/microcosm-build/tests/test_us_graph_survey_population.py @@ -547,7 +547,11 @@ def profile(frame, event, arg): assert injected == [field] -def test_authenticated_late_store_io_frame_mutation_refuses(tmp_path, monkeypatch): +@pytest.mark.parametrize("destination", ["observation", "manifest"]) +@pytest.mark.parametrize("return_values", [False, True]) +def test_authenticated_late_store_io_frame_mutation_refuses( + tmp_path, monkeypatch, destination, return_values +): arguments = authenticated_arguments(tmp_path, monkeypatch) populations, injected = {}, [] @@ -562,8 +566,16 @@ def profile(frame, event, arg): and frame.f_locals["node_id"] == graph.ALLOCATION_NODE and frame.f_locals["name"] == "frame_context" ): - table = populations[graph.ALLOCATION_NODE].frame.person + observed = populations[graph.ALLOCATION_NODE].frame.person + attached = ( + frame.f_locals["manifest"].population(graph.ALLOCATION_NODE).person + ) + assert observed is not attached + before = attached.loc[attached.index[0], "age"] + table = observed if destination == "observation" else attached table.loc[table.index[0], "age"] += 1 + other = attached if destination == "observation" else observed + assert other.loc[other.index[0], "age"] == before injected.append("after-last-artifact-load") previous = sys.getprofile() @@ -572,7 +584,9 @@ def profile(frame, event, arg): with pytest.raises( graph.SurveyPopulationGraphError, match="MATERIALIZED_FRAME_VALUES" ): - graph.run_authenticated_survey_population(**arguments, clones=False) + graph.run_authenticated_survey_population( + **arguments, clones=False, return_values=return_values + ) finally: sys.setprofile(previous) assert injected == ["after-last-artifact-load"] From c51c5a606257c35c38dd34b03c57a0f9839b32a7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:28:11 -0400 Subject: [PATCH 185/255] Add deterministic property tax leaf rebase and numerical gate --- changelog.d/us-property-tax-leaves.added.md | 6 + docs/us-property-tax-leaves.md | 114 ++++ .../us_runtime/graph_property_tax_leaves.py | 466 ++++++++++++++++ .../test_us_graph_property_tax_leaves.py | 497 ++++++++++++++++++ 4 files changed, 1083 insertions(+) create mode 100644 changelog.d/us-property-tax-leaves.added.md create mode 100644 docs/us-property-tax-leaves.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py create mode 100644 packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py diff --git a/changelog.d/us-property-tax-leaves.added.md b/changelog.d/us-property-tax-leaves.added.md new file mode 100644 index 000000000..fe4905618 --- /dev/null +++ b/changelog.d/us-property-tax-leaves.added.md @@ -0,0 +1,6 @@ +Add a deterministic three-node property tax rebase with explicit receiving +version, separate interest/dividend knownness, maintained fraction splits, +subtraction complements and typed numerical completeness evidence. Preserve +unknowns and retirement-account earnings; refuse receiving shapes that the +current FILTER operation cannot preserve. Country and PUF host integration +remains separate. diff --git a/docs/us-property-tax-leaves.md b/docs/us-property-tax-leaves.md new file mode 100644 index 000000000..93186359b --- /dev/null +++ b/docs/us-property-tax-leaves.md @@ -0,0 +1,114 @@ +# Property tax leaf rebase + +`graph_property_tax_leaves.py` provides three deterministic graph nodes for an +already checked population with property components and incumbent tax leaves. +It does not qualify sources, fit models, attach clones, grant a checked-run +handle or wire the country/PUF host. The owning host verifies the original +component observations, clone pairing and complete frame before using it. + +## Numerical operation + +`split_property_tax_leaves(person)` returns four float64 columns indexed by exact +int64 `person_id`. It imports the maintained fractions from `cps_carried`: + +| Input component | Primary leaf | Complement leaf | +| --- | --- | --- | +| `property_ordinary_interest` (`O`) | `taxable_interest_income = O * 0.680` | `tax_exempt_interest_income = O - taxable_interest_income` | +| `property_dividends` (`D`) | `qualified_dividend_income = D * 0.448` | `non_qualified_dividend_income = D - qualified_dividend_income` | + +The complement uses subtraction after computing the primary. This is a declared +operation order, including for the dividend complement that legacy code computes +by multiplying the complementary fraction. These fractions are modelling +assumptions; they do not establish a person's observed tax treatment. + +Interest and dividend knownness are separate. A finite nonnegative input produces +its two leaves; a missing input produces two NaNs, replacing any earlier tax +predictions. Zero stays an observed numerical zero. A negative or infinite O/D +input, an infinite retirement component, incompatible dtype or ambiguous person +identity refuses. `property_retirement_interest` is retained as an auxiliary +component and never enters either tax-interest leaf. Its missing value does not +invalidate known O/D values. Broad property receipts, net anchor sign, source age, +donor-fit eligibility and earlier predictions never supply missing O/D values. +No under-15 zero completion or missing-adult prediction occurs here. + +## Three graph nodes + +Call `property_tax_leaf_nodes(frame, population=base_version, +projection=projection_input, reconciliation=reconciliation_input, atol=..., +rtol=...)`. Both tolerances must be explicit finite nonnegative Python floats. +The two `ArtifactInput` aliases must be `projection` and `reconciliation`; the +host supplies their actual producer IDs and nominal types. Register +`PropertyTaxReceivingKernel`, `PropertyTaxLeavesKernel` and +`PropertyTaxLeafGateKernel`. + +1. `survey_property.tax_receiving` opens a new population version using + FILTER-all. This inherits the prior owners and permits explicit replacement of + the four incumbent leaves. The executor adds a real FILTER conservation record + to the population mass ledger. It preserves the existing `Frame.mass_log`. +2. `survey_property.tax_leaves` replaces the four leaves on that version and + emits `diagnostics` of type `microcosm.us.property_tax_leaf_diagnostics/v1`. +3. `survey_property.tax_leaf_gate` recomputes the leaves and their knownness, + checks exact leaf bits, partition sums within the declared tolerances, and the + complete diagnostics payload. It emits `verification` of type + `microcosm.us.property_tax_leaf_verification/v1`. + +The gate binds ordered person IDs without floating-point conversion, current +O/D/retirement bytes, all four leaf byte digests, fractions and operation order, +individual knownness, and the typed projection/reconciliation artifact keys and +payload hashes. It includes every physical person, including dependents and +zero-weight members. `complete` is false if any person lacks O or D; a successful +numerical check with missing inputs reports `evidence_absent`. Complete numerical +inputs report `pass`. Neither outcome establishes source, model, tax-validity or +release authority. The artifact explicitly leaves full-frame verification to +the host. + +## Supported receiving shapes and host responsibilities + +`Frame.select(all)` currently refuses link tables and normalizes group indices. +The declaration factory therefore refuses links, nondefault group indices, +orphan groups, empty person frames and groups lacking a readable non-ID column. +The last condition ensures the kernel context exposes group IDs for the +receiving check. Orphans are already invalid at Frame construction; the factory +also detects a later membership mutation. The kernel rechecks exposed group +membership and logical group indices immediately before returning its keep mask. +It never silently prunes a group or switches structural operation. + +For supported frames, person rows/order/IDs, entity tables, weights and their +kinds, schema, strata, metadata and frame mass log are preserved, apart from the +four declared leaf replacements. The new population version, owners and +conservation record are intentional changes. Kernel contexts do not expose the +complete schema, metadata or population ledger, so the host must retain and +verify those independently. It must also qualify the complete supported shape +before adding these nodes, bind actual artifact producers and stored payloads, +and recheck the original owners and output after its final relevant I/O. + +For independent expected-result reconstruction, the host can call each kernel's +`run(KernelContext)` on the retained current population and authenticated typed +artifact aliases. The existing executor `_project_context` and `_apply_result` +provide the canonical context projection and result application. FILTER results +must be materialized through the validated keep mask before `population.patch`; +passing its raw `keep` result directly to that lower-level patch function omits +the executor's frame selection. Ordinary rebase/gate results use the existing +patch path. These deterministic kernels consume no random draws. Expected +artifact bytes and receipts can then be compared with the actual store and node +states, and expected full populations with the host's retained observations. +This fragment adds no separate executor, authority registry or public run issuer. + +Future PUF integration must depend on this verification artifact and require +complete tax leaves over every physical member the PUF consumer needs. The +fragment does not add that host dependency itself. Matching checked inputs give +matching outputs on paired clones; the fragment does not infer or repair clone +pairing. Capital gains retain their earlier conditioning and are not rebased by +this change. + +## Verification scope + +The 32 invented controls exercise a real four-node graph including its source, +ContentStore cold execution and required-cache replay. They cover negative, +zero-net and positive anchors with loss offsets, independent O/D unknowns, +known O/D with unknown retirement/broad components, integer IDs above `2**53`, +row permutations, empty pure input, malformed values, altered leaf bits, +diagnostics and artifact bindings, full-frame preservation, zero-weight records, +the population conservation record and unsupported receiving shapes. No native +survey, country engine, PUF execution, fitting or publication is part of this +acceptance. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py new file mode 100644 index 000000000..177167a4a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py @@ -0,0 +1,466 @@ +"""Strict deterministic property splits and a numerical completeness gate. + +The host owns source qualification, clone pairing and full-Frame lifetime checks. +These operations grant no source or release authority and never complete unknown +components. The receiving FILTER opens a real population version and ledger entry. +""" + +from __future__ import annotations + +import hashlib +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + KernelRole, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.keys import opaque_artifact_key + +from . import cps_carried +from .property_income_constants import PROPERTY_COMPONENTS + +PROTOCOL = "microcosm.us.property-tax-leaves.v1" +RECEIVING_NODE = "survey_property.tax_receiving" +TAX_LEAVES_NODE = "survey_property.tax_leaves" +GATE_NODE = "survey_property.tax_leaf_gate" +TAX_LEAF_COLUMNS = ( + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", +) +DIAGNOSTICS_TYPE = ArtifactType("microcosm.us.property_tax_leaf_diagnostics", 1) +VERIFICATION_TYPE = ArtifactType("microcosm.us.property_tax_leaf_verification", 1) +_INPUTS = (PROPERTY_COMPONENTS[0], PROPERTY_COMPONENTS[2], PROPERTY_COMPONENTS[1]) + + +def _require(condition, reason): + if not condition: + raise ValueError("PROPERTY_TAX_LEAVES_" + reason) + + +def _parameters(atol, rtol): + _require( + all(type(v) is float and np.isfinite(v) and v >= 0 for v in (atol, rtol)), + "TOLERANCES", + ) + fractions = ( + cps_carried.TAXABLE_INTEREST_FRACTION, + cps_carried.QUALIFIED_DIVIDEND_FRACTION, + ) + _require( + all(type(v) is float and np.isfinite(v) and 0 < v < 1 for v in fractions), + "FRACTIONS", + ) + return { + "protocol": PROTOCOL, + "taxable_interest_fraction": fractions[0], + "qualified_dividend_fraction": fractions[1], + "complements": "total_minus_computed_primary", + "unknown": "nan", + "retirement_interest": "auxiliary_unchanged", + "atol": atol, + "rtol": rtol, + } + + +def _person(person): + _require( + type(person) is pd.DataFrame + and person.columns.is_unique + and {"person_id", *_INPUTS} <= set(person), + "PERSON_COLUMNS", + ) + _require( + person.person_id.dtype == np.dtype("int64") and person.person_id.is_unique, + "PERSON_IDENTITY", + ) + for name in _INPUTS: + _require(person[name].dtype == np.dtype("float64"), "INPUT_DTYPE:" + name) + values = person[name].to_numpy(copy=False) + _require(not np.isinf(values).any(), "INFINITE_COMPONENT:" + name) + if name != PROPERTY_COMPONENTS[1]: + _require(not (values < 0).any(), "NEGATIVE_COMPONENT:" + name) + return pd.Index(person.person_id.to_numpy(copy=True), name="person_id") + + +def split_property_tax_leaves(person: pd.DataFrame) -> pd.DataFrame: + """Split individually known O/D amounts; retain NaN independently per family. + + Returned float64 columns are keyed by exact integer person IDs. Source age, + net anchors, donor eligibility, retirement earnings, and old tax predictions + never establish knownness or select the amount of either split. + """ + index = _person(person) + params = _parameters(0.0, 0.0) + columns = {} + for component, primary, complement, fraction in ( + ( + PROPERTY_COMPONENTS[0], + *TAX_LEAF_COLUMNS[:2], + params["taxable_interest_fraction"], + ), + ( + PROPERTY_COMPONENTS[2], + *TAX_LEAF_COLUMNS[2:], + params["qualified_dividend_fraction"], + ), + ): + amount = person[component].to_numpy(copy=False) + known = np.isfinite(amount) + first = np.full(len(person), np.nan, dtype=np.float64) + second = first.copy() + first[known] = amount[known] * fraction + second[known] = amount[known] - first[known] + _require( + np.isfinite(first[known]).all() and np.isfinite(second[known]).all(), + "UNSTABLE_SPLIT", + ) + columns[primary], columns[complement] = first, second + return pd.DataFrame(columns, index=index) + + +def _slices(frame): + _require(type(frame) is Frame and frame.schema.person_entity == "person", "FRAME") + _require(not frame.links, "UNSUPPORTED_LINKS") + _person(frame.person) + _require(len(frame.person) > 0, "EMPTY_RECEIVING_FRAME") + result = [] + for entity in frame.entities: + table = frame.table(entity) + columns = tuple(c for c in table if not c.endswith("_id")) + _require(bool(columns), "NO_READABLE_ENTITY_COLUMN:" + entity) + if entity != "person": + _require( + type(table.index) is pd.RangeIndex + and table.index.equals(pd.RangeIndex(len(table))) + and table.index.name is None, + "UNSUPPORTED_GROUP_INDEX:" + entity, + ) + ids = table[frame.schema.entity_id_column(entity)] + members = frame.person[frame.schema.membership_column(entity)] + _require(set(ids) == set(members), "ORPHAN_GROUP:" + entity) + result.append(Slice(entity, columns)) + for name in TAX_LEAF_COLUMNS: + _require( + name in frame.person and frame.person[name].dtype == np.dtype("float64"), + "INCUMBENT_LEAF:" + name, + ) + return tuple(result) + + +def _nodes(base, slices, projection, reconciliation, atol, rtol): + _require( + type(projection) is type(reconciliation) is ArtifactInput + and projection.name == "projection" + and reconciliation.name == "reconciliation", + "ARTIFACT_EDGES", + ) + params = _parameters(atol, rtol) + edges = (projection, reconciliation) + receiving = Node( + RECEIVING_NODE, + PropertyTaxReceivingKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=slices, + params={"protocol": PROTOCOL, "mode": "all_rows_supported_shape"}, + artifact_inputs=edges, + description="Open a tax-rebase version with all original rows; refuse orphan groups, links and group-index normalization.", + ) + rebase = Node( + TAX_LEAVES_NODE, + PropertyTaxLeavesKernel.ref, + population=RECEIVING_NODE, + inputs=(Slice("person", (*_INPUTS, *TAX_LEAF_COLUMNS)),), + outputs=tuple( + Owned("person", c, "float64", rewrite=True) for c in TAX_LEAF_COLUMNS + ), + params=params, + artifact_inputs=edges, + artifact_outputs=(ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE),), + description="Rebase four tax leaves from individually known ordinary interest and dividends; preserve unknowns and retirement-account earnings.", + ) + gate = Node( + GATE_NODE, + PropertyTaxLeafGateKernel.ref, + population=RECEIVING_NODE, + inputs=(Slice("person", (*_INPUTS, *TAX_LEAF_COLUMNS)),), + params=params, + artifact_inputs=( + *edges, + ArtifactInput("rebase", TAX_LEAVES_NODE, "diagnostics", DIAGNOSTICS_TYPE), + ), + artifact_outputs=(ArtifactOutput("verification", VERIFICATION_TYPE),), + description="Recompute all four leaves, partition sums and individual knownness; missing member inputs keep completeness false.", + ) + return receiving, rebase, gate + + +def property_tax_leaf_nodes( + frame: Frame, + *, + population: str, + projection: ArtifactInput, + reconciliation: ArtifactInput, + atol: float, + rtol: float, +) -> tuple[Node, Node, Node]: + """Declare an explicit receiving version, four rewrites and numeric gate. + + The host must bind/recheck this complete input Frame. FILTER-all cannot + preserve orphan groups, link tables or nondefault group indices; refuse + those shapes before execution. Group tables need a declared non-ID column + so the receiving kernel can check their actual IDs through KernelContext. + """ + return _nodes(population, _slices(frame), projection, reconciliation, atol, rtol) + + +def _bindings(context): + _require( + set(context.artifacts) == {e.name for e in context.node.artifact_inputs}, + "ARTIFACT_ROSTER", + ) + result = {} + for edge in context.node.artifact_inputs: + value = context.artifacts[edge.name] + _require( + value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact), + "ARTIFACT_BINDING", + ) + if edge.name != "rebase": + result[edge.name] = { + "producer": edge.producer, + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": hashlib.sha256(value.payload).hexdigest(), + "type": (edge.type.name, edge.type.schema_version), + } + return result + + +def _diagnostics(person, split, params, bindings): + interest = np.isfinite(person[PROPERTY_COMPONENTS[0]].to_numpy()) + dividend = np.isfinite(person[PROPERTY_COMPONENTS[2]].to_numpy()) + return { + "protocol": PROTOCOL, + "parameters": dict(params), + "input_artifacts": bindings, + "person_ids": [str(value) for value in person.person_id], + "interest_known": interest.tolist(), + "dividend_known": dividend.tolist(), + "known_leaf_origin": "derived_maintained_fraction_and_subtraction_complement", + "unknown_leaf_origin": "unknown_component_no_completion", + "leaf_knownness_groups": (TAX_LEAF_COLUMNS[:2], TAX_LEAF_COLUMNS[2:]), + "input_sha256": { + c: hashlib.sha256( + person[c].to_numpy().astype(" KernelResult: + node = context.node + edges = {e.name: e for e in node.artifact_inputs} + _require(set(edges) == {"projection", "reconciliation"}, "ARTIFACT_EDGES") + expected = _nodes( + node.base, + node.inputs, + edges["projection"], + edges["reconciliation"], + 0.0, + 0.0, + )[0] + _require( + node.normative() == expected.normative() + and not context.sources + and dict(context.params) == dict(node.params), + "RECEIVING_DECLARATION", + ) + _bindings(context) + _require( + {s.entity for s in node.inputs} == set(context.tables) + and "person" in context.tables, + "RECEIVING_TABLES", + ) + person = context.tables["person"] + ids = _person(person) + _require(len(ids) > 0, "EMPTY_RECEIVING_FRAME") + for entity, table in context.tables.items(): + if entity == "person": + continue + _require( + table.index.equals(pd.RangeIndex(len(table))) + and table.index.name is None, + "UNSUPPORTED_GROUP_INDEX:" + entity, + ) + _require( + set(table[entity + "_id"]) == set(person["person_" + entity + "_id"]), + "ORPHAN_GROUP:" + entity, + ) + return KernelResult( + keep=pd.Series(True, index=ids), + receipt={ + "protocol": PROTOCOL, + "persons": len(ids), + "all_rows_retained": True, + "population_ledger_transition": "filter_conserve", + "source_authority": False, + }, + ) + + +class PropertyTaxLeavesKernel(_Kernel): + ref = "us.property_tax_leaves@1" + + def run(self, context: KernelContext) -> KernelResult: + bindings = self._check(context, False) + person = context.tables["person"] + split = split_property_tax_leaves(person) + document = _diagnostics(person, split, context.params, bindings) + return KernelResult( + columns={("person", c): split[c] for c in TAX_LEAF_COLUMNS}, + artifacts={"diagnostics": codec.encode_json(document)}, + receipt=document, + ) + + +class PropertyTaxLeafGateKernel(_Kernel): + ref = "us.property_tax_leaf_gate@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + role=KernelRole.GATE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + bindings = self._check(context, True) + person = context.tables["person"] + split = split_property_tax_leaves(person) + for name in TAX_LEAF_COLUMNS: + _require( + name in person and person[name].dtype == np.dtype("float64"), + "LEAF_DTYPE", + ) + _require( + np.array_equal( + person[name].to_numpy().view("uint64"), + split[name].to_numpy().view("uint64"), + ), + "LEAF_BITS:" + name, + ) + for component, names in ( + (PROPERTY_COMPONENTS[0], TAX_LEAF_COLUMNS[:2]), + (PROPERTY_COMPONENTS[2], TAX_LEAF_COLUMNS[2:]), + ): + amount = person[component].to_numpy() + known = np.isfinite(amount) + with np.errstate(over="ignore", invalid="ignore"): + summed = split.loc[:, list(names)].to_numpy()[known].sum(axis=1) + _require(np.isfinite(summed).all(), "CONSERVATION_OVERFLOW") + error = np.abs(summed - amount[known]) + scale = amount[known] + relative = np.divide( + error, scale, out=np.zeros_like(error), where=scale != 0 + ) + _require( + ( + (error <= context.params["atol"]) + | ((scale != 0) & (relative <= context.params["rtol"])) + ).all(), + "PARTITION_CONSERVATION", + ) + document = _diagnostics(person, split, context.params, bindings) + _require( + context.artifacts["rebase"].payload == codec.encode_json(document), + "DIAGNOSTIC_PAYLOAD", + ) + document = {**document, "numeric_verified": True} + return KernelResult( + artifacts={"verification": codec.encode_json(document)}, + receipt={ + "outcome": "pass" if document["complete"] else "evidence_absent", + "evidence": document, + }, + ) diff --git a/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py b/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py new file mode 100644 index 000000000..5ee886508 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py @@ -0,0 +1,497 @@ +"""Deterministic tax splits on invented components; no native source or engine.""" + +import hashlib +import importlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime.property_income_constants import PROPERTY_REPORTED_TOTAL +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.frame.schema import LinkSpec +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + load_source, + run_graph, +) + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.graph_property_tax_leaves" + ) + + +def people(owner): + # First three rows have negative, zero-net, and positive anchors, each with + # positive ordinary interest and an offsetting signed broad-property amount. + data = pd.DataFrame( + { + "person_id": np.arange(8, dtype=np.int64) + 2**53 + 21, + "person_household_id": [301, 301, 302, 302, 303, 303, 304, 304], + owner.PROPERTY_COMPONENTS[0]: [ + 100.0, + 100.0, + 100.0, + np.nan, + 20.0, + 0.0, + 40.0, + 0.0, + ], + owner.PROPERTY_COMPONENTS[1]: [ + 40.0, + 40.0, + 40.0, + 5.0, + np.nan, + 0.0, + np.nan, + 0.0, + ], + owner.PROPERTY_COMPONENTS[2]: [ + 50.0, + 50.0, + 50.0, + 30.0, + np.nan, + 0.0, + 80.0, + 0.0, + ], + owner.PROPERTY_COMPONENTS[3]: [ + -250.0, + -190.0, + -90.0, + np.nan, + 5.0, + 0.0, + np.nan, + 0.0, + ], + "unrelated_nullable": pd.array( + [None, 1.0, 2.0, None, 0.0, 4.0, 5.0, 6.0], dtype="Int64" + ), + "geography": pd.array(["001"] * 8, dtype="string"), + "property_components_known": [ + True, + True, + True, + False, + False, + True, + False, + True, + ], + } + ) + data[PROPERTY_REPORTED_TOTAL] = [ + -60.0, + 0.0, + 100.0, + np.nan, + np.nan, + 0.0, + np.nan, + 0.0, + ] + for column in owner.TAX_LEAF_COLUMNS: + data[column] = 999.0 + data.index = pd.Index([13, 4, 19, 2, 7, 22, 1, 31], name="original_row") + return data + + +def test_split_uses_individual_knownness_and_declared_complements(owner): + data = people(owner) + before = data.copy(deep=True) + result = owner.split_property_tax_leaves(data) + np.testing.assert_allclose( + result.iloc[0], [68.0, 32.0, 22.4, 27.6], rtol=0, atol=1e-14 + ) + assert result.iloc[3, :2].isna().all() + assert result.iloc[3, 2:].notna().all() + assert result.iloc[4, :2].notna().all() + assert result.iloc[4, 2:].isna().all() + assert result.iloc[5].eq(0).all() + assert result.iloc[6].notna().all() # R and broad receipts unknown. + assert result.index.tolist() == data.person_id.tolist() + assert result.index.name == "person_id" + pd.testing.assert_frame_equal(data, before, check_exact=True) + assert result.iloc[:3].eq(result.iloc[0]).all().all() + + +@pytest.mark.parametrize("column", [0, 2]) +@pytest.mark.parametrize("bad", [-1.0, np.inf, -np.inf]) +def test_invalid_component_refuses(owner, column, bad): + data = people(owner) + data.iloc[0, data.columns.get_loc(owner.PROPERTY_COMPONENTS[column])] = bad + with pytest.raises(ValueError): + owner.split_property_tax_leaves(data) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_id", + "float_id", + "object_component", + "float32_component", + "missing_component", + ], +) +def test_identity_and_dtype_refuse(owner, defect): + data = people(owner) + if defect == "duplicate_id": + data.iloc[1, data.columns.get_loc("person_id")] = data.person_id.iloc[0] + elif defect == "float_id": + data["person_id"] = data.person_id.astype(float) + elif defect == "object_component": + data[owner.PROPERTY_COMPONENTS[0]] = data[owner.PROPERTY_COMPONENTS[0]].astype( + object + ) + elif defect == "float32_component": + data[owner.PROPERTY_COMPONENTS[0]] = data[owner.PROPERTY_COMPONENTS[0]].astype( + "float32" + ) + else: + data = data.drop(columns=owner.PROPERTY_COMPONENTS[2]) + with pytest.raises(ValueError): + owner.split_property_tax_leaves(data) + + +def test_permutation_empty_and_exact_operation_order(owner): + data = people(owner) + result = owner.split_property_tax_leaves(data) + shuffled = owner.split_property_tax_leaves(data.iloc[::-1]) + pd.testing.assert_frame_equal(shuffled.loc[result.index], result, check_exact=True) + assert owner.split_property_tax_leaves(data.iloc[:0]).empty + ordinary = data[owner.PROPERTY_COMPONENTS[0]].to_numpy() + dividends = data[owner.PROPERTY_COMPONENTS[2]].to_numpy() + expected = np.column_stack( + ( + ordinary * 0.680, + ordinary - ordinary * 0.680, + dividends * 0.448, + dividends - dividends * 0.448, + ) + ) + np.testing.assert_array_equal( + result.to_numpy().view("uint64"), expected.view("uint64") + ) + + +PROJECTION = ArtifactType("test.property_projection", 1) +RECONCILIATION = ArtifactType("test.property_reconciliation", 1) + + +class SourceKernel(KernelBase): + ref = "test.property_tax_source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult( + frame=load_source("frame-store", context.sources["invented"]), + artifacts={ + "projection": b'{"invented":true}', + "reconciliation": b'{"numerical_fixture":true}', + }, + ) + + +def declaration(owner, frame): + return owner.property_tax_leaf_nodes( + frame, + population="source", + projection=ArtifactInput("projection", "source", "projection", PROJECTION), + reconciliation=ArtifactInput( + "reconciliation", "source", "reconciliation", RECONCILIATION + ), + atol=1e-10, + rtol=1e-12, + ) + + +@pytest.fixture +def actual(owner, tmp_path, request): + data = people(owner) + if getattr(request, "param", False): + # A separate fully known invented input, not production completion. + for component in (owner.PROPERTY_COMPONENTS[0], owner.PROPERTY_COMPONENTS[2]): + data[component] = data[component].fillna(10.0) + frame = Frame( + { + "person": data, + "household": pd.DataFrame( + { + "household_id": [301, 302, 303, 304], + "retained_group_value": [3, 4, 5, 6], + } + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights([1.0, 2.0, 0.0, 0.0], WeightKind.DESIGN)}, + metadata={"invented": True, "keep": {"nested": (1, "retained")}}, + ) + store = ContentStore(tmp_path / "store") + path = store.put_frame( + hashlib.sha256(b"invented-property-tax-frame").hexdigest(), frame + ) + source = Node( + "source", + SourceKernel.ref, + sources=("invented",), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(entity, column, str(frame.table(entity)[column].dtype)) + for entity in frame.entities + for column in frame.table(entity) + if not column.endswith("_id") + ), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION), + ArtifactOutput("reconciliation", RECONCILIATION), + ), + ) + receiving, rebase, gate = declaration(owner, frame) + graph = Graph( + country="us", + nodes=(source, receiving, rebase, gate), + sources=(SourceRef("invented", "frame-store"),), + ) + compiled = compile_graph(graph) + contexts = {} + + class RecordingReceiving(owner.PropertyTaxReceivingKernel): + def run(self, context): + contexts["receiving"] = context + return super().run(context) + + class RecordingSplit(owner.PropertyTaxLeavesKernel): + def run(self, context): + contexts["split"] = context + return super().run(context) + + class RecordingGate(owner.PropertyTaxLeafGateKernel): + def run(self, context): + contexts["gate"] = context + return super().run(context) + + registry = KernelRegistry() + for kernel in ( + SourceKernel(), + RecordingReceiving(), + RecordingSplit(), + RecordingGate(), + ): + registry.register(kernel) + cold = run_graph( + compiled, + store=store, + kernels=registry, + sources={"invented": path}, + resume="auto", + ) + warm = run_graph( + compiled, + store=store, + kernels=registry, + sources={"invented": path}, + resume="require", + ) + return frame, compiled, cold, warm, contexts + + +def test_actual_graph_cold_required_preserves_full_frame_and_unknowns(owner, actual): + before, compiled, cold, warm, contexts = actual + after = cold.population(owner.RECEIVING_NODE) + assert len(compiled.order) == 4 + assert not any(node.hit for node in cold.nodes.values()) + assert all(node.hit for node in warm.nodes.values()) + assert cold.key == warm.key + for entity in before.entities: + retained = [c for c in before.table(entity) if c not in owner.TAX_LEAF_COLUMNS] + pd.testing.assert_frame_equal( + after.table(entity)[retained], + before.table(entity)[retained], + check_exact=True, + ) + pd.testing.assert_frame_equal( + after.table(entity), + warm.population(owner.RECEIVING_NODE).table(entity), + check_exact=True, + ) + assert after.schema == before.schema and after.links == before.links + assert after.metadata == before.metadata and after.mass_log == before.mass_log + pd.testing.assert_series_equal(after.strata, before.strata) + np.testing.assert_array_equal( + after.weights_for("household").values, before.weights_for("household").values + ) + assert after.weights_for("household").kind is WeightKind.DESIGN + original_ledger = cold.mass_ledger("source") + receiving_ledger = cold.mass_ledger(owner.RECEIVING_NODE) + assert receiving_ledger[:-1] == original_ledger + transition = receiving_ledger[-1] + assert transition.node_id == owner.RECEIVING_NODE + assert transition.operation == "filter" + assert transition.policy == "conserve" + assert transition.before_total == transition.after_total + assert receiving_ledger == warm.mass_ledger(owner.RECEIVING_NODE) + result = owner.PropertyTaxLeafGateKernel().run(contexts["gate"]) + doc = json.loads(result.artifacts["verification"]) + assert doc["complete"] is False + assert doc["numeric_verified"] is True + assert doc["unknown_interest_persons"] == doc["unknown_dividend_persons"] == 1 + assert result.receipt["outcome"] == "evidence_absent" + assert doc["source_authority"] is False + assert doc["clone_pairing"] == "host_required" + assert doc["whole_frame_verification"] == "host_required" + + +@pytest.mark.parametrize("actual", [True], indirect=True) +def test_complete_gate_does_not_require_retirement_or_broad_components(owner, actual): + context = actual[-1]["gate"] + assert context.tables["person"][owner.PROPERTY_COMPONENTS[1]].isna().any() + result = owner.PropertyTaxLeafGateKernel().run(context) + doc = json.loads(result.artifacts["verification"]) + assert doc["complete"] is True + assert doc["unknown_interest_persons"] == doc["unknown_dividend_persons"] == 0 + assert result.receipt["outcome"] == "pass" + + +@pytest.mark.parametrize( + "defect", + [ + "leaf_bit", + "unknown_leaf", + "retirement_component", + "person_id", + "diagnostic_payload", + "producer_key", + "missing_artifact", + "params", + ], +) +def test_actual_gate_rejects_changed_values_or_receipt(owner, actual, defect): + context = actual[-1]["gate"] + table = context.tables["person"].copy(deep=True) + artifacts = dict(context.artifacts) + node = context.node + if defect == "leaf_bit": + name = owner.TAX_LEAF_COLUMNS[0] + table.iloc[0, table.columns.get_loc(name)] = np.nextafter( + table[name].iloc[0], np.inf + ) + elif defect == "unknown_leaf": + table.iloc[3, table.columns.get_loc(owner.TAX_LEAF_COLUMNS[0])] = 0.0 + elif defect == "retirement_component": + table.iloc[0, table.columns.get_loc(owner.PROPERTY_COMPONENTS[1])] += 1.0 + elif defect == "person_id": + table.iloc[0, table.columns.get_loc("person_id")] += 1 + elif defect == "diagnostic_payload": + old = artifacts["rebase"] + artifacts["rebase"] = replace(old, payload=b'{"complete":true}') + elif defect == "producer_key": + old = artifacts["rebase"] + artifacts["rebase"] = ArtifactValue( + old.payload, old.type, old.key, "0" * 64, old.numerics + ) + elif defect == "missing_artifact": + artifacts.pop("rebase") + else: + node = replace(node, params={**node.params, "unknown": "zero"}) + changed = replace( + context, + node=node, + params=node.params, + tables={"person": table}, + artifacts=artifacts, + ) + with pytest.raises(ValueError): + owner.PropertyTaxLeafGateKernel().run(changed) + + +@pytest.mark.parametrize("bad", [-1.0, float("inf"), float("nan"), True]) +def test_tolerance_contract_refuses(owner, bad): + with pytest.raises(ValueError): + owner.property_tax_leaf_nodes( + _supported_frame(owner), + population="source", + projection=ArtifactInput("projection", "source", "projection", PROJECTION), + reconciliation=ArtifactInput( + "reconciliation", "source", "reconciliation", RECONCILIATION + ), + atol=bad, + rtol=1e-12, + ) + + +def _supported_frame(owner, *, defect=None): + groups = pd.DataFrame( + {"household_id": [301, 302, 303, 304], "retained": [1, 2, 3, 4]} + ) + weights = [1.0, 1.0, 0.0, 0.0] + if defect == "group_index": + groups.index = pd.Index([4, 3, 2, 1], name="keep_index") + elif defect == "id_only": + groups = groups[["household_id"]] + tables = {"person": people(owner), "household": groups} + links = () + if defect == "links": + links = (LinkSpec("connections", "person", "household"),) + tables["connections"] = pd.DataFrame( + {"person_id": [tables["person"].person_id.iloc[0]], "household_id": [301]} + ) + frame = Frame( + tables, + EntitySchema(group_entities=("household",), links=links), + {"household": Weights(weights, WeightKind.DESIGN)}, + ) + if defect == "orphan": + # Constructor validation already refuses orphans. Exercise the factory's + # defensive check after an invented post-construction source mutation. + frame.person.loc[ + frame.person.person_household_id == 304, "person_household_id" + ] = 303 + return frame + + +@pytest.mark.parametrize("defect", ["orphan", "group_index", "id_only", "links"]) +def test_receiving_refuses_shapes_that_select_would_change_or_cannot_inspect( + owner, defect +): + source = _supported_frame(owner, defect=defect) + before = source.table("household").copy(deep=True) + with pytest.raises(ValueError): + declaration(owner, source) + pd.testing.assert_frame_equal(source.table("household"), before, check_exact=True) + + +def test_receiving_rechecks_membership_after_declaration(owner, actual): + context = actual[-1]["receiving"] + tables = dict(context.tables) + tables["person"] = tables["person"].copy(deep=True) + person = tables["person"] + person.loc[person.person_household_id == 304, "person_household_id"] = 303 + with pytest.raises(ValueError, match="ORPHAN_GROUP:household"): + owner.PropertyTaxReceivingKernel().run(replace(context, tables=tables)) From 5451d0a7d2317371ac3c114234f779042fed2bbb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:53:45 -0400 Subject: [PATCH 186/255] Add optional checked property tax rebase to survey financial host --- changelog.d/atomic-property-tax-host.added.md | 5 + docs/us-atomic-property-financial.md | 54 +++- docs/us-property-tax-leaves.md | 10 + .../graph_atomic_survey_financial.py | 281 +++++++++++++++++- .../us_runtime/graph_property_tax_leaves.py | 54 +++- .../graph_puf55_survey_recipients.py | 13 +- .../us_runtime/puf55_survey_recipients.py | 1 + ..._us_graph_atomic_property_tax_financial.py | 156 ++++++++++ .../test_us_graph_property_tax_leaves.py | 80 +++++ 9 files changed, 624 insertions(+), 30 deletions(-) create mode 100644 changelog.d/atomic-property-tax-host.added.md create mode 100644 packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py diff --git a/changelog.d/atomic-property-tax-host.added.md b/changelog.d/atomic-property-tax-host.added.md new file mode 100644 index 000000000..f3ef5b12a --- /dev/null +++ b/changelog.d/atomic-property-tax-host.added.md @@ -0,0 +1,5 @@ +Add an explicit optional 38-node survey financial host that retains its complete +19- and 35-node populations, rebases property tax leaves with strict unknowns, +and verifies the new full population and numerical artifacts. Add a conditional +typed PUF gate dependency and refuse PUF qualification when required rebased +inputs remain unknown. The existing 19- and 35-node options remain unchanged. diff --git a/docs/us-atomic-property-financial.md b/docs/us-atomic-property-financial.md index 05e096d50..4380c0f91 100644 --- a/docs/us-atomic-property-financial.md +++ b/docs/us-atomic-property-financial.md @@ -58,16 +58,60 @@ verified model artifacts' byte identities. They do not refit or reapply the models. Copied public result dataclasses have no run authority. See the [model receipt contract](us-property-model-receipts.md). -This first extension leaves the eight existing financial leaves unchanged. +The thirty-five-node option leaves the eight existing financial leaves unchanged. Its run document therefore records `tax_split_rebased: false` and the legacy capital-gain model's conditioning on its earlier interest and dividend draws. -The new property components do not yet establish coherent replacement tax -inputs. A later rebase must use ordinary interest rather than account earnings, -preserve separately known ASEC observations, and explicitly complete any -required unknown inputs before full PUF use. +It does not itself replace tax inputs. Optional tax rebasing is a separate +three-node extension, described below. The PUF host derives its upstream node count and checks the actual final financial writer. Its recipient nodes read all columns of that population, including the new property fields, so the dependency is present in the graph. This interface does not establish that a native PUF run, tax rebase, calibration or release has passed. Candidate execution evidence belongs in `experiments/`. + +## Optional tax rebase + +Pass `rebase_property_taxes=True` together with `property_income=options` to +append the deterministic receiving version, tax split and numerical gate. The +default is `False`; requesting a rebase without property options refuses before +source I/O. This path has 38 nodes. It retains the complete legacy financial +population and the complete 35-node property population separately from the +final `survey_property.tax_receiving` version. + +The rebase uses ordinary interest and dividends with the maintained fractions +and subtraction complements. Separate missing O/D inputs produce missing tax +leaves, replacing earlier predictions; retirement-account interest remains +auxiliary. A new FILTER conservation record is intentional, while existing +Frame mass history and all non-rebased cells remain unchanged. See the +[numerical fragment contract](us-property-tax-leaves.md). + +The host constructs the declarations from the checked clone Frame and the +existing financial/property output descriptors. It never supplies fake values +for future columns. After the real 35-node output is available, it independently +re-declares and executes the three deterministic operations, checks stored +artifact bytes and complete node receipts, and compares every expected full +population with actual cold or cached observations. The issued run retains +the source, declaration, model, artifact, intermediate population and final +population seals. Later checks reconstruct the same tax operations before the +final source requalification and pure lifetime fence. + +An incomplete numerical gate is valid development evidence. The run document +reports `tax_split_rebased: true`, three extra nodes and the actual +`tax_leaf_complete` value; it remains `release_eligible: false`. PUF recipient +qualification explicitly requires a complete independently checked numerical +gate before it reads its additional source inputs. Both PUF recipient graph +nodes carry a typed edge to that gate on the actual receiving version. The +19- and 35-node paths do not add that edge or claim a tax rebase. + +No child zero completion or missing-adult model is introduced. Consequently the +invented source fixture with under-15 and missing-anchor cases must remain a +development run and refuse full PUF qualification. Capital gains retain their +earlier conditioning; rebasing interest and dividends does not update that model. + +The bounded invented-source acceptance covers 11 host controls, including actual +cold and required-cache 38-node runs, issued-view parity, intermediate/final +population preservation, mutation refusals and incomplete PUF qualification. +The numerical fragment has a separate 40-control acceptance, including a +complete-input positive gate and descriptor-only declaration checks. These +checks do not use native survey data, run a country engine or execute full PUF. diff --git a/docs/us-property-tax-leaves.md b/docs/us-property-tax-leaves.md index 93186359b..afc520911 100644 --- a/docs/us-property-tax-leaves.md +++ b/docs/us-property-tax-leaves.md @@ -41,6 +41,16 @@ host supplies their actual producer IDs and nominal types. Register `PropertyTaxReceivingKernel`, `PropertyTaxLeavesKernel` and `PropertyTaxLeafGateKernel`. +When compiling before the earlier financial/property nodes execute, the host +may pass `anticipated_outputs=tuple_of_owned_descriptors`. The factory combines +those existing declarations with the real checked Frame's entity/axis/group +shapes and dtypes. It creates no placeholder values or synthetic population. +It rejects duplicate descriptors, unknown entities, structural ID fields, +masked outputs and incompatible or unclaimed rewrites. At execution the host +re-declares the nodes from the actual complete post-property Frame and requires +the same normative declarations before reconstructing the results. The default +empty descriptor tuple still requires a complete actual input Frame. + 1. `survey_property.tax_receiving` opens a new population version using FILTER-all. This inherits the prior owners and permits explicit replacement of the four incumbent leaves. The executor adds a real FILTER conservation record diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py index 21de59046..4fbd50c84 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -20,11 +20,23 @@ from microcosm.fit import qrf_target from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel -from microcosm.graph import KernelResult, StructuralDelta, compile_graph, run_graph +from microcosm.graph import ( + ArtifactInput, + ArtifactValue, + KernelResult, + StructuralDelta, + compile_graph, + run_graph, +) from microcosm.graph import population as population_ops -from microcosm.graph.artifact_edges import typed_contracts -from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys -from microcosm.graph.keys import _capabilities_projection +from microcosm.graph.artifact_edges import numeric_scope, typed_contracts +from microcosm.graph.executor import ( + _all_node_keys, + _apply_result, + _project_context, + _source_paths_and_keys, +) +from microcosm.graph.keys import _capabilities_projection, opaque_artifact_key from microcosm.graph.population import Population from microcosm.graph.serialize import graph_to_json @@ -101,6 +113,10 @@ class _FinancialRunState: property_income_bytes: bytes | None = None legacy_financial_population: Population | None = None legacy_financial_stamp: object = None + rebase_property_taxes: bool = False + property_population: Population | None = None + property_population_stamp: object = None + tax_verification: bytes | None = None def _property_module(): @@ -110,6 +126,120 @@ def _property_module(): return graph_current_survey_property +def _tax_module(): + from . import graph_property_tax_leaves + + return graph_property_tax_leaves + + +def _tax_nodes(frame, population, options, *, anticipated_outputs=()): + property_graph = _property_module() + reconciliation = options.nodes((property_graph.PROPERTY_REPORTED_TOTAL,))[-1] + output = reconciliation.artifact_outputs[0] + return _tax_module().property_tax_leaf_nodes( + frame, + population=population, + projection=ArtifactInput( + "projection", + property_graph.PROJECTION_NODE, + "projection", + property_graph.PROJECTION_TYPE, + ), + reconciliation=ArtifactInput( + "reconciliation", + reconciliation.id, + output.name, + output.type, + ), + atol=float(options.atol), + rtol=float(options.rtol), + anticipated_outputs=anticipated_outputs, + ) + + +def _reconstruct_tax(population, *, compiled, kernels, keys, loaded, options): + """Replay three deterministic operations on the actual complete post35 Frame.""" + tax = _tax_module() + nodes = _tax_nodes(population.frame, population.version, options) + classes = ( + tax.PropertyTaxReceivingKernel, + tax.PropertyTaxLeavesKernel, + tax.PropertyTaxLeafGateKernel, + ) + populations, results = {}, {} + for node, cls in zip(nodes, classes, strict=True): + require( + node.normative() == compiled.graph.node(node.id).normative(), + "PROPERTY_TAX_DECLARATION", + ) + artifacts = { + edge.name: ArtifactValue( + loaded[edge.producer, edge.artifact], + edge.type, + opaque_artifact_key(keys[edge.producer], edge.artifact), + keys[edge.producer], + numeric_scope( + kernels.get(compiled.graph.node(edge.producer).kernel).capabilities + ), + ) + for edge in node.artifact_inputs + } + context = _project_context( + node, + population, + key=keys[node.id], + sources={}, + tolerances={}, + numerics={}, + artifacts=artifacts, + ) + result = cls().run(context) + require( + all( + loaded[node.id, name] == payload + for name, payload in result.artifacts.items() + ), + "PROPERTY_TAX_ARTIFACT", + ) + population = _apply_result( + node, + result, + population, + mass_partition=compiled.graph.mass_partition, + ) + populations[node.id], results[node.id] = population, result + return populations, results + + +def financial_tax_gate_edges(run): + """Declare the real numerical gate only for an issued opt-in run.""" + state = _run_entry(run)[2] + if not state.rebase_property_taxes: + return () + tax = _tax_module() + return ( + ArtifactInput( + "property_tax_verification", + tax.GATE_NODE, + "verification", + tax.VERIFICATION_TYPE, + ), + ) + + +def require_complete_property_taxes(run): + """Require the already checked numerical gate before PUF source qualification.""" + entry = _run_entry(run) + _pure_run(run, entry) + state = entry[2] + if state.rebase_property_taxes: + document = codec.decode_json(state.tax_verification) + require( + document["numeric_verified"] is True and document["complete"] is True, + "PROPERTY_TAX_INPUTS_INCOMPLETE", + ) + + def _manifest_population_seals(manifest, compiled): """Include transient attached Frames, which portable JSON deliberately omits.""" return tuple( @@ -141,6 +271,8 @@ def _run_entry(run): def financial_output_node(run): """Name the final writer of an already issued financial run.""" state = _run_entry(run)[2] + if state.rebase_property_taxes: + return _tax_module().GATE_NODE return ( financial.ATTACH_NODE if state.property_income is None @@ -187,13 +319,28 @@ def _run_document(run, state): else { "property_income": codec.decode_json(state.property_income_bytes), "property_node_count": 16, - "tax_split_rebased": False, + "tax_split_rebased": state.rebase_property_taxes, "capital_gains_conditioning": _property_module().CAP_LIMITATION, "legacy_financial_frame_sha256": values.source._frame_identity( state.legacy_financial_population.frame ), } ), + **( + {} + if not state.rebase_property_taxes + else { + "tax_rebase_node_count": 3, + "tax_rebased_columns": _tax_module().TAX_LEAF_COLUMNS, + "tax_verification_sha256": codec.sha(state.tax_verification), + "property_frame_sha256": values.source._frame_identity( + state.property_population.frame + ), + "tax_leaf_complete": codec.decode_json(state.tax_verification)[ + "complete" + ], + } + ), } ) @@ -261,6 +408,13 @@ def _pure_run(run, entry): == state.legacy_financial_stamp, "PROPERTY_OPTIONS_OR_LEGACY_POPULATION_CHANGED", ) + if state.rebase_property_taxes: + require( + state.property_income is not None + and reconstruction._population_stamp(state.property_population) + == state.property_population_stamp, + "PROPERTY_TAX_PARENT_POPULATION_CHANGED", + ) for name, population, stamp in state.populations: actual = ( run.financial_population if name == "financial" else getattr(prefix, name) @@ -271,7 +425,7 @@ def _pure_run(run, entry): ) require( _run_document(run, state) == entry[1] - and _live(state.property_income) == state.live, + and _live(state.property_income, state.rebase_property_taxes) == state.live, "FINAL_FINANCIAL_RUN_SEAL", ) require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") @@ -303,6 +457,19 @@ def check_atomic_survey_financial_run(run): == state.artifact_hashes, "FINANCIAL_RUN_ARTIFACT_CHANGED", ) + if state.rebase_property_taxes: + tax_populations, _ = _reconstruct_tax( + state.property_population, + compiled=run.compiled, + kernels=run.kernels, + keys=keys, + loaded=loaded, + options=state.property_income, + ) + atomic.same_replayed_population( + tax_populations[_tax_module().GATE_NODE], + run.financial_population, + ) # These exact fitted artifacts were checked at actual execution/required # replay before issuance. Rechecking their identities needs no new pickle # decode or fit; the materialized verifier freshly derives current values. @@ -335,7 +502,11 @@ def check_atomic_survey_financial_run(run): prefix.allocated_population, prefix.clone_population, legacy_population=state.legacy_financial_population, - population=run.financial_population, + population=( + state.property_population + if state.rebase_property_taxes + else run.financial_population + ), host_pins=codec.decode_json(state.pins), options=state.property_income, artifacts=loaded, @@ -364,6 +535,8 @@ def _issue_run( live, property_income=None, legacy_financial_population=None, + rebase_property_taxes=False, + property_population=None, ): """Called only after this runner's complete materialization/replay checks.""" prefix = result.prefix @@ -430,6 +603,14 @@ def _issue_run( None if legacy_financial_population is None else reconstruction._population_stamp(legacy_financial_population), + rebase_property_taxes, + property_population, + None + if property_population is None + else reconstruction._population_stamp(property_population), + None + if not rebase_property_taxes + else loaded[_tax_module().GATE_NODE, "verification"], ) identifier = id(result) @@ -443,7 +624,7 @@ def forget(reference): _pure_run(result, _run_entry(result)) -def _live(property_income=None): +def _live(property_income=None, rebase_property_taxes=False): """Pure final fence over this composition and its existing owner closure.""" result = dict(values.host.survey_budget._live()) modules = ( @@ -492,6 +673,24 @@ def _live(property_income=None): current_asec_property_basis.OTHER_UNSPECIFIED_CATEGORY, ) ) + if rebase_property_taxes: + tax = _tax_module() + modules = (*modules, tax, tax.cps_carried) + result["property_tax_contract"] = values.source._runtime_marker( + ( + tax.PROTOCOL, + tax.RECEIVING_NODE, + tax.TAX_LEAVES_NODE, + tax.GATE_NODE, + tax.TAX_LEAF_COLUMNS, + tax.DIAGNOSTICS_TYPE, + tax.VERIFICATION_TYPE, + tax.PROPERTY_COMPONENTS, + tax._INPUTS, + tax.cps_carried.TAXABLE_INTEREST_FRACTION, + tax.cps_carried.QUALIFIED_DIVIDEND_FRACTION, + ) + ) for module in modules: for name, value in vars(module).items(): if isinstance(value, FunctionType): @@ -662,11 +861,17 @@ def run_atomic_survey_financial( demographic_conditioning=False, n_estimators=100, property_income=None, + rebase_property_taxes=False, resume="auto", return_values=False, ): """Verify the base financial graph and its explicitly selected extension.""" require(type(return_values) is bool, "RETURN_VALUES_FLAG") + require(type(rebase_property_taxes) is bool, "PROPERTY_TAX_FLAG") + require( + not rebase_property_taxes or property_income is not None, + "PROPERTY_TAX_REQUIRES_PROPERTY", + ) values.feature_columns(demographic_conditioning) property_graph = None if property_income is None else _property_module() if property_graph is not None: @@ -677,7 +882,7 @@ def run_atomic_survey_financial( property_income_bytes = property_income.to_bytes() else: property_income_bytes = None - live = _live(property_income) + live = _live(property_income, rebase_property_taxes) config_bytes = values.host.survey_budget._config_payload(geography_config) require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") prefix = atomic.run_atomic_survey_population( @@ -766,16 +971,30 @@ def run_atomic_survey_financial( options=property_income, ) ) + tax_nodes = ( + () + if not rebase_property_taxes + else _tax_nodes( + prefix.clone_population.frame, + prefix.clone_population.version, + property_income, + anticipated_outputs=( + *next(n.outputs for n in nodes if n.id == financial.ATTACH_NODE), + *property_graph.owned_columns(), + ), + ) + ) compiled = compile_graph( replace( prefix.compiled.graph, - nodes=(*prefix.compiled.graph.nodes, *nodes, *property_nodes), + nodes=(*prefix.compiled.graph.nodes, *nodes, *property_nodes, *tax_nodes), ) ) require( len(prefix.compiled.order) == 9 and len(property_nodes) == (0 if property_graph is None else 16) - and len(compiled.order) == 19 + len(property_nodes), + and len(tax_nodes) == (3 if rebase_property_taxes else 0) + and len(compiled.order) == 19 + len(property_nodes) + len(tax_nodes), "ATOMIC_COMPILER_ROSTER", ) gate_edge = financial._geography_edge() @@ -818,6 +1037,14 @@ def run_atomic_survey_financial( demographic_conditioning=demographic_conditioning, geography_config=geography_config, ) + if rebase_property_taxes: + tax = _tax_module() + for cls in ( + tax.PropertyTaxReceivingKernel, + tax.PropertyTaxLeavesKernel, + tax.PropertyTaxLeafGateKernel, + ): + kernels.register(cls()) _, source_keys = _source_paths_and_keys(compiled, sources, store) keys, implementations = _all_node_keys(compiled, kernels, source_keys) base_expected = { @@ -920,10 +1147,25 @@ def observe(node_id, population): ) ) receipts.update({name: result.receipt for name, result in property_results.items()}) - expected, current = {}, {} + expected, current, tax_populations = {}, {}, {} for node_id in compiled.order: node, version = compiled.graph.node(node_id), compiled.versions[node_id] - if node_id in property_results: + if rebase_property_taxes and node_id == _tax_module().RECEIVING_NODE: + tax_populations, tax_results = _reconstruct_tax( + current[node.base], + compiled=compiled, + kernels=kernels, + keys=keys, + loaded=loaded, + options=property_income, + ) + receipts.update( + {name: result.receipt for name, result in tax_results.items()} + ) + population = tax_populations[node_id] + elif node_id in tax_populations: + population = tax_populations[node_id] + elif node_id in property_results: incumbent = ( version if node.structural is StructuralDelta.NONE else node.base ) @@ -979,6 +1221,9 @@ def observe(node_id, population): final_node = ( financial.ATTACH_NODE if property_graph is None else property_graph.ATTACH_NODE ) + property_population = observed[final_node] if rebase_property_taxes else None + if rebase_property_taxes: + final_node = _tax_module().GATE_NODE legacy_population = observed[financial.ATTACH_NODE] result = AtomicSurveyFinancialRunValues( prefix, @@ -1020,7 +1265,11 @@ def observe(node_id, population): prefix.allocated_population, prefix.clone_population, legacy_population=legacy_population, - population=result.financial_population, + population=( + property_population + if rebase_property_taxes + else result.financial_population + ), host_pins=pins, options=property_income, artifacts=loaded, @@ -1068,7 +1317,7 @@ def observe(node_id, population): and tuple(sorted(prefix.sources.items())) == source_items and graph_to_json(compiled.graph) == declaration and compiled == compile_graph(compiled.graph) - and _live(property_income) == live, + and _live(property_income, rebase_property_taxes) == live, "ATOMIC_FINAL_BINDINGS", ) for name, (population, stamp) in retained.items(): @@ -1117,5 +1366,7 @@ def observe(node_id, population): legacy_financial_population=None if property_graph is None else legacy_population, + rebase_property_taxes=rebase_property_taxes, + property_population=property_population, ) return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py index 177167a4a..43fb40247 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_property_tax_leaves.py @@ -32,6 +32,7 @@ StructuralDelta, source_hash, ) +from microcosm.graph.decl import ROWS_ALL from microcosm.graph.keys import opaque_artifact_key from . import cps_carried @@ -138,15 +139,50 @@ def split_property_tax_leaves(person: pd.DataFrame) -> pd.DataFrame: return pd.DataFrame(columns, index=index) -def _slices(frame): +def _slices(frame, anticipated_outputs=()): _require(type(frame) is Frame and frame.schema.person_entity == "person", "FRAME") _require(not frame.links, "UNSUPPORTED_LINKS") - _person(frame.person) + _require(type(anticipated_outputs) is tuple, "ANTICIPATED_OUTPUTS") + dtypes = { + entity: {c: str(frame.table(entity)[c].dtype) for c in frame.table(entity)} + for entity in frame.entities + } + seen = set() + for owned in anticipated_outputs: + _require( + type(owned) is Owned + and owned.entity in dtypes + and not owned.column.endswith("_id") + and owned.rows == ROWS_ALL, + "ANTICIPATED_OUTPUT", + ) + coordinate = (owned.entity, owned.column) + _require(coordinate not in seen, "DUPLICATE_ANTICIPATED_OUTPUT") + seen.add(coordinate) + existing = dtypes[owned.entity].get(owned.column) + _require( + (existing is None and not owned.rewrite) + or (owned.rewrite and existing == owned.dtype), + "CONFLICTING_ANTICIPATED_OUTPUT", + ) + dtypes[owned.entity][owned.column] = owned.dtype + if not anticipated_outputs: + _person(frame.person) + else: + _require( + frame.person.person_id.dtype == np.dtype("int64") + and frame.person.person_id.is_unique, + "PERSON_IDENTITY", + ) + _require( + all(dtypes["person"].get(c) == "float64" for c in _INPUTS), + "ANTICIPATED_INPUT_DTYPE", + ) _require(len(frame.person) > 0, "EMPTY_RECEIVING_FRAME") result = [] for entity in frame.entities: table = frame.table(entity) - columns = tuple(c for c in table if not c.endswith("_id")) + columns = tuple(c for c in dtypes[entity] if not c.endswith("_id")) _require(bool(columns), "NO_READABLE_ENTITY_COLUMN:" + entity) if entity != "person": _require( @@ -161,7 +197,7 @@ def _slices(frame): result.append(Slice(entity, columns)) for name in TAX_LEAF_COLUMNS: _require( - name in frame.person and frame.person[name].dtype == np.dtype("float64"), + dtypes["person"].get(name) == "float64", "INCUMBENT_LEAF:" + name, ) return tuple(result) @@ -223,6 +259,7 @@ def property_tax_leaf_nodes( reconciliation: ArtifactInput, atol: float, rtol: float, + anticipated_outputs: tuple[Owned, ...] = (), ) -> tuple[Node, Node, Node]: """Declare an explicit receiving version, four rewrites and numeric gate. @@ -231,7 +268,14 @@ def property_tax_leaf_nodes( those shapes before execution. Group tables need a declared non-ID column so the receiving kernel can check their actual IDs through KernelContext. """ - return _nodes(population, _slices(frame), projection, reconciliation, atol, rtol) + return _nodes( + population, + _slices(frame, anticipated_outputs), + projection, + reconciliation, + atol, + rtol, + ) def _bindings(context): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py index 87ee4b942..5ef6435b9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py @@ -42,7 +42,7 @@ } -def _edges(): +def _edges(run): return ( ArtifactInput( "preparation", @@ -62,6 +62,7 @@ def _edges(): "matrix", model_input.RECIPIENT_MATRIX_TYPE, ), + *financial.financial_tax_gate_edges(run), ) @@ -75,7 +76,7 @@ def _pins(run): "artifact_key": opaque_artifact_key(keys[edge.producer], edge.artifact), "payload_sha256": hashes[edge.producer, edge.artifact], } - for edge in _edges() + for edge in _edges(run) } @@ -138,7 +139,7 @@ def puf55_survey_recipient_nodes(qualified): PROJECTION_NODE, Puf55SurveyRecipientProjectionKernel.ref, **common, - artifact_inputs=_edges(), + artifact_inputs=_edges(qualified.financial_run), artifact_outputs=(ArtifactOutput("projection", PROJECTION_TYPE),), ) matrix = Node( @@ -146,7 +147,7 @@ def puf55_survey_recipient_nodes(qualified): Puf55SurveyRecipientMatrixKernel.ref, **common, artifact_inputs=( - *_edges(), + *_edges(qualified.financial_run), ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE), ), artifact_outputs=tuple( @@ -230,7 +231,9 @@ def _context(self, context, qualified): "financial_projection": entry[2].projection, "financial_matrix": entry[2].matrix, } - for edge in _edges(): + if entry[2].rebase_property_taxes: + payloads["property_tax_verification"] = entry[2].tax_verification + for edge in _edges(qualified.financial_run): artifact = host.shared.artifact(context, edge.name, edge.type) values._require( { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py index 7f3965be2..527480ffb 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py @@ -480,6 +480,7 @@ def qualify_puf55_survey_recipients(financial_run): Source receipt objects and caller-supplied role tables are not accepted. """ financial.check_atomic_survey_financial_run(financial_run) + financial.require_complete_property_taxes(financial_run) entry = financial._run_entry(financial_run) state = entry[2] preparation = state.prefix.preparation diff --git a/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py new file mode 100644 index 000000000..1c34772f3 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_atomic_property_tax_financial.py @@ -0,0 +1,156 @@ +"""Optional 38-node financial host over actual invented source owners.""" + +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_atomic_property_financial import property_host_source # noqa: F401 + +from microcosm.build.us_runtime import graph_atomic_survey_financial as runner +from microcosm.build.us_runtime import graph_property_tax_leaves as tax +from microcosm.build.us_runtime import graph_puf55_survey_recipients as recipients + + +@pytest.fixture(scope="module") +def tax_host_cold(request): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=case.options, + rebase_property_taxes=True, + resume="auto", + ) + + +@pytest.fixture(scope="module") +def tax_host_warm(request, tax_host_cold): + case = request.getfixturevalue("property_host_source") + return runner.run_atomic_survey_financial( + **case.call, + property_income=case.options, + rebase_property_taxes=True, + resume="require", + ) + + +def test_opt_in_cold_required_issued_output(tax_host_cold, tax_host_warm): + cold, warm = tax_host_cold, tax_host_warm + assert len(cold.compiled.order) == len(warm.compiled.order) == 38 + assert cold.manifest.key == warm.manifest.key + assert all(record.hit for record in warm.manifest.nodes.values()) + runner.atomic.same_replayed_population( + cold.financial_population, warm.financial_population + ) + checked = cold.checked_view() + assert checked.payload == warm.checked_view().payload + document = runner.codec.decode_json(checked.payload) + assert document["tax_split_rebased"] is True + assert document["tax_rebase_node_count"] == 3 + assert document["tax_leaf_complete"] is False + assert document["release_eligible"] is False + assert runner.financial_output_node(cold) == tax.GATE_NODE + assert cold.financial_population.version == tax.RECEIVING_NODE + + +def test_opt_in_preserves_full_prior_populations_and_unknowns(tax_host_cold): + run = tax_host_cold + state = runner._run_entry(run)[2] + legacy, before, after = ( + state.legacy_financial_population, + state.property_population, + run.financial_population, + ) + assert before.version == legacy.version != after.version + for entity in before.frame.entities: + kept = [ + name + for name in before.frame.table(entity) + if name not in tax.TAX_LEAF_COLUMNS + ] + pd.testing.assert_frame_equal( + before.frame.table(entity)[kept], + after.frame.table(entity)[kept], + check_exact=True, + ) + assert before.frame.schema == after.frame.schema + assert before.frame.metadata == after.frame.metadata + assert before.frame.mass_log == after.frame.mass_log + assert after.mass_ledger[:-1] == before.mass_ledger + assert after.mass_ledger[-1].node_id == tax.RECEIVING_NODE + pd.testing.assert_series_equal(before.frame.strata, after.frame.strata) + for entity in before.design_weights: + np.testing.assert_array_equal( + before.design_weights[entity], after.design_weights[entity] + ) + expected = tax.split_property_tax_leaves(before.frame.person) + for name in tax.TAX_LEAF_COLUMNS: + np.testing.assert_array_equal( + after.frame.person[name].to_numpy().view("uint64"), + expected[name].to_numpy().view("uint64"), + ) + assert after.frame.person[list(tax.TAX_LEAF_COLUMNS)].isna().any().any() + assert legacy.frame.person[list(tax.TAX_LEAF_COLUMNS)].notna().all().all() + + +def test_incomplete_development_run_refuses_puf_before_source_qualification( + tax_host_cold, +): + run = tax_host_cold + assert ( + runner.codec.decode_json(runner._run_entry(run)[2].tax_verification)["complete"] + is False + ) + with pytest.raises(ValueError, match="PROPERTY_TAX_INPUTS_INCOMPLETE"): + recipients.values.qualify_puf55_survey_recipients(run) + edges = recipients._edges(run) + edge = next(e for e in edges if e.name == "property_tax_verification") + assert edge.producer == tax.GATE_NODE and edge.type == tax.VERIFICATION_TYPE + assert recipients._pins(run)[edge.name]["payload_sha256"] == runner.codec.sha( + runner._run_entry(run)[2].tax_verification + ) + + +@pytest.mark.parametrize("target", ["property", "leaf", "unknown"]) +def test_retained_tax_lifetime_refuses_changes(tax_host_cold, target): + run = tax_host_cold + state = runner._run_entry(run)[2] + population = ( + state.property_population if target == "property" else run.financial_population + ) + table = population.frame.person + column = ( + tax.PROPERTY_COMPONENTS[0] if target == "property" else tax.TAX_LEAF_COLUMNS[0] + ) + row = table.index[ + table[column].isna() if target == "unknown" else table[column].notna() + ][0] + old = table.loc[row, column] + table.loc[row, column] = 17.0 + try: + with pytest.raises(ValueError, match="POPULATION_CHANGED"): + run.checked_view() + finally: + table.loc[row, column] = old + runner._pure_run(run, runner._run_entry(run)) + + +def test_rebased_dataclass_copy_remains_unissued(tax_host_cold): + with pytest.raises(ValueError, match="UNISSUED_FINANCIAL_RUN"): + replace(tax_host_cold).checked_view() + + +@pytest.mark.parametrize("flag", [None, 1, "yes", True]) +def test_invalid_or_unconfigured_rebase_refuses_before_source_io(flag): + with pytest.raises( + ValueError, match="PROPERTY_TAX_FLAG|PROPERTY_TAX_REQUIRES_PROPERTY" + ): + runner.run_atomic_survey_financial( + None, + snapshot_root=None, + store_root=None, + fraction=1.0, + seed=17, + geography_config=None, + rebase_property_taxes=flag, + ) diff --git a/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py b/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py index 5ee886508..c5471f2df 100644 --- a/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py +++ b/packages/microcosm-build/tests/test_us_graph_property_tax_leaves.py @@ -495,3 +495,83 @@ def test_receiving_rechecks_membership_after_declaration(owner, actual): person.loc[person.person_household_id == 304, "person_household_id"] = 303 with pytest.raises(ValueError, match="ORPHAN_GROUP:household"): owner.PropertyTaxReceivingKernel().run(replace(context, tables=tables)) + + +def test_anticipated_outputs_declare_same_graph_without_invented_values(owner): + complete = _supported_frame(owner) + added = (*owner.PROPERTY_COMPONENTS[:3], *owner.TAX_LEAF_COLUMNS) + source = Frame( + { + "person": complete.person.drop(columns=list(added)), + "household": complete.table("household"), + }, + complete.schema, + {"household": complete.weights_for("household")}, + ) + before = source.person.copy(deep=True) + descriptors = tuple(Owned("person", name, "float64") for name in added) + nodes = owner.property_tax_leaf_nodes( + source, + population="source", + projection=ArtifactInput("projection", "source", "projection", PROJECTION), + reconciliation=ArtifactInput( + "reconciliation", "source", "reconciliation", RECONCILIATION + ), + atol=1e-10, + rtol=1e-12, + anticipated_outputs=descriptors, + ) + # Actual appended column order defines the same declared receiving slices. + restored = Frame( + { + "person": pd.concat([source.person, complete.person[list(added)]], axis=1), + "household": complete.table("household"), + }, + complete.schema, + {"household": complete.weights_for("household")}, + ) + assert [n.normative() for n in nodes] == [ + n.normative() for n in declaration(owner, restored) + ] + pd.testing.assert_frame_equal(source.person, before, check_exact=True) + assert not set(added) & set(source.person) + + +@pytest.mark.parametrize( + "defect", + [ + "container", + "duplicate", + "entity", + "structural", + "mask", + "conflict", + "unclaimed_rewrite", + ], +) +def test_anticipated_output_descriptor_refusal(owner, defect): + source = _supported_frame(owner) + column = Owned("person", "anticipated", "float64") + entries = { + "container": [column], + "duplicate": (column, column), + "entity": (Owned("absent", "anticipated", "float64"),), + "structural": (Owned("person", "other_id", "int64"),), + "mask": (Owned("person", "anticipated", "float64", rows="subset"),), + "conflict": ( + Owned("person", owner.TAX_LEAF_COLUMNS[0], "int64", rewrite=True), + ), + "unclaimed_rewrite": (Owned("person", "anticipated", "float64", rewrite=True),), + }[defect] + with pytest.raises(ValueError, match="ANTICIPATED"): + owner.property_tax_leaf_nodes( + source, + population="source", + projection=ArtifactInput("projection", "source", "projection", PROJECTION), + reconciliation=ArtifactInput( + "reconciliation", "source", "reconciliation", RECONCILIATION + ), + atol=1e-10, + rtol=1e-12, + anticipated_outputs=entries, + ) From 79cde4010775ca8082fd92c55db43220284f3741 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:58:14 -0400 Subject: [PATCH 187/255] Provide engine-free HDF readers in the installed wheel gate --- .github/workflows/test.yml | 41 ++++++++++++---------- CLAUDE.md | 5 +++ changelog.d/source-io-wheel-extra.fixed.md | 1 + packages/microcosm-build/README.md | 5 +++ packages/microcosm-build/pyproject.toml | 2 ++ uv.lock | 8 ++++- 6 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 changelog.d/source-io-wheel-extra.fixed.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86d472a21..86fcbf208 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -400,34 +400,38 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk - - name: Add the engine-free US extra the source runtimes require - # microunit is microcosm-frame's `us` extra: the sanctioned tax-unit - # constructor, and the only optional dependency the US source runtimes - # need without a country engine. Every ACS/CREATE stage names it in - # us_runtime.graph_implementation.STAGE_DEPENDENCIES, so an installed - # constellation without it cannot hash those stages: the suite then - # fails closed (SOURCE_PREPARATION_REFUSED / CATALOGUE_ISSUANCE_REFUSED) - # instead of skipping, which is a false packaging verdict. - # - # This runs after the boundary check above, so that check still proves - # the base wheels import with no optional dependency installed at all. - # Requesting the extra against the built wheel — rather than naming - # `microunit` here — keeps the extra declared only in - # packages/microcosm-frame/pyproject.toml and makes this gate prove the - # wheel carries resolvable extra metadata. policyengine-us stays absent: - # it is declared by microcosm-build[us] and microcosm-frame[policyengine], - # and neither is requested. + - name: Add engine-free source preparation extras + # Real source fixtures require microunit for tax-unit construction and + # PyTables for pandas HDF input, alongside h5py. Request the extras from + # built wheels so their metadata, not editable installs, supplies these + # dependencies. The preceding step still checks the base wheel boundary; + # neither source extra installs a country rules engine. run: | frame_wheel="$(ls dist/microcosm_frame-*.whl)" + build_wheel="$(ls dist/microcosm_build-*.whl)" uv pip install --python /tmp/wheels-venv/bin/python \ --constraint /tmp/wheel-constraints.txt \ - "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" \ + "microcosm-build[source-io] @ file://$(realpath "$build_wheel")" env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' from importlib import metadata, util + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import pandas as pd + import tables assert util.find_spec("microunit") is not None, ( "microcosm-frame[us] must install microunit into the wheel venv" ) + # Exercise the same HDF reader used by source preparation. Imports + # alone do not establish that the compiled PyTables backend works. + with TemporaryDirectory() as directory: + path = Path(directory) / "invented-source.h5" + expected = pd.DataFrame({"person_id": [1, 2], "amount": [0.0, 12.5]}) + expected.to_hdf(path, key="person") + pd.testing.assert_frame_equal(pd.read_hdf(path, key="person"), expected) assert util.find_spec("policyengine_us") is None try: metadata.distribution("policyengine-us") @@ -436,6 +440,7 @@ jobs: else: raise AssertionError("policyengine-us must be absent from the base wheel gate") print("microunit", metadata.version("microunit")) + print("h5py", h5py.__version__, "tables", tables.__version__) PY - name: Run the suite against the installed wheels run: | diff --git a/CLAUDE.md b/CLAUDE.md index d5e44c2a3..445c9456e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,6 +39,11 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert the wheel/import boundary and spec digests, and run the suite against installed wheels. +After checking the base wheel boundary, the wheel lane installs the built +`microcosm-frame[us]` and `microcosm-build[source-io]` extras for engine-free +source tests. An invented pandas HDF round trip checks the compiled reader; +the country rules engine must still be absent. + The shared engine lane also runs a bounded source/seed identity diagnostic on Python 3.14.4 after its normal tests, including when those tests fail. Its six JSON artifacts retain candidate digests and their complete canonical seed diff --git a/changelog.d/source-io-wheel-extra.fixed.md b/changelog.d/source-io-wheel-extra.fixed.md new file mode 100644 index 000000000..9e8232f55 --- /dev/null +++ b/changelog.d/source-io-wheel-extra.fixed.md @@ -0,0 +1 @@ +Declare an engine-free HDF source extra and install it in the isolated wheel test environment. Verify an invented pandas HDF round trip before source tests, while retaining the base wheel import boundary and excluding the US rules engine. diff --git a/packages/microcosm-build/README.md b/packages/microcosm-build/README.md index dcc65be05..d2ea129cd 100644 --- a/packages/microcosm-build/README.md +++ b/packages/microcosm-build/README.md @@ -49,6 +49,11 @@ The `us` extra adds the rules engine for formula/export checks. Country source loaders are not Python dependencies: source stages are declared in packaged JSON manifests and executed by shared Microcosm runtimes. +The `source-io` extra installs h5py and PyTables for HDF survey input without a +country rules engine. US source preparation also uses `microcosm-frame[us]` +for tax-unit construction. These extras provide readers and constructors; +they do not download or authorize any source data. + Country namespaces under `microcosm.build.us` and `microcosm.build.uk` are resource packages only. They may contain specs and data artifacts, but no Python modules; guard tests enforce this so country content stays declarative. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 8f64f0a5d..e9ee60538 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ ] [project.optional-dependencies] +# Survey HDF inputs can be prepared without installing a country rules engine. +source-io = ["h5py>=3", "tables>=3"] # The US extra adds the rules engine for formula-owned export checks. Source # content is declared in packaged manifests and interpreted by shared Microcosm # runtimes; country source loaders must not depend on incumbent data packages. diff --git a/uv.lock b/uv.lock index c2da5a850..2b3a768ae 100644 --- a/uv.lock +++ b/uv.lock @@ -748,6 +748,10 @@ dependencies = [ ] [package.optional-dependencies] +source-io = [ + { name = "h5py" }, + { name = "tables" }, +] uk = [ { name = "h5py" }, { name = "openpyxl" }, @@ -769,6 +773,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "h5py", marker = "extra == 'source-io'", specifier = ">=3" }, { name = "h5py", marker = "extra == 'uk'", specifier = ">=3" }, { name = "h5py", marker = "extra == 'us'", specifier = ">=3" }, { name = "huggingface-hub", specifier = ">=0.20" }, @@ -788,10 +793,11 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6" }, { name = "referencing", specifier = ">=0.35,<1" }, { name = "scipy", specifier = ">=1.13" }, + { name = "tables", marker = "extra == 'source-io'", specifier = ">=3" }, { name = "tables", marker = "extra == 'uk'", specifier = ">=3" }, { name = "tables", marker = "extra == 'us'", specifier = ">=3" }, ] -provides-extras = ["us", "uk"] +provides-extras = ["source-io", "us", "uk"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=8" }] From 8ebce1c8e27f8b3129080c0801032809c856744d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:02:11 -0400 Subject: [PATCH 188/255] Record checked property tax integration and native build status --- docs/us-uk-release-path.md | 19 ++++- .../us-launch-consolidation-20260913.md | 77 +++++++++++++++++++ .../tests/test_us_spine_blindness.py | 5 ++ 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 experiments/us-launch-consolidation-20260913.md diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md index aae4317e7..d6db94236 100644 --- a/docs/us-uk-release-path.md +++ b/docs/us-uk-release-path.md @@ -97,8 +97,13 @@ not establish completed native financial acceptance. The corrected v3 harness passes two actual invented financial cold/replay controls and thirteen bounded source and diagnostic controls. After independent review of its exact packet, one fresh native run started with unchanged calculation source, native inputs -and resource limits. Its result is pending; no retry pass or native PUF result -is claimed. +and resource limits. On 13 September, the owned process was no longer present +and neither a final receipt nor a PUF manifest existed. Its last recorded +progress was about 5.76 hours wall time, 6.41 hours process CPU and 28.29 GB peak +RSS. The exit code and stopping cause are unavailable; the CPU total exceeding +the configured budget does not establish that cause. This attempt is closed +without verified native PUF output. A successor requires a fresh reviewed +packet; the preceding source, input and failure evidence remain preserved. The maintained financial graph now retains an eighth field: the tax-exempt interest remainder of its existing observed or modeled total. The split remains @@ -203,10 +208,16 @@ pass. An explicit property-income option now integrates the source bridge, four conditional component models, signed reconciliation and clone attachment into the existing financial host. Its actual invented 35-node construction and required replay pass, preserving the full preceding population. The new columns -still coexist with the earlier tax inputs: the tax-input rebase, retirement -bridge and complete PUF/native adoption remain separate work. The focused PUF +remain available beside the retained earlier population. A further explicit +three-node extension rebases four interest/dividend tax inputs and checks their +conservation and completeness. Its 38-node cold/required host passes eleven +invented-source controls; unknown inputs remain unknown and an incomplete gate +refuses PUF qualification. Retirement completion and complete PUF/native adoption +remain separate work. The focused PUF handoff retest passes after correction of an invalid invented source record; see the [scoped host evidence](../experiments/us-property-financial-host-20260912.md). +The [13 September integration record](../experiments/us-launch-consolidation-20260913.md) +also covers retirement source qualification, UK lineage and the CI repairs. The remaining critical sequence is complete PUF/input integration, a small real full build, model and calibration evaluation, then progressive scale diff --git a/experiments/us-launch-consolidation-20260913.md b/experiments/us-launch-consolidation-20260913.md new file mode 100644 index 000000000..fde31efac --- /dev/null +++ b/experiments/us-launch-consolidation-20260913.md @@ -0,0 +1,77 @@ +# US launch integration: 13 September 2026 + +This batch advances the checked survey graph and repairs the three causes of +the preceding CI failures. It does not certify a native release file. + +## Property income and tax inputs + +The opt-in 35-node host combines qualified ACS/ASEC property observations, +four conditional component models, signed reconciliation and clone attachment. +Its cold construction, required replay and retained-population checks pass. +The separately repeated PUF-recipient handoff passes after fixing an invalid +invented source record. The original failed attempt remains recorded in +[the 35-node host evidence](us-property-financial-host-20260912.md). + +The next opt-in adds three nodes: a receiving population version, four tax-leaf +rewrites, and a numerical gate. All 11 integration controls pass over actual +invented source owners, including cold construction and required replay of the +38-node graph. The run took 645.283 seconds wall time and 0.567 GB peak RSS. +Forty smaller numerical and declaration controls also pass. + +Ordinary interest supplies taxable and tax-exempt interest; dividends supply +qualified and non-qualified dividends. The maintained split fractions remain +explicit assumptions. Retirement-account earnings stay separate. Unknown +components stay unknown, including under-15 observations outside survey +reporting universes. Full preceding populations remain available and checked. +An incomplete numerical gate refuses PUF source qualification. A complete-source +PUF run through this extension remains unverified. See +[the host contract](../docs/us-atomic-property-financial.md) and +[the tax-leaf contract](../docs/us-property-tax-leaves.md). + +The host receipt SHA256 is +`6a1e5798d2182e25eb08ac6e073c71cec3a31c03f1f647df3b8d131222f1cb02`. +All frozen source and resource hashes remained unchanged; no country engine, +native source or full PUF run was executed in that test packet. + +## CI repairs + +- **Retained observations:** final checks now compare the complete detached + observation Frame with its verified manifest Frame after the last artifact + I/O. The previous test reproduced an escaping mutation. Five controls now + pass: actual cold/required cloning and late mutations of either copy with + both return modes. No source revalidation or ownership check was removed. +- **Country identities:** the AM, BE and UK expected fingerprints now include + the already reviewed solver snapshot changes. Replacing only that source + digest in a detached envelope reconstructs all three old fingerprints + exactly. Four controls pass; authored country resources are unchanged. +- **Installed wheels:** `microcosm-build[source-io]` declares h5py and PyTables + without a country engine. CI installs the built extras after checking base + imports and verifies an invented HDF read/write round trip. A fresh local + six-wheel installation passes these checks with no US or UK engine. Locked + dependency versions are unchanged. This smoke test does not replace the + full remote wheel suite. + +## Other integrated source work + +ASEC retirement qualification retains raw slot and aggregate observations, +including the annuity not-in-universe literal. Its comparison follows the +declared normalization without rewriting the source; 62 controls pass. +Retirement measurement and regularity assumptions still require an explicit +bridge before they can supply complete tax inputs. + +The UK household-lineage helper preserves exact original identities through +declared selection and cloning. Forty-three controls pass, including large +integer IDs and graph operations. This is a supplied-lineage contract; it +does not certify a new native UK graph or replace the UK country work. + +## Native run and remaining work + +The preceding native PUF attempt ended without a final receipt or PUF manifest. +Its stopping cause is unknown. It provides no verified native PUF output. +A new attempt needs a separately reviewed packet and resource budget. + +Remaining release work includes honest completion of applicable unknown inputs, +full PUF composition, a small native end-to-end build, engine and calibration +evaluation, national and congressional-district quality checks, progressive +local scale, and full/compact exports. The +[US/UK release path](../docs/us-uk-release-path.md) tracks those gates. diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 76c7f5844..83ea016aa 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -415,6 +415,8 @@ "graph_property_income.py", # Source-blind fit/history and recipient-draw verification "graph_property_income_receipts.py", + # Deterministic ordinary-interest/dividend tax split and numeric gate + "graph_property_tax_leaves.py", # Source-blind PUF clone placement and original-channel preservation "graph_puf55_route_attachment.py", # Complete PUF composition and retained checked-output lifetime @@ -3486,6 +3488,9 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: # Declared numerical target columns and (node, artifact) byte maps; # no original-source routing or provenance selectors. "graph_property_income_receipts.py", + # Reviewed 2026-09-13: declared entity/ID columns, fixed property and + # tax-leaf families, dtype maps and typed artifact names only. + "graph_property_tax_leaves.py", # Reviewed 2026-09-12: exact geography fields/CSR masks, typed artifact # names and producer keys, retained registry entries and node histories. # These four remain scanned for provenance columns and accessor calls. From 78e067a6068b8cd8804e8f71ed3a92025f9983f5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:13:13 -0400 Subject: [PATCH 189/255] Refresh the seed diagnostic lock pin for source reader extras --- .../893-diagnostic-source-io-lock.fixed.md | 1 + .../tests/test_spec_seed_identity_lock.py | 40 +++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 changelog.d/893-diagnostic-source-io-lock.fixed.md create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_lock.py diff --git a/changelog.d/893-diagnostic-source-io-lock.fixed.md b/changelog.d/893-diagnostic-source-io-lock.fixed.md new file mode 100644 index 000000000..f192b5b45 --- /dev/null +++ b/changelog.d/893-diagnostic-source-io-lock.fixed.md @@ -0,0 +1 @@ +Update the source identity diagnostic's exact lockfile pin after adding the engine-free source reader extra, and verify that the checked-in lock passes while an incorrect pin still refuses. diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_lock.py b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py new file mode 100644 index 000000000..07764e2f8 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_lock.py @@ -0,0 +1,40 @@ +"""Checked-in source hashing only; no diagnostic bootstrap or runtime execution.""" + +import hashlib +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("seed_lock_controls", source) + module = importlib.util.module_from_spec(spec) + # Read the checked-in source directly; do not consume a cached bytecode file. + exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__) + return module + + +def test_checked_in_lock_passes_actual_source_stamps(diagnostic): + root = Path(__file__).resolve().parents[3] + stamps = diagnostic.source_stamps(root) + assert ( + stamps["uv.lock"] == hashlib.sha256((root / "uv.lock").read_bytes()).hexdigest() + ) + assert set(stamps) == { + *diagnostic.SOURCE_PATHS, + "tools/spec_seed_identity_diagnostics.py", + "uv.lock", + ".github/workflows/test.yml", + } + + +def test_actual_source_stamps_refuses_an_incorrect_lock_pin(diagnostic, monkeypatch): + root = Path(__file__).resolve().parents[3] + monkeypatch.setattr(diagnostic, "LOCK_SHA256", "0" * 64) + with pytest.raises(diagnostic.RefusalError, match="^LOCK$"): + diagnostic.source_stamps(root) diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 3b3fb618c..25f1f382a 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -21,7 +21,7 @@ from functools import cached_property from pathlib import Path -LOCK_SHA256 = "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9" +LOCK_SHA256 = "14d7f749f14e1dc1fa32a064c13f50a76b73bf03ddd9b6c0534d9ede2d9ff44e" UPLOAD_ACTION_SHA = "ea165f8d65b6e75b540449e92b4886f43607fa02" CAPS = { "candidate-digests.json": 64 * 1024, From 21f02c2ab081c44565bcf54148af3c05768d5c82 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:17:00 -0400 Subject: [PATCH 190/255] Verify independent property observations survive donor exclusions --- ...ndent-observation-preservation.internal.md | 1 + docs/us-current-survey-property-graph.md | 2 + .../test_us_current_survey_property_graph.py | 84 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 changelog.d/property-independent-observation-preservation.internal.md diff --git a/changelog.d/property-independent-observation-preservation.internal.md b/changelog.d/property-independent-observation-preservation.internal.md new file mode 100644 index 000000000..cb1fc6a60 --- /dev/null +++ b/changelog.d/property-independent-observation-preservation.internal.md @@ -0,0 +1 @@ +Add a focused invented-source regression confirming that joint property-donor exclusions preserve independently qualified ASEC interest and dividends on both initial clones and through the existing tax split. Production behavior is unchanged. diff --git a/docs/us-current-survey-property-graph.md b/docs/us-current-survey-property-graph.md index 68cf42d7c..61c0c4a45 100644 --- a/docs/us-current-survey-property-graph.md +++ b/docs/us-current-survey-property-graph.md @@ -29,6 +29,8 @@ The attachment owns the four `property_*` components, their four `draw_property_ ASEC persons retain each individually qualified component and reported total, including NaN for an unresolved value. Donor exclusion does not erase otherwise known amounts. The source-basis discrepancies remain available, and the complete source qualification/exclusion descriptions stay bound to the projection. ASEC raw-draw and reconciliation diagnostics are absent because no draw or reconciliation is performed on these observations. +For example, a qualified survivor estate/trust route can exclude an ASEC person from the joint four-component fit while that person's ordinary interest and dividends remain known. The attachment copies those independent amounts to both initial clones. The separate tax-leaf successor splits each known amount independently; the donor exclusion does not turn it into an unknown tax input. A focused regression exercises this path with authenticated invented survey files and the real attachment and split functions, without fitting a QRF or rerunning the complete financial host. + Eligible ACS persons receive one ordered joint draw and its reconciled components. Their original `person_id` and pandas row index are checked separately, then the original support ID, native source ID, source label, and exact clone roles establish the two-row fan-out. Excluded ACS persons receive NaN for model components and the attached anchor, with false `property_anchor_known` and `property_components_known`. Their original source anchor status remains in the qualified source projection; under-15 values are not silently converted into a new analytical zero. `property_anchor_known` describes the attached anchor's finite value. `property_components_known` describes all four attached components being finite; it is distinct from eligibility for donor fitting. Reconciliation bound flags use nullable booleans so an unperformed reconciliation does not become a false observation. diff --git a/packages/microcosm-build/tests/test_us_current_survey_property_graph.py b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py index b0bd9c093..3bba5a9bd 100644 --- a/packages/microcosm-build/tests/test_us_current_survey_property_graph.py +++ b/packages/microcosm-build/tests/test_us_current_survey_property_graph.py @@ -21,6 +21,90 @@ OPTIONS = dict(scales=(1.0, 1.0, 1.0, 1.0), atol=1e-10, rtol=1e-12, n_estimators=2) +def test_joint_fit_exclusion_preserves_independent_asec_components(actual): # noqa: F811 + """Actual invented source qualification and pure attachment, without fitting. + + Native fixture person107 reports interest/dividends but survivor source8 + excludes the person from the joint fit. Those independent observations + remain usable on both clones. ACS draw inputs below are invented numbers, + not fitted or authenticated model artifacts; they only exercise the + separate recipient assignment in the pure attachment function. + """ + from microcosm.build.us_runtime import current_property_income_sources as sources + from microcosm.build.us_runtime import graph_property_tax_leaves as tax + + live, qualified, _ = actual + basis = qualified.donor_basis + excluded = basis.person.index[basis.person.native_person_id.eq(107)] + assert len(excluded) == 1 + origin = excluded[0] + assert basis.exclusions.loc[origin, "survivor_possible_property"] + assert not basis.person.loc[origin, "joint_component_fit_eligible"] + assert origin not in qualified.donor_columns.index + assert origin not in qualified.recipient_columns.index + assert basis.provenance.loc[origin, "interest.TRDINT_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.DIV_VAL_amount_known"] + assert basis.provenance.loc[origin, "dividend.SUR_SC2_code"] == 8 + source_seal = sources.property_income_sources_seal(qualified) + frame = live.clone_population.frame + frame_seal = sources._frame_seal(frame) + + draws = pd.DataFrame( + np.tile([11.0, 7.0, 13.0, -2.0], (len(qualified.recipient_columns), 1)), + index=qualified.recipient_columns.index, + columns=graph.PROPERTY_DRAW_COLUMNS, + ) + reconciled = graph._reconciliation_result( + qualified, graph.PropertyIncomeOptions(**OPTIONS), draws + ) + legacy = pd.DataFrame(0.0, index=draws.index, columns=shared.TARGETS) + columns = graph.complete_property_columns( + qualified, frame, draws, reconciled, legacy + ) + people = frame.person.copy(deep=True) + for (entity, name), values in columns.items(): + assert entity == "person" + np.testing.assert_array_equal(values.index, people.person_id) + people[name] = values.array.copy() + split = tax.split_property_tax_leaves(people) + source_ids = people[shared.provenance.support_source_id_column("person")] + paired = people.loc[source_ids.eq(origin)] + assert len(paired) == 2 + assert set(paired[shared.provenance.support_clone_index_column("person")]) == { + 0, + 1, + } + for component, known_column, primary, complement in ( + ( + graph.PROPERTY_COMPONENTS[0], + "interest.TRDINT_VAL_amount", + *tax.TAX_LEAF_COLUMNS[:2], + ), + ( + graph.PROPERTY_COMPONENTS[2], + "dividend.DIV_VAL_amount", + *tax.TAX_LEAF_COLUMNS[2:], + ), + ): + amount = np.float64(basis.provenance.loc[origin, known_column]) + assert amount == 500.0 + np.testing.assert_array_equal( + paired[component].to_numpy().view("uint64"), + np.repeat(amount, 2).view("uint64"), + ) + parts = split.loc[paired.person_id, [primary, complement]] + assert np.isfinite(parts.to_numpy()).all() + np.testing.assert_array_equal(parts.sum(axis=1), np.repeat(amount, 2)) + assert paired[list(graph.PROPERTY_DRAW_COLUMNS)].isna().all().all() + # Every original ASEC component, including unknowns, survives unfiltered. + for original_id, row in basis.person.iterrows(): + pair = people.loc[source_ids.eq(original_id)] + for component in graph.PROPERTY_COMPONENTS: + np.testing.assert_array_equal(pair[component], np.repeat(row[component], 2)) + assert sources.property_income_sources_seal(qualified) == source_seal + assert sources._frame_seal(frame) == frame_seal + + def test_explicit_frozen_options_and_complete_owned_roster(): options = graph.PropertyIncomeOptions(**OPTIONS) assert graph.codec.decode_json(options.to_bytes()) == graph.codec.decode_json( From 4fd4c16794e3bf3dc52f7aa8c62009691bff63b4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:57:06 -0400 Subject: [PATCH 191/255] Add conservative ASEC retirement candidate ledger --- .../us-current-asec-retirement-basis.added.md | 1 + docs/us-current-asec-retirement-basis.md | 139 +++ .../current_asec_retirement_basis.py | 824 ++++++++++++++++++ .../test_us_current_asec_retirement_basis.py | 477 ++++++++++ 4 files changed, 1441 insertions(+) create mode 100644 changelog.d/us-current-asec-retirement-basis.added.md create mode 100644 docs/us-current-asec-retirement-basis.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py create mode 100644 packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py diff --git a/changelog.d/us-current-asec-retirement-basis.added.md b/changelog.d/us-current-asec-retirement-basis.added.md new file mode 100644 index 000000000..1b1fb2b7e --- /dev/null +++ b/changelog.d/us-current-asec-retirement-basis.added.md @@ -0,0 +1 @@ +Add a pure ASEC retirement candidate ledger with explicit measurement assumptions, source routing and knownness, conservative accounting intervals, preserved unresolved survivor/distribution scope, and original DESIGN-weight support diagnostics. It produces no fiscal inputs or source authority. diff --git a/docs/us-current-asec-retirement-basis.md b/docs/us-current-asec-retirement-basis.md new file mode 100644 index 000000000..d6c3eb58c --- /dev/null +++ b/docs/us-current-asec-retirement-basis.md @@ -0,0 +1,139 @@ +# ASEC retirement candidate ledger + +`current_asec_retirement_basis.py` describes potential retirement-bridge support +from the accepted retirement-detail and income-routing projections. It preserves +source observations, route uncertainty and published accounting differences. +It produces no fiscal inputs, fitted model, graph host or source authority. + +## API + +`build_asec_retirement_basis` accepts keyword arguments `retirement_detail`, +`income_routing`, `original_household_membership`, +`original_household_design_weights`, and `assumptions`. The two DataFrames must +have identical ordered int64 `person_id` indices, exact native person IDs, and +original interview ages. Shared amount observations and pension receipt codes +must agree. A joint permutation is accepted; a one-sided permutation refuses. +The current contract requires at least one selected person. + +It returns `AsecRetirementBasis` with six descriptive fields: + +- `person`: source identifiers, observed routed subtotals, candidate family + intervals, accounting differences and candidate-support flags. +- `slots`: ten records per person, retaining six pension/disability/survivor + slots and four applicable/off-route distribution slots, source codes, + published amounts, knownness, source statuses and bridge routes. +- `provenance`: detached, prefixed copies of both complete supplied projections, + including allocation and disclosure flags. +- `exclusions`: uncertainty and route diagnostic masks, including cases that + remain numerically usable. These masks are not all applied as donor filters. +- `summary`: counts and supplied original DESIGN-weight support for each + candidate tier and diagnostic mask. +- `assumptions_payload`: canonical immutable bytes identifying the choices. + +The source projection dataclasses and these outputs contain mutable descriptive +tables. Neither their type nor a digest grants source admission. A future host +must retain and requalify the actual original source owners, complete person +scope, table seals and original DESIGN weights around its relevant I/O. This +pure function does not write to either input, calibrate weights or attach clones. + +## Required choices + +Every `RetirementCandidateAssumptions` field must be supplied. No country caller +or development scenario is enabled by this change. + +| Field | Supported values | +| --- | --- | +| `pension_annuity_regularity` | `unresolved`, `assume_regular` | +| `disability_pension_eligibility` | `unresolved`, `assume_qualifying` | +| `survivor_annuity_overlap` | `unresolved` | +| `withdrawal_regularity_netting` | `unresolved` | +| `aggregate_accounting` | `exact_visible_balance_only` | + +`assume_regular` is a measurement scenario about pension/annuity regularity. +`assume_qualifying` explicitly assumes both the disability-pension scope and +regularity needed for the candidate bridge. Neither is inferred from an annual +amount, age, `DIS_HP`, or `DIS_CS`. The latter source answers remain provenance. +There are no private-pension, regular-withdrawal or taxable fractions. Changing +either supported scenario changes `to_bytes()`; unsupported choices refuse. + +## Routes and candidate amounts + +The accepted [retirement-detail source](us-current-asec-retirement-detail-source.md) +and [income-routing source](us-current-asec-income-routing-source.md) retain the +published codebooks. This ledger imports those definitions and applies a separate +bridge route classification: + +| Family | Candidate routes | Separate or unresolved routes | +| --- | --- | --- | +| Pension | Codes 1–6 | 7 Railroad; 8 unresolved | +| Disability | Codes 2–5, subject to the explicit scenario | 6 Railroad; 1/7/8/9 other compensation; 10 unresolved | +| Survivor | Codes 1–4 are visible candidate subtotals | 5 Railroad; 8 property; 9 annuity overlap; 6/7/10 unresolved | +| Distribution | Applicable account codes 1–7 retained individually | Regularity/netting remain unknown; account 4 is a type of IRA | +| Other income | 2/13 indicate additional retirement candidates | 1 Social Security overlap; 8 property overlap; 19 unspecified | + +The retirement ledger never consumes account-earnings amounts or adds Railroad +amounts to Social Security. Other-income amounts are retained separately and +never added to the candidate sum. Categories 2/13/19 leave combined candidate +scope unresolved; known other categories can remain outside that sum. These +routes do not establish any person's tax treatment. + +A source-known amount and a published literal remain separate. A declared +positive slot contributes to its observed routed subtotal. A readable unused +NIU slot can contribute an explicitly labeled structural zero to a numerical +comparison, but its source-known amount remains missing. The subtotal is only +a sum of the readable known slots; it never asserts that unknown slots are zero. +Yes-receipt dollar zeros stay ambiguous where the source domain says none/NIU. +Annuity preserves its distinct valid dollar zero and published `-1` NIU encoding. + +The four published total-minus-slot comparisons are recomputed and checked +against R1. `DBTN_VAL` compares the main distribution slots only; the applicable +known distribution total is supplied independently by the routing owner. No +comparison is allocated to another source or attributed causally to disclosure. + +For pension/disability candidate bounds, all relevant slots must be readable and +receipt-consistent, and their exact integer-dollar sum must equal the aggregate. +Positive discrepancy leaves additional scope unresolved; negative discrepancy +is contradictory accounting. Neither is clipped, used as a cap, or silently +allocated. Readable outside routes are subtracted only under this exact balance. +With unresolved regularity/scope the lower bound is zero; under the named +scenario it includes only known candidate-route amounts. Unresolved source +routes can widen the candidate interval without becoming identified labels. + +Positive survivor receipt keeps full survivor bounds unknown, even when its +published comparison balances. Visible slots do not prove the scope of sources +3/4. Only qualified no receipt with two NIU slots and zero total supplies a +zero survivor family amount. Source 9 alongside positive annuity is explicitly +flagged; no overlap priority or max/sum assumption is applied. + +A fully resolved applicable distribution composition produces a zero-to-total +candidate interval. Missing/unreadable account types, ambiguous active zeros or +off-route answers/dollars prevent that interval. The routing owner's known total +is still preserved when the stricter candidate composition test fails. Account +subtotals are finite only when the complete account composition is known. + +Under-15 people retain their source evidence with outside, unresolved-outside or +contradictory-outside status and no candidate zeros. Combined bounds are available +only when every family and other-income scope passes. `candidate_basis_eligible` +means this conservative numerical support exists. `candidate_point_under_assumptions` +means the candidate bounds coincide; `point_identified` and +`fiscal_outputs_produced` remain false. No training label is created merely by +calling a quantity an interval endpoint. + +## Source and validation boundaries + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +provides the source categories and domains. The [2024 ACS subject definitions, +page 95](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf#page=95) +provide the target's regularity and disability scope. The ledger's conservative +accounting and scenario choices are separate decisions. It does not equate the +surveys' reference periods, allocate joint income, or derive source regularity +or tax treatment from those documents. + +Tests use the actual pure R1/routing projectors on invented records. They check +source routes, exact accounting, unknowns, assumption identities, interview-age +routes, native IDs above `2**53`, one-sided mutations, permutations, detached +provenance and descriptive design-weight support. Person mass uses the supplied +original household weight for each included person; household mass counts each +included household once. The Series has no independent weight-kind authority. +Allocation/disclosure flags do not filter candidates. No native data, source +capture, engine, fitting, calibration, PUF or release acceptance is involved. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py new file mode 100644 index 000000000..0db19ff8f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py @@ -0,0 +1,824 @@ +"""Pure retirement candidate ledger, with explicit measurement assumptions. + +A descriptive table or weight Series grants no source authority. The host owns +complete original-source admission. Candidate bounds are neither fiscal amounts +nor identified ACS labels; aggregate differences are never allocated. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from math import fsum +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from . import current_asec_income_routing_source as routing +from . import current_asec_retirement_detail_source as detail + +PROTOCOL = "microcosm.us.asec-retirement-candidate-basis.v1" +# These bridge route sets are decisions, distinct from the source code labels. +CANDIDATE_CODES = MappingProxyType( + { + "pension": frozenset(range(1, 7)), + "disability": frozenset(range(2, 6)), + "survivor": frozenset(range(1, 5)), + } +) +RAILROAD_CODES = MappingProxyType({"pension": 7, "disability": 6, "survivor": 5}) +FAMILIES = ( + ("pension", "PEN", "PNSN_VAL"), + ("disability", "DIS", "DSAB_VAL"), + ("survivor", "SUR", "SRVS_VAL"), +) +DISTRIBUTION_SLOTS = ( + ("slot1", "DST_VAL1", False), + ("slot2", "DST_VAL2", False), + ("slot1_young", "DST_VAL1_YNG", True), + ("slot2_young", "DST_VAL2_YNG", True), +) +OTHER_RETIREMENT_CANDIDATES = frozenset((2, 13)) +OTHER_UNSPECIFIED = 19 + + +def _require(condition, reason): + if not condition: + raise ValueError("ASEC_RETIREMENT_BASIS_" + reason) + + +@dataclass(frozen=True) +class RetirementCandidateAssumptions: + """Required choices; no constructor defaults or observed-treatment claims.""" + + pension_annuity_regularity: str + disability_pension_eligibility: str + survivor_annuity_overlap: str + withdrawal_regularity_netting: str + aggregate_accounting: str + + def __post_init__(self): + allowed = { + "pension_annuity_regularity": ("unresolved", "assume_regular"), + "disability_pension_eligibility": ("unresolved", "assume_qualifying"), + "survivor_annuity_overlap": ("unresolved",), + "withdrawal_regularity_netting": ("unresolved",), + "aggregate_accounting": ("exact_visible_balance_only",), + } + _require( + all( + type(getattr(self, k)) is str and getattr(self, k) in v + for k, v in allowed.items() + ), + "ASSUMPTIONS", + ) + + def to_bytes(self) -> bytes: + self.__post_init__() + return ( + json.dumps( + {"protocol": PROTOCOL, **asdict(self)}, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode() + + +@dataclass(frozen=True) +class AsecRetirementBasis: + """Detached descriptive tables and immutable assumption bytes; no issuer.""" + + person: pd.DataFrame + slots: pd.DataFrame + provenance: pd.DataFrame + exclusions: pd.DataFrame + summary: pd.DataFrame + assumptions_payload: bytes + + +def _numeric(series, label): + _require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "NUMERIC:" + label, + ) + values = series.to_numpy(dtype="float64", na_value=np.nan, copy=True) + _require(not np.isinf(values).any(), "INFINITE:" + label) + return values + + +def _text(frame, name): + _require(name in frame, "COLUMN:" + name) + values = frame[name].to_numpy(copy=True) + _require(all(type(v) is str for v in values), "TEXT:" + name) + return values + + +def _axis(frame): + _require( + type(frame) is pd.DataFrame + and frame.columns.is_unique + and frame.index.dtype == np.dtype("int64") + and frame.index.name == "person_id" + and frame.index.is_unique, + "PERSON_AXIS", + ) + _require({"native_person_id", "source_age"} <= set(frame), "IDENTITY_COLUMNS") + _require( + frame.native_person_id.dtype == np.dtype("int64") + and frame.native_person_id.is_unique, + "NATIVE_AXIS", + ) + age = _numeric(frame.source_age, "AGE") + _require( + np.isfinite(age).all() + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "AGE", + ) + return age + + +def _equal(left, right, label): + _require(np.array_equal(left, right, equal_nan=True), "AGREEMENT:" + label) + + +def _codes(frame, prefix, domain): + values = _numeric(frame[prefix + "_code"], prefix) + status = _text(frame, prefix + "_literal_status") + valid = status == "in_printed_range" + _require( + np.array_equal(valid, np.isfinite(values)) + and np.isin(values[valid], tuple(domain)).all(), + "CODE:" + prefix, + ) + return values, status + + +def _published(frame): + values = {} + for name, entry in detail.amount_entries().items(): + amount = _numeric(frame[name + "_published_amount"], name) + status = _text(frame, name + "_literal_status") + valid = status == "in_printed_range" + _require( + np.array_equal(valid, np.isfinite(amount)) + and ( + (amount[valid] >= entry.encoded_minimum) + & (amount[valid] <= entry.encoded_maximum) + & (amount[valid] == np.floor(amount[valid])) + ).all(), + "PUBLISHED:" + name, + ) + values[name] = amount + return values + + +def _known_routing(frame, prefix): + status = _text(frame, prefix + "_reporting_status") + amount = _numeric(frame[prefix + "_known_amount"], prefix) + known = np.isin(status, routing.KNOWN_AMOUNT_STATUSES) + _require( + np.array_equal(known, np.isfinite(amount)) + and (amount[known] >= 0).all() + and ( + amount[np.isin(status, ("known_nonreceipt", "known_recipient_zero"))] == 0 + ).all(), + "KNOWN_ROUTING:" + prefix, + ) + return amount, status + + +def _reference_agreement(d, r, published, ages): + for family, field, receipt_prefix in ( + ("pension", "PNSN_VAL", "pension_annuity_pension_receipt"), + ("annuity", "ANN_VAL", "pension_annuity_annuity_receipt"), + ): + prefix = "pension_annuity_" + family + receipt, receipt_status = _codes(r, receipt_prefix, routing.RECEIPT_CODE_DOMAIN) + if family == "pension": + other, other_status = _codes(d, "PEN_YN", routing.RECEIPT_CODE_DOMAIN) + _equal(receipt, other, "PEN_YN") + _require(np.array_equal(receipt_status, other_status), "PEN_YN_STATUS") + amount, statuses = _known_routing(r, prefix) + source = published[field].copy() + kinds = np.where( + np.isnan(source), + "missing", + np.where( + source == -1, "declared_niu", np.where(source == 0, "zero", "nonzero") + ), + ) + source[kinds == "declared_niu"] = np.nan + _equal(source, _numeric(r[prefix + "_source_total"], prefix), field) + expected = [ + routing.receipt_status( + bool(age >= 15), + (None if np.isnan(code) else int(code), literal), + kind, + net_measure=False, + zero_is_dollars=field == "ANN_VAL", + ) + for age, code, literal, kind in zip( + ages, receipt, receipt_status, kinds, strict=True + ) + ] + _require(np.array_equal(statuses, expected), "ROUTING_STATUS:" + field) + _equal( + amount, + np.where(np.isin(expected, routing.KNOWN_AMOUNT_STATUSES), source, np.nan), + "KNOWN:" + field, + ) + for slot, field, _ in DISTRIBUTION_SLOTS: + _equal( + published[field], + _numeric(r["retirement_distribution_" + slot + "_amount"], slot), + field, + ) + + +def _route(family, code): + if not np.isfinite(code): + return "unresolved" + if code == 0: + return "unused_or_nonreceipt" + if code in CANDIDATE_CODES[family]: + return "candidate" + if code == RAILROAD_CODES[family]: + return "railroad" + if family == "disability" and code in (1, 7, 8, 9): + return "other_compensation" + if family == "survivor" and code == 8: + return "property" + if family == "survivor" and code == 9: + return "annuity_overlap" + return "unresolved" + + +def _detail_slots(d, published, ages): + result = {family: [] for family, *_ in FAMILIES} + for family, prefix, _ in FAMILIES: + receipt, receipt_literal = _codes( + d, prefix + "_YN", routing.RECEIPT_CODE_DOMAIN + ) + for slot in (1, 2): + field = prefix + "_VAL" + str(slot) + code, literal = _codes( + d, prefix + "_SC" + str(slot), detail.SOURCE_CODES[prefix] + ) + statuses = _text(d, field + "_reporting_status") + amount = _numeric(d[field + "_amount"], field) + known = d[field + "_amount_known"] + _require( + pd.api.types.is_bool_dtype(known.dtype) and known.notna().all(), + "KNOWN_MASK:" + field, + ) + expected = [ + detail._slot_status( + age, + (None if np.isnan(rec) else int(rec), recstat), + (None if np.isnan(c) else int(c), cstat), + (None if np.isnan(v) else int(v), vs), + ) + for age, rec, recstat, c, cstat, v, vs in zip( + ages, + receipt, + receipt_literal, + code, + literal, + published[field], + _text(d, field + "_literal_status"), + strict=True, + ) + ] + _require(np.array_equal(statuses, expected), "SLOT_STATUS:" + field) + expected_known = np.isin(statuses, routing.KNOWN_AMOUNT_STATUSES) + _require( + np.array_equal(known.to_numpy(dtype=bool), expected_known), + "KNOWN_MASK:" + field, + ) + _equal( + amount, + np.where(expected_known, published[field], np.nan), + "SLOT_AMOUNT:" + field, + ) + for i in range(len(d)): + structural = ( + statuses[i] == "unreported_source_slot" + and code[i] == 0 + and published[field][i] == 0 + ) + result[family].append( + { + "position": i, + "family": family, + "slot": slot, + "amount_field": field, + "source_code": code[i], + "source_literal_status": literal[i], + "published_amount": published[field][i], + "reporting_status": statuses[i], + "source_known_amount": amount[i], + "source_amount_known": bool(expected_known[i]), + "route": _route(family, code[i]), + "applicable": bool(ages[i] >= 15), + "structural_zero_comparison": bool(structural), + } + ) + return result + + +def _comparison(d, published, family, total, first, second): + difference = published[total] - published[first] - published[second] + suffix = "main_slots" if family == "distribution" else "visible_slots" + _equal( + difference, + _numeric(d[family + "_total_minus_" + suffix], family), + "COMPARISON:" + family, + ) + return difference + + +def _family_row(family, receipt, total, difference, slots, age, assumptions): + lower, upper = np.nan, np.nan + route_amounts = { + route: fsum( + s["source_known_amount"] + for s in slots + if s["route"] == route and s["source_amount_known"] + ) + for route in ( + "candidate", + "railroad", + "property", + "annuity_overlap", + "other_compensation", + "unresolved", + ) + } + usable = all( + s["source_amount_known"] or s["structural_zero_comparison"] for s in slots + ) + no = ( + receipt == 2 + and total == 0 + and all(s["reporting_status"] == "known_nonreceipt" for s in slots) + ) + contradiction = ( + any("contradictory" in s["reporting_status"] for s in slots) + or (np.isfinite(difference) and difference < 0) + or (receipt == 2 and np.isfinite(total) and total != 0) + ) + if age < 15: + status = ( + "contradictory_outside_reporting_universe" + if contradiction or (np.isfinite(total) and total != 0) + else "unresolved_outside_reporting_universe" + if not np.isfinite(total) + or any(s["reporting_status"] != "outside_reporting_universe" for s in slots) + else "outside_reporting_universe" + ) + elif contradiction: + status = "contradictory_accounting" + elif no: + lower = upper = 0.0 + status = "known_nonreceipt" + elif ( + not usable + or not np.isfinite(total) + or not np.isfinite(difference) + or receipt != 1 + or total <= 0 + ): + status = "unresolved_source_accounting" + elif difference > 0: + status = "additional_scope_unresolved" + elif family == "survivor": + status = "additional_survivor_scope_unresolved" + else: + excluded = route_amounts["railroad"] + route_amounts["other_compensation"] + upper = total - excluded + _require( + upper >= 0 + and upper == route_amounts["candidate"] + route_amounts["unresolved"], + "CANDIDATE_ACCOUNTING", + ) + assumed = ( + assumptions.pension_annuity_regularity == "assume_regular" + if family == "pension" + else assumptions.disability_pension_eligibility == "assume_qualifying" + ) + lower = route_amounts["candidate"] if assumed else 0.0 + status = ( + "route_outside_only" + if upper == 0 + else "candidate_under_assumptions" + if assumed + else "candidate_regularity_or_scope_unresolved" + ) + return { + family + "_candidate_lower": lower, + family + "_candidate_upper": upper, + family + "_status": status, + family + "_accounting_difference": difference, + **{ + family + "_observed_" + route + "_subtotal": amount + for route, amount in route_amounts.items() + }, + } + + +def _distributions(r, published, ages): + total, status = _known_routing(r, "retirement_distribution") + receipt, receipt_literal = _codes( + r, "retirement_distribution_receipt", routing.RECEIPT_CODE_DOMAIN + ) + codes = { + slot: _codes( + r, "retirement_distribution_" + slot + "_account", routing.ACCOUNT_CODES + ) + for slot, *_ in DISTRIBUTION_SLOTS + } + raw_status = { + slot: _text(r, "retirement_distribution_" + slot + "_slot_status") + for slot, *_ in DISTRIBUTION_SLOTS + } + offroute_receipts = { + name: _codes( + r, "retirement_distribution_receipt_" + name, routing.RECEIPT_CODE_DOMAIN + )[0] + for name in ("young", "58") + } + roster, rows = [], [] + for i, age in enumerate(ages): + known_composition, account_amounts, sum_applicable = ( + True, + {code: 0.0 for code in routing.ACCOUNT_CODES if code != 0}, + 0.0, + ) + offroute = False + for slot_number, (slot, field, young) in enumerate(DISTRIBUTION_SLOTS, 1): + applicable = bool((age < 58) == young) + declared = r["retirement_distribution_" + slot + "_applicable"].iloc[i] + _require( + isinstance(declared, (bool, np.bool_)) and bool(declared) == applicable, + "DISTRIBUTION_AGE_ROUTE", + ) + code, literal = codes[slot][0][i], codes[slot][1][i] + value = published[field][i] + active = ( + applicable + and literal == "in_printed_range" + and code > 0 + and np.isfinite(value) + and value > 0 + and raw_status[slot][i] == "known_slot" + ) + niu = ( + applicable + and code == 0 + and literal == "in_printed_range" + and value == 0 + and raw_status[slot][i] == "niu_slot" + ) + if applicable: + known_composition &= bool(active or niu) + sum_applicable += value + if active: + account_amounts[int(code)] += value + else: + offroute |= bool( + (np.isfinite(value) and value != 0) + or (np.isfinite(code) and code != 0) + ) + roster.append( + { + "position": i, + "family": "distribution", + "slot": slot_number, + "amount_field": field, + "source_code": code, + "source_literal_status": literal, + "published_amount": value, + "reporting_status": raw_status[slot][i], + "source_known_amount": value if active else np.nan, + "source_amount_known": bool(active), + "route": "withdrawal_regularity_netting_unresolved" + if active + else "unused_or_unresolved", + "applicable": applicable, + "structural_zero_comparison": bool(niu), + } + ) + other = "young" if age >= 58 else "58" + other_receipt = offroute_receipts[other] + offroute |= bool(np.isfinite(other_receipt[i]) and other_receipt[i] != 0) + if np.isfinite(total[i]) and known_composition: + _require(total[i] == sum_applicable, "DISTRIBUTION_TOTAL") + eligible = ( + age >= 15 + and known_composition + and not offroute + and np.isfinite(total[i]) + and receipt_literal[i] == "in_printed_range" + and receipt[i] in (1, 2) + ) + if eligible: + _require( + (receipt[i] == 1 and total[i] > 0) + or (receipt[i] == 2 and total[i] == 0), + "DISTRIBUTION_RECEIPT", + ) + rows.append( + { + "distribution_candidate_lower": 0.0 if eligible else np.nan, + "distribution_candidate_upper": total[i] if eligible else np.nan, + "distribution_status": "known_nonreceipt" + if eligible and total[i] == 0 + else "regularity_netting_unresolved" + if eligible + else "unresolved_source_composition", + "distribution_source_known_amount": total[i], + "distribution_source_reporting_status": status[i], + "distribution_account_composition_known": bool( + known_composition and not offroute + ), + **{ + "distribution_account_" + str(c) + "_amount": v + if known_composition and not offroute and age >= 15 + else np.nan + for c, v in account_amounts.items() + }, + } + ) + return rows, roster + + +def _other_income(r, age): + amount, status = _known_routing(r, "other_income") + receipt, _ = _codes(r, "other_income_receipt", routing.RECEIPT_CODE_DOMAIN) + category, _ = _codes(r, "other_income_category", routing.OTHER_INCOME_CATEGORIES) + route = _text(r, "other_income_routing_status") + no = ( + (receipt == 2) + & (category == 0) + & (amount == 0) + & (status == "known_nonreceipt") + & (route == "niu_category") + ) + yes = ( + (receipt == 1) + & (category > 0) + & (amount > 0) + & (status == "known_receipt") + & (route == "reported_category") + ) + clear = (age >= 15) & ( + no + | (yes & ~np.isin(category, (*OTHER_RETIREMENT_CANDIDATES, OTHER_UNSPECIFIED))) + ) + return ( + amount, + clear, + yes & np.isin(category, tuple(OTHER_RETIREMENT_CANDIDATES)), + yes & (category == OTHER_UNSPECIFIED), + yes & (category == 1), + yes & (category == 8), + ) + + +def _summary(index, membership, weights, masks): + _require( + type(membership) is pd.Series + and membership.index.equals(index) + and membership.index.name == "person_id" + and membership.dtype == np.dtype("int64"), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + type(weights) is pd.Series + and weights.index.dtype == np.dtype("int64") + and weights.index.name == "household_id" + and weights.index.is_unique + and set(weights.index) == set(membership), + "DESIGN_MEMBERSHIP", + ) + values = _numeric(weights, "DESIGN_WEIGHTS") + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_WEIGHTS") + design = weights.loc[membership].to_numpy(dtype="float64") + rows = [] + for name, mask in masks.items(): + households = membership.iloc[np.flatnonzero(mask)].unique() + person_mass = fsum(design[mask]) + household_mass = fsum(weights.loc[households].to_numpy(dtype="float64")) + _require( + np.isfinite(person_mass) and np.isfinite(household_mass), "DESIGN_MASS" + ) + rows.append( + (name, int(mask.sum()), len(households), person_mass, household_mass) + ) + return pd.DataFrame( + rows, + columns=( + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ), + ).set_index("selection") + + +def build_asec_retirement_basis( + *, + retirement_detail: pd.DataFrame, + income_routing: pd.DataFrame, + original_household_membership: pd.Series, + original_household_design_weights: pd.Series, + assumptions: RetirementCandidateAssumptions, +) -> AsecRetirementBasis: + """Describe aligned original ASEC source evidence; perform no I/O or fitting. + + Every bound is a conditional candidate accounting result, never a fiscal + label. Positive survivor receipt and nonzero aggregate differences remain + unresolved. DESIGN Series custody and complete-person admission stay external. + """ + _require(type(assumptions) is RetirementCandidateAssumptions, "ASSUMPTIONS_TYPE") + payload = assumptions.to_bytes() + d, r = retirement_detail, income_routing + age = _axis(d) + other_age = _axis(r) + _require( + d.index.equals(r.index) and d.native_person_id.equals(r.native_person_id), + "SOURCE_ALIGNMENT", + ) + _equal(age, other_age, "AGE") + _require(len(d) > 0, "EMPTY_CANDIDATE_SCOPE") + published = _published(d) + _reference_agreement(d, r, published, age) + records = _detail_slots(d, published, age) + by_person = {family: [[] for _ in range(len(d))] for family in records} + for family, entries in records.items(): + for entry in entries: + by_person[family][entry["position"]].append(entry) + differences = { + family: _comparison(d, published, family, total, first, second) + for family, total, first, second in detail.COMPARISONS + } + distribution, distribution_records = _distributions(r, published, age) + annuity, annuity_status = _known_routing(r, "pension_annuity_annuity") + other_amount, other_clear, oi_candidate, oi_unspecified, oi_ss, oi_property = ( + _other_income(r, age) + ) + receipts = { + family: _codes(d, prefix + "_YN", routing.RECEIPT_CODE_DOMAIN)[0] + for family, prefix, _ in FAMILIES + } + rows = [] + for i in range(len(d)): + row = { + "native_person_id": int(d.native_person_id.iloc[i]), + "source_age": int(age[i]), + } + for family, _, total in FAMILIES: + slots = by_person[family][i] + row.update( + _family_row( + family, + receipts[family][i], + published[total][i], + differences[family][i], + slots, + age[i], + assumptions, + ) + ) + known_annuity = age[i] >= 15 and np.isfinite(annuity[i]) + row.update( + annuity_candidate_lower=( + annuity[i] + if assumptions.pension_annuity_regularity == "assume_regular" + else 0.0 + ) + if known_annuity + else np.nan, + annuity_candidate_upper=annuity[i] if known_annuity else np.nan, + annuity_status=annuity_status[i], + other_income_reported_amount=other_amount[i], + other_income_scope_clear=bool(other_clear[i]), + distribution_accounting_difference=differences["distribution"][i], + **distribution[i], + ) + families = ("pension", "disability", "survivor", "annuity", "distribution") + available = ( + age[i] >= 15 + and other_clear[i] + and all( + np.isfinite(row[f + "_candidate_lower"]) + and np.isfinite(row[f + "_candidate_upper"]) + for f in families + ) + ) + lower = ( + fsum(row[f + "_candidate_lower"] for f in families) if available else np.nan + ) + upper = ( + fsum(row[f + "_candidate_upper"] for f in families) if available else np.nan + ) + row.update( + retirement_candidate_lower=lower, + retirement_candidate_upper=upper, + candidate_interval_available=bool(available), + candidate_point_under_assumptions=bool(available and lower == upper), + candidate_basis_eligible=bool(available), + point_identified=False, + fiscal_outputs_produced=False, + ) + rows.append(row) + person = pd.DataFrame(rows, index=d.index.copy()) + all_records = [ + s.copy() for family in records.values() for s in family + ] + distribution_records + for item in all_records: + i = item.pop("position") + item["person_id"] = int(d.index[i]) + item["native_person_id"] = int(d.native_person_id.iloc[i]) + item["source_age"] = int(age[i]) + slots = ( + pd.DataFrame(all_records) + .sort_values(["person_id", "family", "slot"], kind="stable") + .reset_index(drop=True) + ) + exclusions = pd.DataFrame( + { + "under15": age < 15, + "survivor_additional_sources_unresolved": (age >= 15) + & (receipts["survivor"] == 1), + **{ + family + "_reported_" + route: person[ + family + "_observed_" + route + "_subtotal" + ].to_numpy() + > 0 + for family in records + for route in ( + "railroad", + "property", + "annuity_overlap", + "other_compensation", + "unresolved", + ) + }, + "other_income_scope_unresolved": ~other_clear, + "other_income_retirement_candidate": oi_candidate, + "other_income_unspecified": oi_unspecified, + "other_income_ss_overlap": oi_ss, + "other_income_property_overlap": oi_property, + "survivor_annuity_overlap": ( + person.survivor_observed_annuity_overlap_subtotal.to_numpy() > 0 + ) + & (annuity > 0), + **{ + family + "_candidate_unresolved": ~np.isfinite( + person[family + "_candidate_upper"].to_numpy() + ) + for family in ( + "pension", + "disability", + "survivor", + "annuity", + "distribution", + ) + }, + **{ + family + "_aggregate_nonzero_or_unknown": ~np.isfinite(diff) + | (diff != 0) + for family, diff in differences.items() + }, + }, + index=d.index.copy(), + ) + provenance = pd.concat( + [ + d.copy(deep=True).add_prefix("retirement_detail."), + r.copy(deep=True).add_prefix("income_routing."), + ], + axis=1, + ) + masks = { + "all": np.ones(len(d), dtype=bool), + **{ + name: person[name].to_numpy() + for name in ( + "candidate_basis_eligible", + "candidate_interval_available", + "candidate_point_under_assumptions", + ) + }, + **{"diagnostic:" + name: exclusions[name].to_numpy() for name in exclusions}, + } + summary = _summary( + d.index, original_household_membership, original_household_design_weights, masks + ) + return AsecRetirementBasis(person, slots, provenance, exclusions, summary, payload) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py new file mode 100644 index 000000000..5ea318635 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py @@ -0,0 +1,477 @@ +"""Pure retirement candidate support over actual projectors and invented literals.""" + +import dataclasses +import importlib +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_income_routing_source as routing +from microcosm.build.us_runtime import current_asec_retirement_detail_source as detail + + +@pytest.fixture +def owner(): + return importlib.import_module( + "microcosm.build.us_runtime.current_asec_retirement_basis" + ) + + +def options(owner, **changes): + return owner.RetirementCandidateAssumptions( + **{ + "pension_annuity_regularity": "unresolved", + "disability_pension_eligibility": "unresolved", + "survivor_annuity_overlap": "unresolved", + "withdrawal_regularity_netting": "unresolved", + "aggregate_accounting": "exact_visible_balance_only", + **changes, + } + ) + + +def inputs(owner, changes=({},), **assumptions): + rows = [] + for i, change in enumerate(changes): + row = { + **{n: "0" for n in detail.READ_COLUMNS}, + **{n: "0" for n in routing.RECEIPT_ENTRIES}, + **{n: "0" for n in routing.ACCOUNT_ENTRIES}, + **{n: "0" for n in routing.AMOUNT_FIELDS}, + "PERIDNUM": str(i + 1).zfill(22), + "PH_SEQ": str(i // 2 + 1), + "A_LINENO": str(i % 2 + 1), + "A_AGE": "40", + "OI_OFF": "0", + "PEN_YN": "2", + "DIS_YN": "2", + "SUR_YN": "2", + "ANN_YN": "2", + "DST_YN_YNG": "2", + "OI_YN": "2", + **change, + } + rows.append(row) + raw = pd.DataFrame(rows) + ages = np.array([int(r["A_AGE"]) for r in rows], dtype="int64") + values, statuses = {}, {} + for n in routing.AMOUNT_FIELDS: + data, labels = [], [] + for r in rows: + token = r[n] + value = np.nan if token == "" else float(token) + code = ( + routing.money.CodebookStatus.MISSING_NULL + if token == "" + else routing.money.CodebookStatus.DECLARED_NIU + if n == "ANN_VAL" and value == -1 + else routing.money.CodebookStatus.ZERO_NONE_OR_NIU + if value == 0 + else routing.money.CodebookStatus.AMOUNT_NONZERO + ) + data.append( + 0.0 if code == routing.money.CodebookStatus.DECLARED_NIU else value + ) + labels.append(int(code)) + values[n], statuses[n] = ( + np.array(data, dtype="float64"), + np.array(labels, dtype="u1"), + ) + arrays = { + "amounts": values, + "statuses": statuses, + "allocations": { + n: ([0] * len(rows), ["in_printed_range"] * len(rows)) + for n in routing.ALLOCATION_ENTRIES + }, + **{ + n: [r[n] for r in rows] + for n in (*routing.RECEIPT_ENTRIES, *routing.ACCOUNT_ENTRIES, "OI_OFF") + }, + } + projected = { + "retirement_detail": detail.project_retirement_detail_literals(raw), + "income_routing": routing.project_income_routing(arrays, ages), + } + index = pd.Index(np.arange(len(rows), dtype="int64") + 2**53 + 19, name="person_id") + for table in projected.values(): + table.index = index + table["native_person_id"] = np.arange(len(rows), dtype="int64") + 17 + membership = pd.Series(np.arange(len(rows), dtype="int64") // 2 + 101, index=index) + households = pd.Index(membership.unique(), dtype="int64", name="household_id") + return { + **projected, + "original_household_membership": membership, + "original_household_design_weights": pd.Series( + np.arange(len(households)) + 2.0, index=households + ), + "assumptions": options(owner, **assumptions), + } + + +def build(owner, changes=({},), **assumptions): + return owner.build_asec_retirement_basis(**inputs(owner, changes, **assumptions)) + + +def test_named_options_are_required_canonical_and_never_source_facts(owner): + value = options(owner) + assert json.loads(value.to_bytes())["pension_annuity_regularity"] == "unresolved" + assert ( + dataclasses.replace( + value, pension_annuity_regularity="assume_regular" + ).to_bytes() + != value.to_bytes() + ) + assert ( + dataclasses.replace( + value, disability_pension_eligibility="assume_qualifying" + ).to_bytes() + != value.to_bytes() + ) + with pytest.raises(TypeError): + owner.RetirementCandidateAssumptions() + for field, bad in ( + ("pension_annuity_regularity", True), + ("aggregate_accounting", "clip"), + ("withdrawal_regularity_netting", "all_regular"), + ): + with pytest.raises(ValueError): + options(owner, **{field: bad}) + + +@pytest.mark.parametrize( + "code,route", + [(1, "candidate"), (6, "candidate"), (7, "railroad"), (8, "unresolved")], +) +def test_balanced_pension_routes_and_named_regularity(owner, code, route): + row = {"PEN_YN": "1", "PEN_SC1": str(code), "PEN_VAL1": "100", "PNSN_VAL": "100"} + result = build(owner, [row]) + selected = result.slots.query("family == 'pension' and slot == 1").iloc[0] + assert selected.route == route and selected.source_known_amount == 100 + p = result.person.iloc[0] + assert p.pension_candidate_lower == 0 + assert p.pension_candidate_upper == (0 if code == 7 else 100) + assumed = build( + owner, [row], pension_annuity_regularity="assume_regular" + ).person.iloc[0] + assert assumed.pension_candidate_lower == (100 if code in (1, 6) else 0) + assert not p.point_identified and not p.fiscal_outputs_produced + assert ( + result.slots.query("family == 'pension' and slot == 2") + .iloc[0] + .structural_zero_comparison + ) + + +@pytest.mark.parametrize( + "total,status", + [(150, "additional_scope_unresolved"), (80, "contradictory_accounting")], +) +def test_unbalanced_pension_never_clips_or_allocates_residual(owner, total, status): + p = build( + owner, + [{"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": str(total)}], + ).person.iloc[0] + assert p.pension_accounting_difference == total - 100 + assert p.pension_status == status + assert np.isnan(p.pension_candidate_upper) + assert p.pension_observed_candidate_subtotal == 100 + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("code", [1, 5, 8, 9, 10]) +def test_positive_survivor_never_claims_hidden_source_clearance(owner, code): + result = build( + owner, + [ + { + "SUR_YN": "1", + "SUR_SC1": str(code), + "SUR_VAL1": "100", + "SRVS_VAL": "100", + "ANN_YN": "1", + "ANN_VAL": "40", + } + ], + ) + p = result.person.iloc[0] + assert p.survivor_accounting_difference == 0 + assert p.survivor_status == "additional_survivor_scope_unresolved" + assert np.isnan(p.survivor_candidate_upper) and not p.candidate_basis_eligible + assert bool(result.exclusions.survivor_annuity_overlap.iloc[0]) == (code == 9) + assert not p.point_identified + + +@pytest.mark.parametrize( + "code,route", + [ + (2, "candidate"), + (5, "candidate"), + (6, "railroad"), + (9, "other_compensation"), + (10, "unresolved"), + ], +) +def test_disability_does_not_infer_eligibility_from_health_answers(owner, code, route): + row = { + "DIS_YN": "1", + "DIS_SC1": str(code), + "DIS_VAL1": "100", + "DSAB_VAL": "100", + "DIS_HP": "1", + "DIS_CS": "1", + } + result = build(owner, [row]) + p = result.person.iloc[0] + assert ( + result.slots.query("family == 'disability' and slot == 1").iloc[0].route + == route + ) + assert p.disability_candidate_lower == 0 + assert p.disability_candidate_upper == ( + 100 if route in ("candidate", "unresolved") else 0 + ) + changed = build(owner, [{**row, "DIS_HP": "2", "DIS_CS": "2"}]) + pd.testing.assert_frame_equal(result.person, changed.person, check_exact=True) + assumed = build( + owner, [row], disability_pension_eligibility="assume_qualifying" + ).person.iloc[0] + assert assumed.disability_candidate_lower == (100 if route == "candidate" else 0) + + +@pytest.mark.parametrize( + "amount,receipt,known", + [("-1", "0", False), ("0", "1", True), ("0", "2", True), ("40", "1", True)], +) +def test_annuity_niu_zero_and_regular_scenario_are_distinct( + owner, amount, receipt, known +): + source = inputs(owner, [{"ANN_YN": receipt, "ANN_VAL": amount}]) + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert np.isfinite(p.annuity_candidate_upper) == known + if known: + assert p.annuity_candidate_lower == 0 and p.annuity_candidate_upper == float( + amount + ) + source["assumptions"] = options( + owner, pension_annuity_regularity="assume_regular" + ) + assert owner.build_asec_retirement_basis(**source).person.iloc[ + 0 + ].annuity_candidate_lower == float(amount) + + +@pytest.mark.parametrize("age", [57, 58]) +def test_distribution_account_is_not_frequency_or_taxability(owner, age): + young = age < 58 + suffix = "_YNG" if young else "" + row = { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + } + p = build(owner, [row]).person.iloc[0] + assert p.distribution_candidate_lower == 0 and p.distribution_candidate_upper == 100 + assert p.distribution_account_4_amount == 100 + assert not p.point_identified and not p.fiscal_outputs_produced + + +@pytest.mark.parametrize( + "change", + [ + {"DST_SC1_YNG": ""}, + {"DST_SC1_YNG": "9"}, + {"DST_VAL1": "1"}, + {"DST_YN": "2"}, + {"DST_SC2_YNG": "3"}, + ], +) +def test_distribution_unreadable_or_offroute_composition_stays_unknown(owner, change): + row = {"DST_YN_YNG": "1", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100", **change} + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "code,clear", + [(2, False), (13, False), (19, False), (1, True), (8, True), (20, True)], +) +def test_other_income_not_silently_added(owner, code, clear): + p = build( + owner, [{"OI_YN": "1", "OI_OFF": str(code), "OI_VAL": "100"}] + ).person.iloc[0] + assert bool(p.other_income_scope_clear) is clear + assert p.other_income_reported_amount == 100 + assert ( + (p.retirement_candidate_upper == 0) + if clear + else np.isnan(p.retirement_candidate_upper) + ) + + +@pytest.mark.parametrize("age", [0, 14]) +def test_under15_retains_evidence_without_analytic_zero(owner, age): + result = build( + owner, + [ + { + "A_AGE": str(age), + "PEN_YN": "0", + "ANN_YN": "0", + "DIS_YN": "0", + "SUR_YN": "0", + "DST_YN_YNG": "0", + "OI_YN": "0", + } + ], + ) + p = result.person.iloc[0] + assert all( + np.isnan(p[n]) + for n in p.index + if n.endswith(("_candidate_lower", "_candidate_upper")) + ) + assert result.exclusions.under15.iloc[0] and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "change", [{"PEN_YN": ""}, {"PEN_SC1": ""}, {"PEN_VAL1": ""}, {"PEN_VAL1": "0"}] +) +def test_source_unknowns_and_ambiguous_zeros_remain_visible(owner, change): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + **change, + } + p = build(owner, [row]).person.iloc[0] + assert np.isnan(p.pension_candidate_upper) and not p.candidate_basis_eligible + + +@pytest.mark.parametrize( + "defect", + [ + "person_order", + "native_id", + "age", + "shared_amount", + "known_mask", + "comparison", + "bad_weight", + "membership", + ], +) +def test_incompatible_descriptions_refuse(owner, defect): + source = inputs(owner, [{}, {}]) + d, r = source["retirement_detail"], source["income_routing"] + if defect == "person_order": + source["income_routing"] = r.iloc[::-1] + elif defect == "native_id": + r.iloc[0, r.columns.get_loc("native_person_id")] += 100 + elif defect == "age": + r.iloc[0, r.columns.get_loc("source_age")] += 1 + elif defect == "shared_amount": + r.iloc[0, r.columns.get_loc("pension_annuity_pension_source_total")] += 1 + elif defect == "known_mask": + d.iloc[0, d.columns.get_loc("PEN_VAL1_amount_known")] = False + elif defect == "comparison": + d.iloc[0, d.columns.get_loc("pension_total_minus_visible_slots")] = 1 + elif defect == "bad_weight": + source["original_household_design_weights"].iloc[0] = np.nan + else: + source["original_household_membership"] = source[ + "original_household_membership" + ].iloc[::-1] + with pytest.raises(ValueError): + owner.build_asec_retirement_basis(**source) + + +def test_exact_permutation_detachment_and_design_mass(owner): + source = inputs( + owner, + [{}, {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "80"}, {}], + ) + before = { + k: v.copy(deep=True) + for k, v in source.items() + if isinstance(v, (pd.DataFrame, pd.Series)) + } + result = owner.build_asec_retirement_basis(**source) + assert result.summary.loc["all", "design_weighted_person_mass"] == 7 + assert result.summary.loc["all", "union_household_design_mass"] == 5 + assert ( + result.summary.loc["candidate_basis_eligible", "design_weighted_person_mass"] + == 5 + ) + assert ( + result.summary.loc["candidate_basis_eligible", "union_household_design_mass"] + == 5 + ) + for name in ( + "retirement_detail", + "income_routing", + "original_household_membership", + ): + source[name] = source[name].iloc[::-1] + permuted = owner.build_asec_retirement_basis(**source) + pd.testing.assert_frame_equal( + permuted.person.loc[result.person.index], result.person, check_exact=True + ) + pd.testing.assert_frame_equal(permuted.summary, result.summary, check_exact=True) + result.provenance.iloc[0, 0] = "detached" + for name, original in before.items(): + current = source[name].loc[original.index] + if isinstance(original, pd.DataFrame): + pd.testing.assert_frame_equal(current, original, check_exact=True) + else: + pd.testing.assert_series_equal(current, original, check_exact=True) + + +def test_unknown_account_total_is_preserved_without_candidate_admission(owner): + source = inputs( + owner, [{"DST_YN_YNG": "1", "DST_SC1_YNG": "", "DST_VAL1_YNG": "100"}] + ) + assert source["income_routing"].retirement_distribution_known_amount.iloc[0] == 100 + p = owner.build_asec_retirement_basis(**source).person.iloc[0] + assert p.distribution_source_known_amount == 100 + assert np.isnan(p.distribution_candidate_upper) + assert not p.distribution_account_composition_known + + +def test_allocation_and_disclosure_flags_are_provenance_not_filters(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "100"} + before = build(owner, [row]) + changed = build(owner, [{**row, "I_PENVAL1": "4", "TPEN_VAL1": "1"}]) + pd.testing.assert_frame_equal(before.person, changed.person, check_exact=True) + pd.testing.assert_frame_equal(before.summary, changed.summary, check_exact=True) + assert changed.provenance["retirement_detail.I_PENVAL1_code"].iloc[0] == 4 + + +def test_outside_universe_unreadable_and_contradictory_evidence_stay_distinct(owner): + unresolved = build( + owner, [{"A_AGE": "14", "PEN_YN": "0", "PEN_VAL1": ""}] + ).person.iloc[0] + contradictory = build( + owner, + [ + { + "A_AGE": "14", + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + } + ], + ).person.iloc[0] + assert unresolved.pension_status == "unresolved_outside_reporting_universe" + assert contradictory.pension_status == "contradictory_outside_reporting_universe" + assert np.isnan(unresolved.pension_candidate_upper) and np.isnan( + contradictory.pension_candidate_upper + ) From 8bda23c2dfb07aa2bcf514c1d750e8981352eee4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:26:52 -0400 Subject: [PATCH 192/255] Fix retirement candidate accounting and unreadable off-route evidence --- .../us-current-asec-retirement-basis.added.md | 2 + docs/us-current-asec-retirement-basis.md | 33 ++- .../current_asec_retirement_basis.py | 46 ++-- .../test_us_current_asec_retirement_basis.py | 216 ++++++++++++++++++ 4 files changed, 279 insertions(+), 18 deletions(-) diff --git a/changelog.d/us-current-asec-retirement-basis.added.md b/changelog.d/us-current-asec-retirement-basis.added.md index 1b1fb2b7e..d7da7254e 100644 --- a/changelog.d/us-current-asec-retirement-basis.added.md +++ b/changelog.d/us-current-asec-retirement-basis.added.md @@ -1 +1,3 @@ Add a pure ASEC retirement candidate ledger with explicit measurement assumptions, source routing and knownness, conservative accounting intervals, preserved unresolved survivor/distribution scope, and original DESIGN-weight support diagnostics. It produces no fiscal inputs or source authority. + +Require exact applicable distribution aggregate balance and readable off-route evidence before admitting candidate intervals; preserve account evidence and signed residuals separately from interval availability. diff --git a/docs/us-current-asec-retirement-basis.md b/docs/us-current-asec-retirement-basis.md index d6c3eb58c..890b15df8 100644 --- a/docs/us-current-asec-retirement-basis.md +++ b/docs/us-current-asec-retirement-basis.md @@ -15,6 +15,14 @@ original interview ages. Shared amount observations and pension receipt codes must agree. A joint permutation is accepted; a one-sided permutation refuses. The current contract requires at least one selected person. +Membership must cover exactly those selected person IDs in the same order. +The weight Series must have a unique int64 `household_id` index containing +exactly the households represented by that membership, with finite nonnegative +values. Extra unselected household keys refuse, including keys from a larger +original survey. The caller must select the exact original household weights +without replacing them with clone, importance, or calibrated weights. Original +weight custody remains external; a Series does not authenticate DESIGN status. + It returns `AsecRetirementBasis` with six descriptive fields: - `person`: source identifiers, observed routed subtotals, candidate family @@ -89,6 +97,12 @@ The four published total-minus-slot comparisons are recomputed and checked against R1. `DBTN_VAL` compares the main distribution slots only; the applicable known distribution total is supplied independently by the routing owner. No comparison is allocated to another source or attributed causally to disclosure. +For age 58 and over, the main slots apply and this exact comparison must be +finite and zero before a distribution candidate interval is available. A +positive residual leaves additional scope unresolved; a negative residual is +contradictory accounting. Both retain the published DBTN total, applicable slot +total and signed difference without a cap, clipping or redistribution. The main +slot comparison does not constrain the separate applicable young slots below 58. For pension/disability candidate bounds, all relevant slots must be readable and receipt-consistent, and their exact integer-dollar sum must equal the aggregate. @@ -106,10 +120,21 @@ zero survivor family amount. Source 9 alongside positive annuity is explicitly flagged; no overlap priority or max/sum assumption is applied. A fully resolved applicable distribution composition produces a zero-to-total -candidate interval. Missing/unreadable account types, ambiguous active zeros or -off-route answers/dollars prevent that interval. The routing owner's known total -is still preserved when the stricter candidate composition test fails. Account -subtotals are finite only when the complete account composition is known. +candidate interval only when the applicable aggregate check also passes. +Missing/unreadable account types, ambiguous active zeros or off-route +answers/dollars prevent that interval. Off-route receipt and account literals +must be readable NIU codes, and off-route amounts must be readable zeros; +unreadable literals are uncertainty, not evidence of absence. The routing owner's +known total is still preserved when the stricter candidate test fails. + +Routed subtotals and distribution account subtotals are evidence only, not +admitted candidate amounts. In particular, account subtotals can be finite when +all account types/amounts are readable but the applicable receipt is unreadable +or the DBTN comparison fails. `distribution_account_composition_known` describes +that account evidence; it does not establish a valid receipt, aggregate balance, +regularity, netting, or an available interval. Consumers must use the candidate +interval availability and family statuses rather than infer admission from a +finite subtotal. Raw observations remain in the detached provenance regardless. Under-15 people retain their source evidence with outside, unresolved-outside or contradictory-outside status and no candidate zeros. Combined bounds are available diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py index 0db19ff8f..3113b96da 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_retirement_basis.py @@ -219,7 +219,7 @@ def _reference_agreement(d, r, published, ages): (None if np.isnan(code) else int(code), literal), kind, net_measure=False, - zero_is_dollars=field == "ANN_VAL", + zero_is_dollars=routing._zero_is_dollars(field), ) for age, code, literal, kind in zip( ages, receipt, receipt_status, kinds, strict=True @@ -430,7 +430,7 @@ def _family_row(family, receipt, total, difference, slots, age, assumptions): } -def _distributions(r, published, ages): +def _distributions(r, published, ages, differences): total, status = _known_routing(r, "retirement_distribution") receipt, receipt_literal = _codes( r, "retirement_distribution_receipt", routing.RECEIPT_CODE_DOMAIN @@ -448,7 +448,7 @@ def _distributions(r, published, ages): offroute_receipts = { name: _codes( r, "retirement_distribution_receipt_" + name, routing.RECEIPT_CODE_DOMAIN - )[0] + ) for name in ("young", "58") } roster, rows = [], [] @@ -489,9 +489,11 @@ def _distributions(r, published, ages): if active: account_amounts[int(code)] += value else: - offroute |= bool( - (np.isfinite(value) and value != 0) - or (np.isfinite(code) and code != 0) + offroute |= not bool( + np.isfinite(value) + and value == 0 + and literal == "in_printed_range" + and code == 0 ) roster.append( { @@ -513,8 +515,10 @@ def _distributions(r, published, ages): } ) other = "young" if age >= 58 else "58" - other_receipt = offroute_receipts[other] - offroute |= bool(np.isfinite(other_receipt[i]) and other_receipt[i] != 0) + other_receipt, other_literal = offroute_receipts[other] + offroute |= not bool( + other_literal[i] == "in_printed_range" and other_receipt[i] == 0 + ) if np.isfinite(total[i]) and known_composition: _require(total[i] == sum_applicable, "DISTRIBUTION_TOTAL") eligible = ( @@ -531,15 +535,27 @@ def _distributions(r, published, ages): or (receipt[i] == 2 and total[i] == 0), "DISTRIBUTION_RECEIPT", ) + candidate_status = ( + "known_nonreceipt" + if eligible and total[i] == 0 + else "regularity_netting_unresolved" + if eligible + else "unresolved_source_composition" + ) + if age >= 58 and (not np.isfinite(differences[i]) or differences[i] != 0): + eligible = False + candidate_status = ( + "unresolved_source_accounting" + if not np.isfinite(differences[i]) + else "contradictory_accounting" + if differences[i] < 0 + else "additional_scope_unresolved" + ) rows.append( { "distribution_candidate_lower": 0.0 if eligible else np.nan, "distribution_candidate_upper": total[i] if eligible else np.nan, - "distribution_status": "known_nonreceipt" - if eligible and total[i] == 0 - else "regularity_netting_unresolved" - if eligible - else "unresolved_source_composition", + "distribution_status": candidate_status, "distribution_source_known_amount": total[i], "distribution_source_reporting_status": status[i], "distribution_account_composition_known": bool( @@ -667,7 +683,9 @@ def build_asec_retirement_basis( family: _comparison(d, published, family, total, first, second) for family, total, first, second in detail.COMPARISONS } - distribution, distribution_records = _distributions(r, published, age) + distribution, distribution_records = _distributions( + r, published, age, differences["distribution"] + ) annuity, annuity_status = _known_routing(r, "pension_annuity_annuity") other_amount, other_clear, oi_candidate, oi_unspecified, oi_ss, oi_property = ( _other_income(r, age) diff --git a/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py index 5ea318635..b98fef380 100644 --- a/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py +++ b/packages/microcosm-build/tests/test_us_current_asec_retirement_basis.py @@ -475,3 +475,219 @@ def test_outside_universe_unreadable_and_contradictory_evidence_stay_distinct(ow assert np.isnan(unresolved.pension_candidate_upper) and np.isnan( contradictory.pension_candidate_upper ) + + +@pytest.mark.parametrize("total", [150, 50]) +def test_distribution_main_aggregate_difference_blocks_candidate_interval(owner, total): + result = build( + owner, + [ + { + "A_AGE": "60", + "DST_YN_YNG": "0", + "DST_YN": "1", + "DST_SC1": "4", + "DST_VAL1": "100", + "DBTN_VAL": str(total), + } + ], + ) + p = result.person.iloc[0] + assert p.distribution_accounting_difference == total - 100 + assert p.distribution_source_known_amount == 100 + assert p.distribution_account_4_amount == 100 + assert ( + result.provenance["retirement_detail.DBTN_VAL_published_amount"].iloc[0] + == total + ) + assert np.isnan(p.distribution_candidate_lower) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available and not p.candidate_basis_eligible + assert p.distribution_status == ( + "additional_scope_unresolved" if total > 100 else "contradictory_accounting" + ) + + +@pytest.mark.parametrize("token", ["", "malformed"]) +@pytest.mark.parametrize( + "age,field,retained", + [ + (40, "DST_YN", "receipt_58"), + (40, "DST_SC1", "slot1_account"), + (40, "DST_SC2", "slot2_account"), + (60, "DST_YN_YNG", "receipt_young"), + (60, "DST_SC1_YNG", "slot1_young_account"), + (60, "DST_SC2_YNG", "slot2_young_account"), + ], +) +def test_unreadable_offroute_distribution_literals_prevent_admission( + owner, age, field, retained, token +): + young = age < 58 + suffix = "_YNG" if young else "" + result = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: token, + } + ], + ) + p = result.person.iloc[0] + assert ( + result.provenance[ + "income_routing.retirement_distribution_" + retained + "_literal" + ].iloc[0] + == token + ) + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_basis_eligible + + +@pytest.mark.parametrize("age,field", [(40, "DST_VAL1"), (60, "DST_VAL1_YNG")]) +def test_unreadable_offroute_distribution_amount_is_not_assumed_zero(owner, age, field): + young = age < 58 + suffix = "_YNG" if young else "" + p = build( + owner, + [ + { + "A_AGE": str(age), + "DST_YN_YNG": "1" if young else "0", + "DST_YN": "0" if young else "1", + "DST_SC1" + suffix: "4", + "DST_VAL1" + suffix: "100", + "DBTN_VAL": "0" if young else "100", + field: "", + } + ], + ).person.iloc[0] + assert not p.distribution_account_composition_known + assert np.isnan(p.distribution_candidate_upper) + + +def test_account_subtotals_are_evidence_when_receipt_interval_is_unknown(owner): + p = build( + owner, [{"DST_YN_YNG": "", "DST_SC1_YNG": "4", "DST_VAL1_YNG": "100"}] + ).person.iloc[0] + assert p.distribution_account_composition_known + assert p.distribution_account_4_amount == 100 + assert np.isnan(p.distribution_source_known_amount) + assert np.isnan(p.distribution_candidate_upper) + assert not p.candidate_interval_available + + +@pytest.mark.parametrize( + "family,prefix,total,code1,code2,lower,upper,status", + [ + ("pension", "PEN", "PNSN_VAL", 1, 8, 60, 100, "candidate_under_assumptions"), + ("pension", "PEN", "PNSN_VAL", 1, 7, 60, 60, "candidate_under_assumptions"), + ( + "disability", + "DIS", + "DSAB_VAL", + 2, + 10, + 60, + 100, + "candidate_under_assumptions", + ), + ("disability", "DIS", "DSAB_VAL", 2, 1, 60, 60, "candidate_under_assumptions"), + ("disability", "DIS", "DSAB_VAL", 6, 1, 0, 0, "route_outside_only"), + ], +) +def test_two_slot_family_routes_preserve_candidate_and_unresolved_bounds( + owner, family, prefix, total, code1, code2, lower, upper, status +): + p = build( + owner, + [ + { + prefix + "_YN": "1", + prefix + "_SC1": str(code1), + prefix + "_SC2": str(code2), + prefix + "_VAL1": "60", + prefix + "_VAL2": "40", + total: "100", + } + ], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert p[family + "_candidate_lower"] == lower + assert p[family + "_candidate_upper"] == upper + assert p[family + "_status"] == status + assert p[family + "_accounting_difference"] == 0 + assert ( + p[family + "_observed_candidate_subtotal"] + + p[family + "_observed_unresolved_subtotal"] + == upper + ) + + +@pytest.mark.parametrize( + "family,total", + [("pension", "PNSN_VAL"), ("disability", "DSAB_VAL"), ("survivor", "SRVS_VAL")], +) +def test_no_receipt_with_nonzero_total_and_zero_slots_remains_contradictory( + owner, family, total +): + p = build(owner, [{total: "100"}]).person.iloc[0] + assert p[family + "_status"] == "contradictory_accounting" + assert p[family + "_accounting_difference"] == 100 + assert p[family + "_observed_candidate_subtotal"] == 0 + assert np.isnan(p[family + "_candidate_upper"]) + assert not p.candidate_basis_eligible + + +def test_equal_bounds_under_explicit_scenarios_do_not_claim_identification(owner): + row = { + "PEN_YN": "1", + "PEN_SC1": "1", + "PEN_VAL1": "100", + "PNSN_VAL": "100", + "DIS_YN": "1", + "DIS_SC1": "2", + "DIS_VAL1": "40", + "DSAB_VAL": "40", + "ANN_YN": "1", + "ANN_VAL": "20", + } + unresolved = build(owner, [row]).person.iloc[0] + p = build( + owner, + [row], + pension_annuity_regularity="assume_regular", + disability_pension_eligibility="assume_qualifying", + ).person.iloc[0] + assert ( + unresolved.retirement_candidate_lower == 0 + and unresolved.retirement_candidate_upper == 160 + ) + assert not unresolved.candidate_point_under_assumptions + assert p.retirement_candidate_lower == p.retirement_candidate_upper == 160 + assert p.candidate_point_under_assumptions and p.candidate_basis_eligible + assert not p.point_identified and not p.fiscal_outputs_produced + + +def test_diagnostic_household_mass_deduplicates_only_selected_households(owner): + row = {"PEN_YN": "1", "PEN_SC1": "1", "PEN_VAL1": "100", "PNSN_VAL": "150"} + result = build(owner, [row, row, {}, row, {}]) + selected = result.summary.loc["diagnostic:pension_candidate_unresolved"] + assert selected.person_count == 3 and selected.household_count == 2 + assert selected.design_weighted_person_mass == 7 + assert selected.union_household_design_mass == 5 + + +def test_design_weights_refuse_an_unselected_household(owner): + source = inputs(owner, [{}]) + source["original_household_design_weights"].loc[999] = 4.0 + with pytest.raises(ValueError, match="DESIGN_MEMBERSHIP"): + owner.build_asec_retirement_basis(**source) From b8f52c4e84a00fe8b3fcf457be208ff4605861ec Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:37:50 -0400 Subject: [PATCH 193/255] Add deterministic property completion routing diagnostics --- .../property-completion-routing.added.md | 1 + docs/us-property-completion-routing.md | 39 ++ .../current_property_completion_routing.py | 599 ++++++++++++++++++ .../test_us_property_completion_routing.py | 401 ++++++++++++ 4 files changed, 1040 insertions(+) create mode 100644 changelog.d/property-completion-routing.added.md create mode 100644 docs/us-property-completion-routing.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py create mode 100644 packages/microcosm-build/tests/test_us_property_completion_routing.py diff --git a/changelog.d/property-completion-routing.added.md b/changelog.d/property-completion-routing.added.md new file mode 100644 index 000000000..b7e842805 --- /dev/null +++ b/changelog.d/property-completion-routing.added.md @@ -0,0 +1 @@ +Add deterministic original-person property completion routing, independent interest/dividend knownness, exact clone identity diagnostics and original DESIGN support summaries. The pure operation assigns no amounts and grants no source or release authority; host artifact wiring remains separate. diff --git a/docs/us-property-completion-routing.md b/docs/us-property-completion-routing.md new file mode 100644 index 000000000..ebeb7ca74 --- /dev/null +++ b/docs/us-property-completion-routing.md @@ -0,0 +1,39 @@ +# Property completion routing + +`current_property_completion_routing.build_property_completion_routing(qualified, clone_frame)` describes the remaining ordinary-interest and dividend work over the original ACS/ASEC source population. It assigns no amounts, fits no model and changes no Frame. Its `PropertyCompletionRouting` return value is descriptive; constructing it grants no source, complete-parent or release authority. + +The inputs are the existing `QualifiedPropertyIncomeSources` description and the complete initial clone Frame. The operation checks original source order, integer identities, source-native agreement, source ages, component knownness, the exact two clone roles and original household DESIGN weights. It preserves the complete input physical descriptions. Actual source custody remains external: a country host must retain and requalify the preparation and source issuers around relevant I/O. Neither the description's type nor a separately supplied DESIGN vector authenticates that custody. + +## Routes and retained reasons + +Each original person receives one route: + +| Route | Meaning | +| --- | --- | +| `unsupported_under15_measurement` | The source age is below15. No child amount or adult extrapolation is supplied. | +| `source_review_required` | A required source amount/receipt or ACS anchor is malformed, outside its published domain, inconsistent or has an invalid adjustment. | +| `carry_known_components` | Both independent ASEC components are known. Joint-fit exclusion does not erase them. | +| `existing_acs_anchor_decomposition` | The adult ACS aggregate anchor is observed and qualifies for the existing decomposition branch. The separate components remain unresolved at this stage. | +| `acs_anchor_completion_review` | An adult ACS anchor remains missing. This operation supplies no fallback. | +| `asec_component_completion_review` | At least one adult ASEC component remains unresolved. Any independently known other component is preserved. | + +Under15 is the first route; malformed, positive-outside-universe and contradictory observations remain visible as overlapping reasons. Adult source errors precede generic missingness. Joint-donor exclusions remain separately named reasons and never by themselves turn otherwise known O/D into unknowns. Unknown new source statuses refuse classification instead of silently entering a generic completion branch. + +The long component table preserves amount, individual knownness and the exact reporting/literal statuses. `observed` denotes a qualified observed amount; `derived` distinguishes a qualified known-nonreceipt zero; `unresolved` retains missing amounts. An observed ACS aggregate is recorded separately from its unresolved individual components. No component receives a `modeled` origin here; only a later operation consuming actual checked model outputs can assign that origin. Allocation and disclosure flags remain source provenance rather than donor filters. + +## Outputs and support + +- `person`: one row per original, with source/native identity, source age, original household, independent O/D knownness, separate ACS anchor and exclusive route. +- `components`: two rows per original for the two required component families. +- `reasons`: all overlapping required-source and joint-fit diagnostics. +- `clones`: exact original/native/source identity and clone roles mapped to the initial receiving person IDs. +- `summary`: raw original-person and unique-household counts, household-inherited DESIGN person mass and union-household DESIGN mass for each route/reason. Clones do not multiply these summaries. Zero-weight support remains visible in raw counts. +- `payload`: deterministic JSON with the tables and explicit statements that it neither assigns amounts nor authenticates a source or complete parent. Integer IDs remain integers, including those above2⁵³; unknown amounts serialize as null. + +These are support diagnostics, not prevalence estimates or calibrated totals. Overlapping reason masses must not be added as if the reasons were disjoint. + +## Integration boundary + +`property_completion_artifact_output()` supplies the typed `completion_routing` artifact declaration. This additive slice does not connect a new node or change the accepted38-node host. The next host integration should emit the payload at the existing property source-projection boundary, bind the actual retained source and clone descriptions, and recompute/compare it during requalification and reconstruction. That host change requires its own source/graph identity and observation checks. + +This operation does not create a child-income measurement model, fill an unknown adult input, assert that the PUF covers a missing source population or relax the existing completeness gate. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py new file mode 100644 index 000000000..9e936603a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_property_completion_routing.py @@ -0,0 +1,599 @@ +"""Pure completion diagnostics over original source descriptions; no authority. + +The retaining country host must qualify sources and check the complete parent +around relevant I/O. This module diagnoses descriptions, never issues that +authority, fits a model, or assigns an amount to a population. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from math import fsum + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ArtifactOutput, ArtifactType + +from . import current_property_income_sources as sources +from . import support_provenance as provenance +from .property_income_constants import PROPERTY_COMPONENTS + +PROTOCOL = "microcosm.us.property-completion-routing.v1" +PROPERTY_COMPLETION_TYPE = ArtifactType("microcosm.us.property_completion_routing", 1) +_COMPONENT_FIELDS = ( + (PROPERTY_COMPONENTS[0], "TRDINT_VAL", "ordinary_interest_known"), + (PROPERTY_COMPONENTS[2], "DIV_VAL", "dividends_known"), +) +# Routing-policy categories, not alternative source codebooks. Raw source +# statuses remain visible and an unreviewed new status refuses classification. +_ASEC_STATUSES = frozenset( + { + "known_receipt", + "known_nonreceipt", + "observed_zero_component", + "outside_reporting_universe", + "contradictory_outside_reporting_universe", + "missing_receipt_literal", + "unrecognized_receipt_literal", + "missing_amount", + "invalid_amount_literal", + "niu", + "contradictory_niu_nonzero", + "contradictory_no_nonzero", + "ambiguous_recipient_zero", + } +) +_ACS_STATUSES = frozenset( + { + "observed", + "missing_source_amount", + "malformed_source_amount", + "outside_published_domain", + "invalid_adjustment", + "outside_universe_blank", + "outside_universe_observation", + } +) +_LITERAL_STATUSES = frozenset( + {"missing", "malformed", "outside_printed_range", "in_printed_range"} +) + + +@dataclass(frozen=True) +class PropertyCompletionRouting: + """Detached diagnostics. This value is not a source or parent admission.""" + + person: pd.DataFrame + components: pd.DataFrame + reasons: pd.DataFrame + clones: pd.DataFrame + summary: pd.DataFrame + payload: bytes + + +def property_completion_artifact_output() -> ArtifactOutput: + """Declaration for future source-projection wiring; not an attached node.""" + return ArtifactOutput("completion_routing", PROPERTY_COMPLETION_TYPE) + + +def _require(value, reason): + if not value: + raise ValueError("PROPERTY_COMPLETION_" + reason) + + +def _ids(values, reason): + _require(values.dtype == np.dtype("int64"), reason) + return values.to_numpy(copy=True) + + +def _axis(table, expected, reason): + _require( + type(table) is pd.DataFrame + and table.columns.is_unique + and table.index.name == "person_id" + and table.index.dtype == np.dtype("int64") + and table.index.is_unique + and table.index.equals(expected.index), + reason, + ) + _require( + table.native_person_id.dtype == np.dtype("int64") + and table.native_person_id.equals(expected.native_person_id), + reason + "_NATIVE", + ) + + +def _source_error(status, literal=None): + return ( + status.startswith("contradictory_") + or status + in { + "invalid_amount_literal", + "unrecognized_receipt_literal", + "malformed_source_amount", + "outside_published_domain", + "invalid_adjustment", + } + or literal in {"malformed", "outside_printed_range"} + ) + + +def _origin_agreement(qualified, origins): + columns = ["person_id", "native_person_id", "source"] + _require( + qualified.shared_predictors.origins[columns].equals(origins[columns]), + "SHARED_ORIGIN_IDENTITY", + ) + document = json.loads(qualified.origin_document) + records = document["persons"] + table = pd.DataFrame(records["rows"], columns=records["columns"]) + _require( + np.array_equal(_ids(table.person_id, "DOCUMENT_PERSON_ID"), origins.index) + and np.array_equal( + _ids(table.selected_receiving_person_id, "DOCUMENT_NATIVE_ID"), + origins.native_person_id, + ) + and np.array_equal(table.source, origins.source), + "DOCUMENT_ORIGIN_IDENTITY", + ) + for frame in (qualified.source_frame, qualified.shared_predictors.source_frame): + _require( + np.array_equal( + _ids( + frame.person[provenance.spine_source_id_column("person")], + "SOURCE_NATIVE_ID", + ), + origins.native_person_id, + ) + and np.array_equal( + frame.person[provenance.support_channel_column("person")].astype(str), + origins.source, + ), + "SOURCE_FRAME_ORIGIN_IDENTITY", + ) + + +def _asec_components(qualified, origins): + tables = ( + qualified.asec_interest_values.person, + qualified.asec_dividend_values.person, + ) + rows = [] + for table, (component, field, _) in zip(tables, _COMPONENT_FIELDS, strict=True): + _axis(table, origins, "ASEC_AXIS") + statuses = table[field + "_reporting_status"] + literals = table[field + "_literal_status"] + _require( + statuses.notna().all() and set(statuses) <= _ASEC_STATUSES, "ASEC_STATUS" + ) + _require( + literals.notna().all() and set(literals) <= _LITERAL_STATUSES, + "LITERAL_STATUS", + ) + known = table[field + "_amount_known"] + amount = table[field + "_amount"].to_numpy(dtype=np.float64, na_value=np.nan) + _require(known.dtype == np.dtype("bool"), "KNOWNNESS_DTYPE") + expected = statuses.isin( + ("known_receipt", "known_nonreceipt", "observed_zero_component") + ) + if field == "DIV_VAL": + _require( + not statuses.eq("observed_zero_component").any(), "DIVIDEND_ZERO_STATUS" + ) + _require( + known.equals(expected.rename(known.name)) + and np.array_equal(known, np.isfinite(amount)) + and not np.isinf(amount).any() + and not (amount < 0).any(), + "COMPONENT_KNOWNNESS", + ) + basis = qualified.donor_basis.person[component].to_numpy(dtype=np.float64) + _require( + np.array_equal(np.isnan(basis), np.isnan(amount)) + and np.array_equal( + basis[known].view("uint64"), amount[known].view("uint64") + ), + "BASIS_COMPONENT_AGREEMENT", + ) + published = table[field + "_published_amount"].to_numpy( + dtype=np.float64, na_value=np.nan + ) + _require( + np.array_equal( + published[known].view("uint64"), amount[known].view("uint64") + ), + "PUBLISHED_COMPONENT_AGREEMENT", + ) + for original, value, is_known, status, literal in zip( + table.index, amount, known, statuses, literals, strict=True + ): + derived = status == "known_nonreceipt" + _require(not derived or value == 0, "NONRECEIPT_NONZERO") + rows.append( + ( + int(original), + component, + float(value), + bool(is_known), + "derived" + if derived + else ("observed" if is_known else "unresolved"), + status, + literal, + "qualified_known_nonreceipt" + if derived + else ( + "observed_component_zero" if is_known and value == 0 else "none" + ), + ) + ) + return rows + + +def _clones(frame, origins): + _require(type(frame) is Frame, "CLONE_FRAME_TYPE") + table = frame.person + ids = _ids(table.person_id, "CLONE_ID_DTYPE") + original = _ids( + table[provenance.support_source_id_column("person")], "CLONE_ORIGIN_DTYPE" + ) + native = _ids( + table[provenance.spine_source_id_column("person")], "CLONE_NATIVE_DTYPE" + ) + role = _ids( + table[provenance.support_clone_index_column("person")], "CLONE_ROLE_DTYPE" + ) + source = table[provenance.support_channel_column("person")].astype(str).to_numpy() + _require( + table.person_id.is_unique and np.isin(original, origins.index).all(), + "CLONE_COVERAGE", + ) + lookup = origins.reindex(original) + _require( + np.array_equal(native, lookup.native_person_id) + and np.array_equal(source, lookup.source), + "CLONE_SOURCE_IDENTITY", + ) + result = pd.DataFrame( + { + "person_id": ids, + "original_person_id": original, + "native_person_id": native, + "source": source, + "clone_index": role, + } + ) + _require( + np.isin(role, (0, 1)).all() + and not result.duplicated(["original_person_id", "clone_index"]).any() + and result.groupby("original_person_id").size().eq(2).all() + and set(original) == set(origins.index), + "WHOLE_CLONE_PAIRS", + ) + return result.set_index("person_id") + + +def _original_design(frame, origins): + _require( + type(frame) is Frame and frame.schema.person_entity == "person", "SOURCE_FRAME" + ) + _require( + np.array_equal(_ids(frame.person.person_id, "SOURCE_ID_DTYPE"), origins.index), + "SOURCE_FRAME_AXIS", + ) + weights = frame.weights_for("household") + _require( + weights.kind is WeightKind.DESIGN + and frame.resolve_weights("person").kind is WeightKind.DESIGN, + "ORIGINAL_DESIGN_KIND", + ) + households = _ids(frame.table("household").household_id, "HOUSEHOLD_ID_DTYPE") + membership = _ids(frame.person.person_household_id, "MEMBERSHIP_DTYPE") + _require( + len(set(households)) == len(households) + and np.isin(membership, households).all(), + "HOUSEHOLD_MEMBERSHIP", + ) + values = np.array(weights.values, dtype=np.float64, copy=True) + _require(np.isfinite(values).all() and (values >= 0).all(), "DESIGN_VALUES") + series = pd.Series(values, index=pd.Index(households, name="household_id")) + return membership, series, series.loc[membership].to_numpy(copy=True) + + +def _table_document(table): + def cell(value): + if value is pd.NA or value is None: + return None + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, float): + if np.isnan(value): + return None + _require(np.isfinite(value), "PAYLOAD_NONFINITE") + _require(type(value) in (str, int, float, bool), "PAYLOAD_SCALAR") + return value + + # itertuples preserves integer scalar types; iterrows can coerce IDs >2**53. + return { + "columns": list(table.columns), + "index": [cell(v) for v in table.index], + "index_name": table.index.name, + "rows": [ + [cell(v) for v in row] for row in table.itertuples(index=False, name=None) + ], + } + + +def build_property_completion_routing( + qualified: sources.QualifiedPropertyIncomeSources, clone_frame: Frame +) -> PropertyCompletionRouting: + """Diagnose original component availability; never consume native I/O or fit. + + The source descriptions and the supplied Frame/weights are not admissions. + An integrating host retains and requalifies their real issuers around I/O. + """ + _require( + type(qualified) is sources.QualifiedPropertyIncomeSources, + "QUALIFIED_DESCRIPTION_TYPE", + ) + before = sources.property_income_sources_seal(qualified) + clone_before = sources._frame_seal(clone_frame) + origins = qualified.origins + _require( + type(origins) is pd.DataFrame + and origins.index.name == "person_id" + and origins.index.dtype == np.dtype("int64") + and origins.index.is_unique + and origins.columns.is_unique + and len(origins) > 0, + "ORIGIN_AXIS", + ) + _ids(origins.native_person_id, "NATIVE_ID_DTYPE") + _require( + np.array_equal(_ids(origins.person_id, "ORIGIN_ID_DTYPE"), origins.index) + and set(origins.source) <= {"acs", "asec"}, + "ORIGIN_IDENTITY", + ) + _origin_agreement(qualified, origins) + asec = origins.loc[origins.source.eq("asec")] + acs = origins.loc[origins.source.eq("acs")] + for table in (qualified.donor_basis.person, qualified.asec_routing_values.person): + _axis(table, asec, "ASEC_BASIS_AXIS") + anchors = qualified.acs_anchor_values.anchors + _axis(anchors, acs, "ACS_AXIS") + ages = pd.Series(index=origins.index, dtype="float64") + ages.loc[asec.index] = qualified.asec_interest_values.person.source_age + ages.loc[acs.index] = pd.to_numeric(anchors.AGEP, errors="coerce") + _require( + np.isfinite(ages).all() + and ages.between(0, 99).all() + and np.equal(ages, np.floor(ages)).all(), + "SOURCE_AGE", + ) + for table in ( + qualified.asec_dividend_values.person, + qualified.asec_routing_values.person, + qualified.donor_basis.person, + ): + _require( + np.array_equal(table.source_age, ages.loc[asec.index]), + "SOURCE_AGE_AGREEMENT", + ) + components = _asec_components(qualified, asec) + anchor_status = anchors.property_income_status + _require( + anchor_status.notna().all() and set(anchor_status) <= _ACS_STATUSES, + "ACS_STATUS", + ) + known_anchor = anchors.property_income_known + anchor_values = anchors.property_income_amount.to_numpy( + dtype=np.float64, na_value=np.nan + ) + _require( + known_anchor.dtype == np.dtype("bool") + and np.array_equal(known_anchor, anchor_status.eq("observed")) + and np.array_equal(known_anchor, np.isfinite(anchor_values)) + and np.array_equal( + anchors.property_income_in_income_universe, ages.loc[acs.index] >= 15 + ), + "ACS_KNOWNNESS_OR_UNIVERSE", + ) + for original in acs.index: + for component, _, _ in _COMPONENT_FIELDS: + components.append( + ( + int(original), + component, + np.nan, + False, + "unresolved", + "component_not_separately_observed", + "not_separately_observed", + "none", + ) + ) + components = pd.DataFrame( + components, + columns=[ + "original_person_id", + "component", + "amount", + "known", + "origin", + "reporting_status", + "literal_status", + "zero_basis", + ], + ) + order = {int(original): i for i, original in enumerate(origins.index)} + components["_order"] = components.original_person_id.map(order) + components = ( + components.sort_values(["_order", "component"], kind="stable") + .drop(columns="_order") + .reset_index(drop=True) + ) + membership, weights, person_design = _original_design( + qualified.source_frame, origins + ) + clones = _clones(clone_frame, origins) + person = origins[["native_person_id", "source"]].copy(deep=True) + person["source_age"] = ages.astype("int64") + person["original_household_id"] = membership + person["original_household_design_weight"] = person_design + person["anchor_known"] = False + person["anchor_amount"] = np.nan + person["anchor_status"] = "not_an_acs_aggregate_anchor" + person["anchor_origin"] = "unresolved" + person.loc[acs.index, "anchor_known"] = known_anchor.to_numpy() + person.loc[acs.index, "anchor_amount"] = anchor_values + person.loc[acs.index, "anchor_status"] = anchor_status.to_numpy() + person.loc[acs.index[known_anchor], "anchor_origin"] = "observed" + reasons = pd.DataFrame( + False, + index=origins.index, + columns=[ + "under15", + "source_error", + "contradictory_evidence", + "positive_outside_universe", + "missing_source_evidence", + "ambiguous_zero", + "niu", + "ordinary_interest_unknown", + "dividends_unknown", + ], + ) + reasons["under15"] = ages < 15 + for component, field, known_name in _COMPONENT_FIELDS: + selected = ( + components.loc[components.component.eq(component)] + .set_index("original_person_id") + .reindex(origins.index) + ) + person[known_name] = selected.known.to_numpy() + reasons[known_name.replace("_known", "_unknown")] = ~person[known_name] + status, literal = selected.reporting_status, selected.literal_status + reasons["source_error"] |= np.array( + [ + _source_error(status_value, literal_value) + for status_value, literal_value in zip(status, literal, strict=True) + ] + ) + reasons["contradictory_evidence"] |= status.str.startswith("contradictory_") + reasons["missing_source_evidence"] |= status.str.startswith("missing_") + reasons["ambiguous_zero"] |= status.eq("ambiguous_recipient_zero") + reasons["niu"] |= status.eq("niu") + table = ( + qualified.asec_interest_values.person + if field == "TRDINT_VAL" + else qualified.asec_dividend_values.person + ) + published = table[field + "_published_amount"].to_numpy( + dtype=np.float64, na_value=np.nan + ) + reasons.loc[asec.index, "positive_outside_universe"] |= ( + ages.loc[asec.index] < 15 + ) & (published > 0) + reasons.loc[acs.index, "source_error"] |= np.array( + [_source_error(s) for s in anchor_status], dtype=bool + ) + reasons.loc[acs.index, "missing_source_evidence"] |= anchor_status.eq( + "missing_source_amount" + ) + reasons.loc[acs.index, "positive_outside_universe"] |= ( + ages.loc[acs.index] < 15 + ) & (pd.to_numeric(anchors.INTP, errors="coerce") > 0) + reasons.loc[acs.index, "contradictory_evidence"] |= anchor_status.eq( + "outside_universe_observation" + ) + for name in qualified.donor_basis.exclusions: + column = qualified.donor_basis.exclusions[name] + _require( + column.index.equals(asec.index) and column.dtype == np.dtype("bool"), + "JOINT_EXCLUSION_AXIS", + ) + reasons["joint:" + name] = False + reasons.loc[asec.index, "joint:" + name] = column.to_numpy() + routes = [] + for original in origins.index: + if reasons.loc[original, "under15"]: + route = "unsupported_under15_measurement" + elif reasons.loc[original, "source_error"]: + route = "source_review_required" + elif ( + person.loc[original, "ordinary_interest_known"] + and person.loc[original, "dividends_known"] + ): + route = "carry_known_components" + elif person.loc[original, "source"] == "acs": + route = ( + "existing_acs_anchor_decomposition" + if person.loc[original, "anchor_known"] + else "acs_anchor_completion_review" + ) + else: + route = "asec_component_completion_review" + routes.append(route) + person["completion_route"] = routes + masks = {"all": np.ones(len(person), dtype=bool)} + masks.update( + { + "route:" + r: person.completion_route.eq(r).to_numpy() + for r in sorted(set(routes)) + } + ) + masks.update({"reason:" + c: reasons[c].to_numpy() for c in reasons}) + summary_rows = [] + for name, mask in masks.items(): + households = np.unique(membership[mask]) + mass = fsum(person_design[mask]) + household_mass = fsum(weights.loc[households]) + _require( + np.isfinite(mass) and np.isfinite(household_mass), "DESIGN_MASS_OVERFLOW" + ) + summary_rows.append( + (name, int(mask.sum()), len(households), mass, household_mass) + ) + summary = pd.DataFrame( + summary_rows, + columns=[ + "selection", + "person_count", + "household_count", + "design_weighted_person_mass", + "union_household_design_mass", + ], + ).set_index("selection") + tables = { + "person": person, + "components": components, + "reasons": reasons, + "clones": clones, + "summary": summary, + } + payload = json.dumps( + { + "protocol": PROTOCOL, + "source_authority": False, + "complete_parent_authority": False, + "amounts_assigned": False, + "model_executed": False, + "source_input": "descriptive_qualified_values; host_custody_required", + "stage": "before_property_models", + "tables": {k: _table_document(v) for k, v in tables.items()}, + }, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + _require( + sources.property_income_sources_seal(qualified) == before + and sources._frame_seal(clone_frame) == clone_before, + "INPUTS_CHANGED", + ) + return PropertyCompletionRouting( + person, components, reasons, clones, summary, payload + ) diff --git a/packages/microcosm-build/tests/test_us_property_completion_routing.py b/packages/microcosm-build/tests/test_us_property_completion_routing.py new file mode 100644 index 000000000..02057158c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_property_completion_routing.py @@ -0,0 +1,401 @@ +"""Invented qualified descriptions, routing and support; no model or engine.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_current_property_income_sources import ( # noqa: F401 + _detached, + _values, + actual, +) + +from microcosm.build.us_runtime import current_property_completion_routing as routing +from microcosm.build.us_runtime import current_property_income_sources as sources +from microcosm.frame import Frame, WeightKind, Weights + + +def test_artifact_declares_diagnostics_without_source_authority(): + output = routing.property_completion_artifact_output() + assert output.name == "completion_routing" + assert output.type == routing.PROPERTY_COMPLETION_TYPE + assert routing.PROTOCOL == "microcosm.us.property-completion-routing.v1" + + +def _rebuild(qualified, edit): + values = list(_detached(_values(qualified))) + edit(values) + # Keep the existing descriptive projection binding coherent after a pure + # fixture mutation. This does not issue a preparation or qualify new bytes. + projection = ( + values[1].anchors.reset_index().to_json(orient="table", index=False).encode() + ) + values[1] = replace( + values[1], + projection=projection, + evidence={ + **values[1].evidence, + "projection_sha256": hashlib.sha256(projection).hexdigest(), + }, + ) + return sources._compose(tuple(values), qualified.origin_document) + + +def _asec_id(qualified, native=107): + return qualified.origins.index[ + qualified.origins.source.eq("asec") + & qualified.origins.native_person_id.eq(native) + ][0] + + +def _set_amount(table, original, field, amount, status): + table.loc[original, field + "_published_amount"] = amount + table.loc[original, field + "_literal_status"] = ( + "missing" if amount is None else "in_printed_range" + ) + table.loc[original, field + "_reporting_status"] = status + known = status in ("known_receipt", "observed_zero_component", "known_nonreceipt") + table.loc[original, field + "_amount_known"] = known + table.loc[original, field + "_amount"] = amount if known else pd.NA + + +def test_actual_source_roster_preserves_independent_knownness_and_all_inputs(actual): # noqa: F811 + live, qualified, _ = actual + before = sources.property_income_sources_seal(qualified) + frame_before = sources._frame_seal(live.clone_population.frame) + result = routing.build_property_completion_routing( + qualified, live.clone_population.frame + ) + assert result.person.index.equals(qualified.origins.index) + assert len(result.components) == 2 * len(result.person) + assert len(result.clones) == 2 * len(result.person) + original = _asec_id(qualified) + assert result.person.loc[original, "completion_route"] == "carry_known_components" + assert result.reasons.loc[original, "joint:survivor_possible_property"] + pair = result.components.loc[result.components.original_person_id.eq(original)] + assert pair.amount.eq(500).all() and pair.known.all() + assert pair.origin.eq("observed").all() + acs = result.person.source.eq("acs") + eligible = acs & result.person.anchor_known + assert eligible.any() + assert ( + result.person.loc[eligible, "completion_route"] + .eq("existing_acs_anchor_decomposition") + .all() + ) + assert result.person.loc[eligible, "anchor_origin"].eq("observed").all() + assert ( + result.components.loc[ + result.components.original_person_id.isin(result.person.index[acs]), + "origin", + ] + .eq("unresolved") + .all() + ) + assert not result.components.origin.eq("modeled").any() + payload = json.loads(result.payload) + assert payload["source_authority"] is False + assert payload["complete_parent_authority"] is False + assert payload["amounts_assigned"] is False + assert sources.property_income_sources_seal(qualified) == before + assert sources._frame_seal(live.clone_population.frame) == frame_before + assert ( + routing.build_property_completion_routing( + qualified, live.clone_population.frame + ).payload + == result.payload + ) + + +@pytest.mark.parametrize("field", ["TRDINT_VAL", "DIV_VAL"]) +def test_missing_one_component_preserves_known_other(actual, field): # noqa: F811 + live, qualified, _ = actual + original = _asec_id(qualified) + + def edit(values): + table = values[2 if field == "TRDINT_VAL" else 4].person + _set_amount(table, original, field, None, "missing_amount") + + changed = _rebuild(qualified, edit) + result = routing.build_property_completion_routing( + changed, live.clone_population.frame + ) + assert ( + result.person.loc[original, "completion_route"] + == "asec_component_completion_review" + ) + rows = result.components.loc[result.components.original_person_id.eq(original)] + assert rows.known.sum() == 1 + assert rows.loc[rows.known, "amount"].iloc[0] == 500.0 + assert rows.loc[~rows.known, "amount"].isna().all() + assert rows.loc[~rows.known, "origin"].eq("unresolved").all() + + +@pytest.mark.parametrize( + "status,amount,expected", + [ + ("missing_source_amount", None, "acs_anchor_completion_review"), + ("malformed_source_amount", None, "source_review_required"), + ("outside_published_domain", None, "source_review_required"), + ("invalid_adjustment", None, "source_review_required"), + ("observed", -100.0, "existing_acs_anchor_decomposition"), + ("observed", 0.0, "existing_acs_anchor_decomposition"), + ], +) +def test_acs_anchor_route_is_not_component_observation( + actual, # noqa: F811 + status, + amount, + expected, +): + live, qualified, _ = actual + original = qualified.recipient_columns.index[0] + + def edit(values): + anchors = values[1].anchors + anchors.loc[original, "property_income_status"] = status + anchors.loc[original, "property_income_amount"] = amount + anchors.loc[original, "property_income_known"] = status == "observed" + + result = routing.build_property_completion_routing( + _rebuild(qualified, edit), live.clone_population.frame + ) + assert result.person.loc[original, "completion_route"] == expected + rows = result.components.loc[result.components.original_person_id.eq(original)] + assert not rows.known.any() and rows.amount.isna().all() + + +@pytest.mark.parametrize( + "status", + [ + "outside_universe_blank", + "outside_universe_observation", + "malformed_source_amount", + ], +) +def test_under15_keeps_all_source_reasons_without_zero(actual, status): # noqa: F811 + live, qualified, _ = actual + anchors = qualified.acs_anchor_values.anchors + original = anchors.index[~anchors.property_income_in_income_universe][0] + + def edit(values): + values[1].anchors.loc[original, "property_income_status"] = status + if status == "outside_universe_observation": + values[1].anchors.loc[original, "INTP"] = "100" + + result = routing.build_property_completion_routing( + _rebuild(qualified, edit), live.clone_population.frame + ) + assert ( + result.person.loc[original, "completion_route"] + == "unsupported_under15_measurement" + ) + assert result.person.loc[original, "anchor_status"] == status + assert result.reasons.loc[original, "under15"] + assert result.reasons.loc[original, "source_error"] == ( + status == "malformed_source_amount" + ) + assert result.reasons.loc[original, "positive_outside_universe"] == ( + status == "outside_universe_observation" + ) + rows = result.components.loc[result.components.original_person_id.eq(original)] + assert not rows.known.any() and rows.amount.isna().all() + + +@pytest.mark.parametrize( + "status", + [ + "invalid_amount_literal", + "contradictory_no_nonzero", + "unrecognized_receipt_literal", + ], +) +def test_asec_source_error_precedes_missingness_but_keeps_other_known(actual, status): # noqa: F811 + live, qualified, _ = actual + original = _asec_id(qualified) + + def edit(values): + _set_amount(values[4].person, original, "DIV_VAL", 500.0, status) + + result = routing.build_property_completion_routing( + _rebuild(qualified, edit), live.clone_population.frame + ) + assert result.person.loc[original, "completion_route"] == "source_review_required" + assert result.person.loc[original, "ordinary_interest_known"] + assert not result.person.loc[original, "dividends_known"] + + +def test_nonreceipt_zero_and_observed_component_zero_have_distinct_origins(actual): # noqa: F811 + live, qualified, _ = actual + original = _asec_id(qualified) + + def edit(values): + _set_amount( + values[2].person, original, "TRDINT_VAL", 0.0, "observed_zero_component" + ) + _set_amount(values[4].person, original, "DIV_VAL", 0.0, "known_nonreceipt") + values[4].person.loc[original, "DIV_YN_code"] = 2 + + result = routing.build_property_completion_routing( + _rebuild(qualified, edit), live.clone_population.frame + ) + rows = result.components.loc[ + result.components.original_person_id.eq(original) + ].set_index("component") + assert rows.loc["property_ordinary_interest", "origin"] == "observed" + assert rows.loc["property_dividends", "origin"] == "derived" + assert rows.loc["property_dividends", "zero_basis"] == "qualified_known_nonreceipt" + assert rows.amount.eq(0).all() + + +def test_allocation_flags_do_not_change_completion_route(actual): # noqa: F811 + live, qualified, _ = actual + original = _asec_id(qualified) + + def edit(values): + values[4].person.loc[original, "I_DIVVAL_code"] = 1 + values[4].person.loc[original, "I_DIVYN_code"] = 8 + values[4].person.loc[original, "I_DIVYN_codebook_status"] = ( + "header_reference_conflict" + ) + + result = routing.build_property_completion_routing( + _rebuild(qualified, edit), live.clone_population.frame + ) + assert result.person.loc[original, "completion_route"] == "carry_known_components" + + +def test_summary_counts_originals_and_union_households_not_clones(actual): # noqa: F811 + live, qualified, _ = actual + result = routing.build_property_completion_routing( + qualified, live.clone_population.frame + ) + row = result.summary.loc["all"] + frame = qualified.source_frame + assert row.person_count == len(result.person) + assert row.household_count == frame.n("household") + assert row.design_weighted_person_mass == sum( + frame.resolve_weights("person").values + ) + assert row.union_household_design_mass == sum(frame.weights_for("household").values) + + +@pytest.mark.parametrize( + "fault", ["missing_clone", "duplicate_role", "native_id", "source_label"] +) +def test_clone_pair_identity_refuses(actual, fault): # noqa: F811 + live, qualified, _ = actual + frame = _detached(live.clone_population.frame) + people = frame.person + p = routing.provenance + if fault == "missing_clone": + keep = np.ones(len(people), dtype=bool) + keep[0] = False + frame = frame.select(keep) + elif fault == "duplicate_role": + people[p.support_clone_index_column("person")] = 0 + elif fault == "native_id": + people.loc[people.index[0], p.spine_source_id_column("person")] += 1 + else: + name = p.support_channel_column("person") + people[name] = people[name].astype(str) + people.loc[people.index[0], name] = "other" + with pytest.raises(ValueError, match="PROPERTY_COMPLETION"): + routing.build_property_completion_routing(qualified, frame) + + +@pytest.mark.parametrize( + "fault", ["source_native", "source_axis", "knownness", "unknown_status"] +) +def test_description_identity_and_knownness_refuse(actual, fault): # noqa: F811 + live, qualified, _ = actual + value = _detached(qualified) + original = _asec_id(value) + if fault == "source_native": + value.asec_dividend_values.person.loc[original, "native_person_id"] += 1 + elif fault == "source_axis": + value.asec_dividend_values.person.index = ( + value.asec_dividend_values.person.index[::-1] + ) + elif fault == "knownness": + value.asec_dividend_values.person.loc[original, "DIV_VAL_amount_known"] = False + else: + value.asec_dividend_values.person.loc[original, "DIV_VAL_reporting_status"] = ( + "new_unreviewed_status" + ) + with pytest.raises(ValueError, match="PROPERTY_COMPLETION"): + routing.build_property_completion_routing(value, live.clone_population.frame) + + +def test_design_kind_required_and_zero_design_mass_is_not_missing(actual): # noqa: F811 + live, qualified, _ = actual + for kind, succeeds in ((WeightKind.IMPORTANCE, False), (WeightKind.DESIGN, True)): + value = _detached(qualified) + frame = value.source_frame + weights = np.ones(frame.n("household")) + weights[0] = 0 + altered = Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + {"household": Weights(weights, kind)}, + frame.strata.copy(deep=True), + ) + object.__setattr__(value, "source_frame", altered) + if succeeds: + result = routing.build_property_completion_routing( + value, live.clone_population.frame + ) + assert result.summary.loc["all", "union_household_design_mass"] == sum( + weights + ) + assert result.summary.loc["all", "person_count"] == len(value.origins) + assert result.person.original_household_design_weight.eq(0).any() + else: + with pytest.raises(ValueError, match="PROPERTY_COMPLETION.*DESIGN"): + routing.build_property_completion_routing( + value, live.clone_population.frame + ) + + +def test_large_native_ids_remain_exact_in_tables_and_json(actual): # noqa: F811 + live, qualified, _ = actual + value, clone = _detached(qualified), _detached(live.clone_population.frame) + original = _asec_id(value) + native = 2**53 + 17 + value.origins.loc[original, "native_person_id"] = native + value.shared_predictors.origins.loc[original, "native_person_id"] = native + document = json.loads(value.origin_document) + column_names = document["persons"]["columns"] + for row in document["persons"]["rows"]: + if row[column_names.index("person_id")] == int(original): + row[column_names.index("selected_receiving_person_id")] = native + object.__setattr__( + value, "origin_document", sources.shared.codec.encode_json(document) + ) + for table in ( + value.asec_interest_values.person, + value.asec_routing_values.person, + value.asec_dividend_values.person, + value.donor_basis.person, + ): + table.loc[original, "native_person_id"] = native + p = routing.provenance + for frame in (value.source_frame, value.shared_predictors.source_frame): + frame.person.loc[ + frame.person.person_id.eq(original), p.spine_source_id_column("person") + ] = native + clone.person.loc[ + clone.person[p.support_source_id_column("person")].eq(original), + p.spine_source_id_column("person"), + ] = native + result = routing.build_property_completion_routing(value, clone) + assert result.person.loc[original, "native_person_id"] == native + payload = json.loads(result.payload) + table = payload["tables"]["person"] + position = table["columns"].index("native_person_id") + row = table["index"].index(int(original)) + assert type(table["rows"][row][position]) is int + assert table["rows"][row][position] == native From d391cac0f0228bd409a4b4cdb9318549bdb575fc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:50:45 -0400 Subject: [PATCH 194/255] Classify retirement and property completion diagnostics --- .../microcosm-build/tests/test_us_spine_blindness.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 83ea016aa..a5d1e67ec 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -144,6 +144,10 @@ "current_asec_interest_source.py", # ASEC interest -> original money owner. # Composes qualified original source axes and DESIGN branches; no fit. "current_property_income_sources.py", + # Reviewed 2026-09-13: source-aware component availability and exact + # clone/original identity diagnostics. Fits nothing, assigns no amounts, + # and grants no source or complete-parent authority. + "current_property_completion_routing.py", # Original ASEC pension/disability/survivor details; no population model. "current_asec_retirement_detail_source.py", "current_asec_unemployment_source.py", # UC literal -> original ASEC ids. @@ -389,12 +393,17 @@ # Pure supplied-table property donor basis and exclusion diagnostics; # no source owner or source-spine access exemption "current_asec_property_basis.py", + # Pure retirement candidate accounting under explicit assumptions; + # source-spine and literal-accessor tripwires still apply. + "current_asec_retirement_basis.py", # Qualified ASEC retirement details and raw aggregate/slot comparisons "current_asec_retirement_detail_source.py", # Qualified UC receipt/amount observations, preserving unknown zeros "current_asec_unemployment_source.py", # Qualified original property donor/recipient branch composition "current_property_income_sources.py", + # Descriptive original component routes and DESIGN support; no model + "current_property_completion_routing.py", # Current UC/health amount qualification and exact two-clone attachment "current_survey_amounts.py", # Pure current-coverage recodes and original-person clone attachment @@ -3482,6 +3491,9 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: # Reviewed 2026-09-12: fixed dividend/interest/RINT-slot amount, status # and code families passed to private helpers; no source-channel reads. "current_asec_property_basis.py", + # Reviewed 2026-09-13: fixed retirement slot/status/family names and + # exact alignment keys; no source-channel or provenance selectors. + "current_asec_retirement_basis.py", # Static leaf/entity closure and engine metadata/default dictionaries, # not population provenance. Assumption execution remains disabled. "fiscal_leaf_policy.py", From 8aed36761662f694fc625e2fb1e095df239abd76 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 07:59:47 -0400 Subject: [PATCH 195/255] Stream the full-PUF population seal parts into the digest _population_stamp expanded every part into one tuple before hashing, so an object strata column retained one encoded scalar per row for the whole traversal. Feed the same parts, in the same order, through itertools.chain into the unchanged _digest, exactly as _table_stamp already does. No part, boundary, length prefix or byte changes; only peak retention does. The new invented test compares the streamed parts part-for-part against a verbatim copy of the pre-change tuple across empty, labelled-index, reversed-row, weightless and wide-object-strata populations, and requires twenty visible and hidden mutations to move both stamps together. Runtime untested in this lane by instruction: no test, import, build or simulation was executed here. Co-Authored-By: Claude Opus 5 --- .../us-population-seal-streaming.changed.md | 1 + .../us_runtime/graph_full_puf_enrichment.py | 48 +- .../test_us_graph_full_puf_seal_stream.py | 418 ++++++++++++++++++ 3 files changed, 447 insertions(+), 20 deletions(-) create mode 100644 changelog.d/us-population-seal-streaming.changed.md create mode 100644 packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py diff --git a/changelog.d/us-population-seal-streaming.changed.md b/changelog.d/us-population-seal-streaming.changed.md new file mode 100644 index 000000000..60002223e --- /dev/null +++ b/changelog.d/us-population-seal-streaming.changed.md @@ -0,0 +1 @@ +Stream the US full-PUF placement's in-process population seal into its digest instead of expanding every part into one tuple first, preserving each part, its order, its length prefix and its bytes exactly. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py index 567162543..7d1622997 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py @@ -145,29 +145,37 @@ def _table_stamp(table): def _population_stamp(population): - """In-process mutation seal only; never source or successor qualification.""" + """In-process mutation seal only; never source or successor qualification. + + The parts are streamed into the unchanged ``_digest``, exactly as + ``_table_stamp`` already does, rather than expanded into one tuple first. + Every part, its order and its length prefix are identical either way; only + the object strata's per-row encodings stop being retained all at once. + """ require(type(population) is Population, "FULL_PUF_POPULATION_REQUIRED") frame = population.frame return _digest( - ( - canonical_json( - { - "schema": asdict(frame.schema), - "entities": frame.entities, - "links": frame.links, - "metadata": store_ops._encode_frame_metadata(frame.metadata), - "mass_log": [asdict(r) for r in frame.mass_log], - "version": population.version, - "owners": sorted(population.owners.items()), - "weight_kind": tuple(population.weight_kind.items()), - "mass_ledger": [asdict(r) for r in population.mass_ledger], - } + chain( + ( + canonical_json( + { + "schema": asdict(frame.schema), + "entities": frame.entities, + "links": frame.links, + "metadata": store_ops._encode_frame_metadata(frame.metadata), + "mass_log": [asdict(r) for r in frame.mass_log], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": tuple(population.weight_kind.items()), + "mass_ledger": [asdict(r) for r in population.mass_ledger], + } + ), ), - *(_table_stamp(frame.table(e)).encode() for e in frame.entities), - *_axis_parts(frame.strata.index), - canonical_json(store_ops._axis_name_payload(frame.strata.name)), - *_series_parts(frame.strata), - *( + (_table_stamp(frame.table(e)).encode() for e in frame.entities), + _axis_parts(frame.strata.index), + (canonical_json(store_ops._axis_name_payload(frame.strata.name)),), + _series_parts(frame.strata), + ( canonical_json( ( e, @@ -179,7 +187,7 @@ def _population_stamp(population): + frame.weights_for(e).values.tobytes() for e in frame.weighted_entities ), - *( + ( canonical_json((e, str(a.dtype), a.shape)) + a.tobytes() for e, a in population.design_weights.items() ), diff --git a/packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py b/packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py new file mode 100644 index 000000000..a10d906bb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_full_puf_seal_stream.py @@ -0,0 +1,418 @@ +"""Invented parity for the streamed in-process population seal. + +Every fixture here is invented in this file: no genuine microdata, no source +admission, no store, archive or native read, and no tax engine. The property +under test is narrow and exact — streaming the seal's parts into the unchanged +``_digest`` must produce the byte-identical digest that the pre-change expanded +tuple produced — so each test compares the live implementation against +``_expanded_parts`` below, which is a verbatim copy of the pre-change tuple. +Mutation coverage is here too, because an optimization that preserved a digest +by *losing* a part would also pass a parity-only test. +""" + +from __future__ import annotations + +from dataclasses import asdict, replace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph import store as store_ops +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import MassRecord, Population + + +def _expanded_parts(population): + """The exact pre-change tuple: every part expanded before hashing.""" + + placement.require(type(population) is Population, "FULL_PUF_POPULATION_REQUIRED") + frame = population.frame + return ( + canonical_json( + { + "schema": asdict(frame.schema), + "entities": frame.entities, + "links": frame.links, + "metadata": store_ops._encode_frame_metadata(frame.metadata), + "mass_log": [asdict(r) for r in frame.mass_log], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": tuple(population.weight_kind.items()), + "mass_ledger": [asdict(r) for r in population.mass_ledger], + } + ), + *(placement._table_stamp(frame.table(e)).encode() for e in frame.entities), + *placement._axis_parts(frame.strata.index), + canonical_json(store_ops._axis_name_payload(frame.strata.name)), + *placement._series_parts(frame.strata), + *( + canonical_json( + ( + e, + frame.weights_for(e).kind, + str(frame.weights_for(e).values.dtype), + frame.weights_for(e).values.shape, + ) + ) + + frame.weights_for(e).values.tobytes() + for e in frame.weighted_entities + ), + *( + canonical_json((e, str(a.dtype), a.shape)) + a.tobytes() + for e, a in population.design_weights.items() + ), + ) + + +def _expanded_population_stamp(population): + return placement._digest(_expanded_parts(population)) + + +def _object_array(values): + """An object ndarray built element-wise, so numpy never re-types a leaf.""" + + array = np.empty(len(values), dtype=object) + for position, value in enumerate(values): + array[position] = value + return array + + +def _person_table( + *, + person_id=(1, 2, 3, 4), + household_of=(10, 10, 20, 30), + amount=(-0.0, 1.5, float("inf"), 2.0), + hidden=(7, 8, 9, 10), + mask=(False, True, False, True), + text=("a", "b", "c", "d"), + leaves=("x", 5, None, 2.5), + index=None, +): + table = pd.DataFrame( + { + "person_id": np.array(person_id, dtype=np.int64), + "person_household_id": np.array(household_of, dtype=np.int64), + "amount": np.array(amount, dtype=np.float64), + # Hidden bytes under the null mask are part of this physical seal. + "nullable": pd.arrays.IntegerArray( + np.array(hidden, dtype=np.int64), np.array(mask, dtype=np.bool_) + ), + "text": pd.array( + list(text), dtype=pd.StringDtype(storage="python", na_value=pd.NA) + ), + }, + index=pd.RangeIndex(len(person_id)) if index is None else index, + ) + table["leaf"] = _object_array(leaves) + return table + + +def _household_table(rent=(0.5, 1.5, 2.5)): + return pd.DataFrame( + { + "household_id": np.array([10, 20, 30], dtype=np.int64), + "rent": np.array(rent, dtype=np.float64), + } + ) + + +def _strata(labels=("s1", "s1", "s2", "s2"), index=None): + return pd.Series( + _object_array(labels), + index=pd.RangeIndex(len(labels)) if index is None else index, + ) + + +def _mass_log(): + return ( + MassChangeRecord( + entity="household", + old_total=6.0, + new_total=6.0, + declared_factor=None, + reason="invented", + ), + ) + + +def _mass_ledger(after_total=4.0): + return ( + MassRecord( + node_id="invented_node", + operation="rewrite", + policy="preserve", + before_total=4.0, + after_total=after_total, + before_by_stratum=(("s1", 2.0), ("s2", 2.0)), + after_by_stratum=(("s1", 2.0), ("s2", 2.0)), + entity="person", + ), + ) + + +def _frame( + *, + person=None, + household=None, + strata=None, + weights=None, + metadata=None, + mass_log=None, +): + return Frame( + { + "person": _person_table() if person is None else person, + "household": _household_table() if household is None else household, + }, + EntitySchema(group_entities=("household",)), + ( + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.DESIGN)} + if weights is None + else weights + ), + _strata() if strata is None else strata, + mass_log=_mass_log() if mass_log is None else mass_log, + metadata={"note": "invented"} if metadata is None else metadata, + ) + + +def _population(*, frame=None, version="v1", design=None, mass_ledger=None): + return Population.from_frame( + _frame() if frame is None else frame, + version, + mass_ledger=_mass_ledger() if mass_ledger is None else mass_ledger, + design_weights=( + {"person": np.array([1.0, 2.0, 3.0, 4.0])} if design is None else design + ), + ) + + +def _empty_population(): + person = pd.DataFrame( + { + "person_id": np.array([], dtype=np.int64), + "person_household_id": np.array([], dtype=np.int64), + "amount": np.array([], dtype=np.float64), + "nullable": pd.arrays.IntegerArray( + np.array([], dtype=np.int64), np.array([], dtype=np.bool_) + ), + "text": pd.array( + [], dtype=pd.StringDtype(storage="python", na_value=pd.NA) + ), + } + ) + person["leaf"] = _object_array(()) + household = pd.DataFrame( + { + "household_id": np.array([], dtype=np.int64), + "rent": np.array([], dtype=np.float64), + } + ) + frame = Frame( + {"person": person, "household": household}, + EntitySchema(group_entities=("household",)), + {}, + pd.Series(_object_array(()), index=person.index), + ) + return Population.from_frame(frame, "v1") + + +def _labelled_index_population(): + index = pd.Index([11, 12, 13, 14], name="row") + return _population( + frame=_frame(person=_person_table(index=index), strata=_strata(index=index)) + ) + + +def _reversed_row_population(): + return _population( + frame=_frame( + person=_person_table( + person_id=(4, 3, 2, 1), + household_of=(30, 20, 10, 10), + amount=(2.0, float("inf"), 1.5, -0.0), + hidden=(10, 9, 8, 7), + mask=(True, False, True, False), + text=("d", "c", "b", "a"), + leaves=(2.5, None, 5, "x"), + ), + strata=_strata(labels=("s2", "s2", "s1", "s1")), + ), + design={"person": np.array([4.0, 3.0, 2.0, 1.0])}, + ) + + +def _no_design_population(): + return _population(design={}) + + +def _wide_object_strata_population(): + """The retention case the streaming change targets: many object leaves.""" + + rows = 512 + person = pd.DataFrame( + { + "person_id": np.arange(1, rows + 1, dtype=np.int64), + "person_household_id": np.arange(1, rows + 1, dtype=np.int64), + "amount": np.linspace(-1.0, 1.0, rows), + } + ) + household = pd.DataFrame({"household_id": np.arange(1, rows + 1, dtype=np.int64)}) + frame = Frame( + {"person": person, "household": household}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(rows), WeightKind.DESIGN)}, + pd.Series(_object_array([f"s{position}" for position in range(rows)])), + ) + return Population.from_frame(frame, "v1") + + +_POPULATIONS = { + "default": _population, + "empty": _empty_population, + "labelled-index": _labelled_index_population, + "reversed-rows": _reversed_row_population, + "no-design-weights": _no_design_population, + "wide-object-strata": _wide_object_strata_population, +} + + +@pytest.mark.parametrize("name", sorted(_POPULATIONS)) +def test_streamed_population_stamp_matches_the_expanded_tuple(name): + population = _POPULATIONS[name]() + assert placement._population_stamp(population) == _expanded_population_stamp( + population + ) + + +@pytest.mark.parametrize("name", sorted(_POPULATIONS)) +def test_streamed_parts_are_the_expanded_parts_part_for_part(name, monkeypatch): + """Same parts, same order, same bytes — and never materialized at once.""" + + population = _POPULATIONS[name]() + captured = {} + original = placement._digest + + def recording_digest(parts): + captured["one_shot"] = iter(parts) is parts + captured["materialized"] = isinstance(parts, (tuple, list, set, frozenset)) + materialized = tuple(parts) + captured["parts"] = materialized + return original(materialized) + + monkeypatch.setattr(placement, "_digest", recording_digest) + digest = placement._population_stamp(population) + monkeypatch.undo() + + assert captured["one_shot"] is True + assert captured["materialized"] is False + assert captured["parts"] == _expanded_parts(population) + assert digest == _expanded_population_stamp(population) + + +def _mutations(): + """(label, population) pairs that must every one move the seal.""" + + yield ( + "visible-value", + _population( + frame=_frame(person=_person_table(amount=(-0.0, 1.5, float("inf"), 2.5))) + ), + ) + yield ( + "negative-zero", + _population( + frame=_frame(person=_person_table(amount=(0.0, 1.5, float("inf"), 2.0))) + ), + ) + yield ( + "infinity-sign", + _population( + frame=_frame(person=_person_table(amount=(-0.0, 1.5, float("-inf"), 2.0))) + ), + ) + yield ( + "hidden-masked-data", + _population(frame=_frame(person=_person_table(hidden=(7, 999, 9, 10)))), + ) + yield ( + "null-mask", + _population(frame=_frame(person=_person_table(mask=(False, True, True, True)))), + ) + yield ( + "string-value", + _population(frame=_frame(person=_person_table(text=("a", "b", "c", "z")))), + ) + yield ( + "object-leaf-type", + _population(frame=_frame(person=_person_table(leaves=("x", 5.0, None, 2.5)))), + ) + yield "row-order", _reversed_row_population() + yield ( + "strata-label", + _population(frame=_frame(strata=_strata(labels=("s1", "s1", "s2", "s3")))), + ) + yield ( + "strata-order", + _population(frame=_frame(strata=_strata(labels=("s2", "s1", "s2", "s1")))), + ) + yield "axis-labels", _labelled_index_population() + yield ( + "group-table-value", + _population(frame=_frame(household=_household_table(rent=(0.5, 1.5, 3.5)))), + ) + yield ( + "weights", + _population( + frame=_frame( + weights={ + "household": Weights(np.array([1.0, 2.0, 4.0]), WeightKind.DESIGN) + } + ) + ), + ) + yield ( + "weight-kind", + _population( + frame=_frame( + weights={ + "household": Weights( + np.array([1.0, 2.0, 3.0]), WeightKind.IMPORTANCE + ) + } + ) + ), + ) + yield "metadata", _population(frame=_frame(metadata={"note": "changed"})) + yield ( + "frame-mass-log", + _population( + frame=_frame(mass_log=(replace(_mass_log()[0], reason="different"),)) + ), + ) + yield ( + "design-weights", + _population(design={"person": np.array([1.0, 2.0, 3.0, 5.0])}), + ) + yield "no-design-weights", _no_design_population() + yield "version", _population(version="v2") + yield "mass-ledger", _population(mass_ledger=_mass_ledger(after_total=5.0)) + + +@pytest.mark.parametrize("label,mutated", list(_mutations()), ids=lambda value: value) +def test_every_mutation_moves_the_streamed_and_expanded_stamps_together(label, mutated): + base = _population() + baseline = placement._population_stamp(base) + assert baseline == _expanded_population_stamp(base) + + moved = placement._population_stamp(mutated) + assert moved == _expanded_population_stamp(mutated) + assert moved != baseline, label + + +def test_the_stamp_still_refuses_a_bare_frame(): + with pytest.raises(ValueError, match="FULL_PUF_POPULATION_REQUIRED"): + placement._population_stamp(_frame()) From 8a0b52727e03bfaf33ca214c6e2067682564d51b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 13 Sep 2026 08:05:26 -0400 Subject: [PATCH 196/255] Take whole series by slice in the physical storage parts _storage_parts took a row-aligned bool mask, so a whole-series caller built an all-True selector and paid for a null-selection copy and a value-selection copy of every row. Accept slice(None) as well: it selects the same rows in the same order, and every payload below is taken through np.ascontiguousarray or an in-order walk, so the bytes are identical. The isna() call also moves below the masked-storage branch, which carries its own nulls and never used it. Converted here: storage_equal's whole-series path and the full-PUF placement's _series_parts, the site the source review named. Subset masks are untouched, including the hidden bytes under a nullable mask, and a wrong-length mask still fails closed with IndexError. Tests compare the live helper against verbatim copies of the pre-change bodies across empty, contiguous, strided, reversed, frame-sliced, non-native-endian, dense, masked, string and object series, with signed zero, subnormals, infinities and two distinct NaN payloads, and require visible and hidden mutations to move the bytes. Runtime untested in this lane by instruction: no test, import, build or simulation was executed here. Co-Authored-By: Claude Opus 5 --- ...raph-storage-parts-whole-series.changed.md | 1 + .../us_runtime/graph_full_puf_enrichment.py | 6 +- .../test_us_graph_full_puf_seal_selection.py | 224 +++++++++++ .../src/microcosm/graph/population.py | 23 +- .../test_graph_storage_parts_whole_series.py | 373 ++++++++++++++++++ 5 files changed, 621 insertions(+), 6 deletions(-) create mode 100644 changelog.d/graph-storage-parts-whole-series.changed.md create mode 100644 packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py create mode 100644 packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py diff --git a/changelog.d/graph-storage-parts-whole-series.changed.md b/changelog.d/graph-storage-parts-whole-series.changed.md new file mode 100644 index 000000000..afe3d3e1c --- /dev/null +++ b/changelog.d/graph-storage-parts-whole-series.changed.md @@ -0,0 +1 @@ +Select whole series with `slice(None)` instead of an all-True mask when folding physical storage parts, and skip the null recomputation that masked storage never uses, preserving the emitted value bytes, null bytes and subset-selection behaviour exactly. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py index 7d1622997..fde7bcc5f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_full_puf_enrichment.py @@ -121,9 +121,9 @@ def _series_parts(series): # Portable typed scalar bytes, never Python object pointers. yield from (store_ops._encode_object_scalar(v) for v in series) else: - yield from population_ops._storage_parts( - series, np.ones(len(series), dtype=np.bool_) - ) + # Whole series: the slice selects every row in order, so no all-True + # selector or selection copy is built. The emitted bytes are identical. + yield from population_ops._storage_parts(series, slice(None)) def _axis_parts(axis): diff --git a/packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py b/packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py new file mode 100644 index 000000000..b5abff7a0 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_full_puf_seal_selection.py @@ -0,0 +1,224 @@ +"""Whole-series selection parity for the full-PUF placement's seal parts. + +Invented series and tables only: no genuine microdata, no source admission, no +store, archive or native read, and no tax engine. ``_series_parts`` now asks +the storage helper for the whole series with ``slice(None)`` instead of +building an all-True selector; the emitted parts, their order and their bytes +must be exactly what the pre-change all-True form emitted, which is what +``_reference_series_parts`` below preserves verbatim. +""" + +from __future__ import annotations + +import struct + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_full_puf_enrichment as placement +from microcosm.graph import population as population_ops +from microcosm.graph import store as store_ops +from microcosm.graph.canonical import canonical_json + + +def _reference_series_parts(series): + """The exact pre-change body, with its all-True whole-series selector.""" + + yield repr(series.dtype).encode() + if pd.api.types.is_object_dtype(series.dtype): + # Portable typed scalar bytes, never Python object pointers. + yield from (store_ops._encode_object_scalar(v) for v in series) + else: + yield from population_ops._storage_parts( + series, np.ones(len(series), dtype=np.bool_) + ) + + +def _reference_axis_parts(axis): + placement.require(not isinstance(axis, pd.MultiIndex), "FULL_PUF_AXIS") + yield type(axis).__name__.encode() + yield canonical_json(store_ops._axis_name_payload(axis.name)) + yield from _reference_series_parts(pd.Series(axis.array, copy=False)) + + +def _reference_table_stamp(table): + parts = [ + *_reference_axis_parts(table.index), + *_reference_axis_parts(table.columns), + str(table.flags.allows_duplicate_labels).encode(), + *(part for column in table for part in _reference_series_parts(table[column])), + ] + return placement._digest(parts) + + +def _nan(payload): + return struct.unpack(" bytes: return bytes(payload) -def _storage_parts(series: pd.Series, selected: np.ndarray) -> tuple[bytes, bytes]: - nulls = series.isna().to_numpy(dtype=np.bool_, copy=False)[selected] +def _storage_parts( + series: pd.Series, selected: np.ndarray | slice +) -> tuple[bytes, bytes]: + """Physical value bytes and null bytes for the ``selected`` rows. + + ``selected`` is a row-aligned bool mask, or ``slice(None)`` for the whole + series. The two forms are byte-identical: an all-True mask selects a copy + in row order and ``slice(None)`` a view in the same order, and every byte + payload below is taken either through ``np.ascontiguousarray`` or by an + in-order walk. Whole-series callers pass the slice so that no all-True + selector, null-selection copy or value-selection copy is built at all. + """ + array = series.array data = getattr(array, "_data", None) mask = getattr(array, "_mask", None) if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + # Masked storage carries its own nulls, so the isna() below — which + # would only rebuild this same mask — stays out of this branch. values = np.ascontiguousarray(data[selected]).tobytes() bitmap = np.ascontiguousarray(mask[selected]).tobytes() return values, bitmap + nulls = series.isna().to_numpy(dtype=np.bool_, copy=False)[selected] if isinstance(series.dtype, pd.StringDtype): payload = bytearray() for value, is_null in zip( diff --git a/packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py b/packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py new file mode 100644 index 000000000..bb40480ae --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_storage_parts_whole_series.py @@ -0,0 +1,373 @@ +"""Whole-series selection parity for the physical storage parts. + +Invented series only: no data file, no store, no archive, no engine. Every +case compares the live ``_storage_parts`` against ``_reference_storage_parts`` +below — a verbatim copy of the pre-change body — in both selector forms, and +every mutation case requires visible *and* hidden bytes to move the parts. +``slice(None)`` and an all-True mask must be byte-identical; a subset mask must +keep behaving exactly as it did. +""" + +from __future__ import annotations + +import struct + +import numpy as np +import pandas as pd +import pytest + +from microcosm.graph import population as population_ops +from microcosm.graph.population import ( + PopulationError, + _object_storage_values, + _storage_parts, + storage_equal, +) + + +def _reference_storage_parts(series, selected): + """The exact pre-change body, kept verbatim as the parity reference.""" + + nulls = series.isna().to_numpy(dtype=np.bool_, copy=False)[selected] + array = series.array + data = getattr(array, "_data", None) + mask = getattr(array, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + values = np.ascontiguousarray(data[selected]).tobytes() + bitmap = np.ascontiguousarray(mask[selected]).tobytes() + return values, bitmap + if isinstance(series.dtype, pd.StringDtype): + payload = bytearray() + for value, is_null in zip( + series.to_numpy(dtype=object, copy=False)[selected], nulls, strict=True + ): + if is_null: + payload.extend((0).to_bytes(8, "little")) + else: + encoded = str(value).encode("utf-8") + payload.extend((len(encoded) + 1).to_bytes(8, "little")) + payload.extend(encoded) + return bytes(payload), np.ascontiguousarray(nulls).tobytes() + values = series.to_numpy(copy=False)[selected] + if values.dtype == object: + return ( + _object_storage_values(values), + np.ascontiguousarray(nulls).tobytes(), + ) + return ( + np.ascontiguousarray(values).tobytes(), + np.ascontiguousarray(nulls).tobytes(), + ) + + +def _nan(payload): + """A float64 NaN with an exact, distinguishable payload.""" + + return struct.unpack(" Date: Sun, 13 Sep 2026 08:08:14 -0400 Subject: [PATCH 197/255] Convert the remaining whole-series storage-parts call sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five US seals still built an all-True selector per series before folding physical storage parts. Each passes slice(None) now, which the helper accepts as the whole series. Every one of them passes a selector of exactly the series' own length today, so the selected rows and their order are unchanged and the digests are byte-identical; current_survey_predictors uses len(table) for series that are all exactly len(table) rows. This is deliberately a separate commit from the helper change: it moves five more modules' source identity for no benefit that has been measured yet, so it can be reverted on its own. The new test compares the two seals whose inputs are cheaply invented — population_input_coverage._series_stamp and survey_origin_budget's storage identity — against verbatim copies of their pre-change bodies, and holds all five sites, including the three whose inputs are not, with a static guard that fails if a call goes back to an all-True selector. Runtime untested in this lane by instruction: no test, import, build or simulation was executed here. Co-Authored-By: Claude Opus 5 --- ...us-whole-series-selection-sites.changed.md | 1 + .../us_runtime/current_survey_predictors.py | 5 +- .../us_runtime/population_input_coverage.py | 3 +- .../us_runtime/puf55_survey_recipients.py | 5 +- .../us_runtime/survey_atomic_geography.py | 3 +- .../build/us_runtime/survey_origin_budget.py | 4 +- .../test_us_whole_series_selection_sites.py | 263 ++++++++++++++++++ 7 files changed, 275 insertions(+), 9 deletions(-) create mode 100644 changelog.d/us-whole-series-selection-sites.changed.md create mode 100644 packages/microcosm-build/tests/test_us_whole_series_selection_sites.py diff --git a/changelog.d/us-whole-series-selection-sites.changed.md b/changelog.d/us-whole-series-selection-sites.changed.md new file mode 100644 index 000000000..e14368e80 --- /dev/null +++ b/changelog.d/us-whole-series-selection-sites.changed.md @@ -0,0 +1 @@ +Take the whole series by slice at the remaining US in-process seal sites (atomic geography, input coverage, origin budget, current survey predictors and PUF55 recipients), preserving each seal's emitted bytes exactly. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py index 9d4768a0b..44190a51b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -278,9 +278,10 @@ def _qualified_seal(value): "shape": list(table.shape), } ) - selected = np.ones(len(table), dtype=np.bool_) + # Every series below is exactly len(table) rows, so the whole-series + # slice selects the same rows in the same order, byte for byte. parts = tuple( - (str(series.dtype), geography._storage_parts(series, selected)) + (str(series.dtype), geography._storage_parts(series, slice(None))) for series in ( pd.Series(table.index.to_numpy(copy=False)), *(table[c] for c in table), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py index f02dce43f..e4a0911c4 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py @@ -142,7 +142,8 @@ def _series_stamp(series): digest.update(len(part).to_bytes(8, "little")) digest.update(part) else: - for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, "little")) digest.update(part) return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py index 527480ffb..264a57a5f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py @@ -93,9 +93,8 @@ def _table_digest(table): "OBJECT_STRING_STORAGE", ) continue - for part in financial.reconstruction._storage_parts( - series, np.ones(len(series), dtype=np.bool_) - ): + # Whole series: the slice selects every row in order, byte for byte. + for part in financial.reconstruction._storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, "big")) digest.update(part) return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py index 63e6efcea..2beb174dc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -258,7 +258,8 @@ def _population_stamp(population): ), frame.strata, ): - for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, "little")) digest.update(part) for entity, values in population.design_weights.items(): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py index 6a72b771d..bdf2029fe 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -283,8 +283,8 @@ def _physical_nonweight_identity(frame): ), frame.strata, ): - selected = np.ones(len(series), dtype=np.bool_) - for part in _storage_parts(series, selected): + # Whole series: the slice selects every row in order, byte for byte. + for part in _storage_parts(series, slice(None)): digest.update(len(part).to_bytes(8, "little")) digest.update(part) return digest.hexdigest() diff --git a/packages/microcosm-build/tests/test_us_whole_series_selection_sites.py b/packages/microcosm-build/tests/test_us_whole_series_selection_sites.py new file mode 100644 index 000000000..262b98a60 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_whole_series_selection_sites.py @@ -0,0 +1,263 @@ +"""Whole-series selection parity at the remaining US in-process seal sites. + +Invented series and frames only: no genuine microdata, no source admission, no +store, archive or native read, and no tax engine. Two of the converted seals +are pure enough to compare byte for byte against verbatim copies of their +pre-change bodies; every converted site — including the three whose inputs are +not cheaply invented — is held by the static guard at the end, which fails if +any call here goes back to building an all-True selector. +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import struct + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_survey_predictors as predictors +from microcosm.build.us_runtime import population_input_coverage as coverage +from microcosm.build.us_runtime import puf55_survey_recipients as recipients +from microcosm.build.us_runtime import survey_atomic_geography as geography +from microcosm.build.us_runtime import survey_origin_budget as budget +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import _storage_parts +from microcosm.graph.store import _axis_name_payload, _encode_object_scalar + +CONVERTED_MODULES = ( + coverage, + geography, + budget, + predictors, + recipients, +) + + +def _reference_series_stamp(series): + """The exact pre-change body of ``population_input_coverage._series_stamp``.""" + + digest = hashlib.sha256() + digest.update(canonical_json((str(series.dtype), _axis_name_payload(series.name)))) + if pd.api.types.is_object_dtype(series.dtype): + # Typed scalar encoding, never process-specific PyObject addresses. + for value in series: + part = _encode_object_scalar(value) + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + else: + for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _reference_physical_nonweight_identity(frame): + """The exact pre-change body of ``survey_origin_budget``'s storage seal.""" + + digest = hashlib.sha256() + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + selected = np.ones(len(series), dtype=np.bool_) + for part in _storage_parts(series, selected): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _nan(payload): + return struct.unpack("